From c36501cde51e0ac75bd42671f619e311795f3f04 Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Thu, 24 Oct 2024 09:42:50 -0600
Subject: [PATCH 001/165] Add script for updating test input json files

---
 scripts/firecloud_api/UpdateTestInputs.py | 69 +++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 scripts/firecloud_api/UpdateTestInputs.py

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
new file mode 100644
index 0000000000..b2db257e6b
--- /dev/null
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -0,0 +1,69 @@
+import argparse
+import json
+
+
+def update_test_inputs(test_inputs, truth_path, test_path):
+    # Update the test inputs JSON to work with the test wrapper WDL
+    # The test wrapper WDL runs the pipeline WDL and verifies the results
+    # The test wrapper WDL requires the following inputs:
+    # - truth_path: The path to the truth data
+    # - test_path: The path to the test data
+    # The test inputs JSON will be updated to include these inputs
+
+    # Add the truth_path and test_path to the test inputs JSON
+    test_inputs["truth_path"] = truth_path
+    test_inputs["test_path"] = test_path
+
+    # Save the updated test inputs JSON
+    with open("updated_test_inputs.json", 'w') as file:
+        json.dump(test_inputs, file, indent=4)
+
+    print("Test inputs JSON updated with truth_path and test_path")
+def main():
+    description = """This script updates the test inputs JSON to work with the test wrapper WDL, 
+    which runs the pipeline and verification"""
+
+    parser = argparse.ArgumentParser(description=description)
+
+    parser.add_argument(
+        "--truth_path",
+        dest="truth_path",
+        required=True,
+        help="The base path where the truth data is stored",
+    )
+
+    parser.add_argument(
+        "--test_path",
+        dest="test_path",
+        required=True,
+        help="The base path where the test data will be stored",
+    )
+
+    parser.add_argument(
+        "--json_inputs",
+        dest="json_inputs",
+        required=True,
+        help="The JSON file containing the test inputs, formatted to run the pipeline WDL. "
+             "This will be updated to run the wrapper Test WDL",
+    )
+
+    parser.add_argument(
+        "--update_truth",
+        dest="update_truth",
+        default=False,
+        required=False,
+        choices=[True, False],
+        help="Boolean flag to update the truth data. If True, the truth data will be updated with the test data. ",
+    )
+
+    args = parser.parse_args()
+
+    # Load the JSON file containing the test inputs
+    with open(args.json_inputs, 'r') as file:
+        test_inputs = json.load(file)
+
+    # Update the test inputs to work with the test wrapper WDL
+    update_test_inputs(test_inputs, args.truth_path, args.test_path)
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file

From c620aee40a57686f42e448448e71a0033939ce9c Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Tue, 29 Oct 2024 10:20:03 -0600
Subject: [PATCH 002/165] Update json formatting script

---
 scripts/firecloud_api/UpdateTestInputs.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index b2db257e6b..b6f39a135c 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -10,15 +10,28 @@ def update_test_inputs(test_inputs, truth_path, test_path):
     # - test_path: The path to the test data
     # The test inputs JSON will be updated to include these inputs
 
+    # Get the pipeline name from the test inputs JSON
+    pipeline_name = next(iter(test_inputs)).split('.')[0]
+
+    #Append "Test" in front of the pipeline name
+    test_name = f"Test{pipeline_name}"
+
+    # Update all keys in the json file to replace the pipeline name with the test name
+    for key in list(test_inputs.keys()):
+        new_key = key.replace(pipeline_name, test_name)
+        test_inputs[new_key] = test_inputs.pop(key)
+
     # Add the truth_path and test_path to the test inputs JSON
-    test_inputs["truth_path"] = truth_path
-    test_inputs["test_path"] = test_path
+    test_inputs[f"{test_name}.truth_path"] = truth_path
+    test_inputs[f"{test_name}.test_path"] = test_path
 
     # Save the updated test inputs JSON
     with open("updated_test_inputs.json", 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
     print("Test inputs JSON updated with truth_path and test_path")
+
+
 def main():
     description = """This script updates the test inputs JSON to work with the test wrapper WDL, 
     which runs the pipeline and verification"""
@@ -40,8 +53,8 @@ def main():
     )
 
     parser.add_argument(
-        "--json_inputs",
-        dest="json_inputs",
+        "--inputs_json",
+        dest="inputs_json",
         required=True,
         help="The JSON file containing the test inputs, formatted to run the pipeline WDL. "
              "This will be updated to run the wrapper Test WDL",
@@ -59,7 +72,7 @@ def main():
     args = parser.parse_args()
 
     # Load the JSON file containing the test inputs
-    with open(args.json_inputs, 'r') as file:
+    with open(args.inputs_json, 'r') as file:
         test_inputs = json.load(file)
 
     # Update the test inputs to work with the test wrapper WDL

From 23f7f8371158ca4897d3bcf61f83ebef07fcfef5 Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Wed, 13 Nov 2024 10:36:19 -0700
Subject: [PATCH 003/165] Update json formatting script

---
 scripts/firecloud_api/UpdateTestInputs.py     | 27 +++++++-----
 .../broad/TerraCopyFilesFromCloudToCloud.wdl  | 44 +++++++++++++++++++
 .../test-wdls/TestIlluminaGenotypingArray.wdl | 12 ++---
 3 files changed, 64 insertions(+), 19 deletions(-)
 create mode 100644 tasks/broad/TerraCopyFilesFromCloudToCloud.wdl

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index b6f39a135c..576f0fb370 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -1,19 +1,27 @@
 import argparse
 import json
+import os
 
 
-def update_test_inputs(test_inputs, truth_path, test_path):
+def update_test_inputs(inputs_json, truth_path, test_path, update_truth):
     # Update the test inputs JSON to work with the test wrapper WDL
     # The test wrapper WDL runs the pipeline WDL and verifies the results
     # The test wrapper WDL requires the following inputs:
     # - truth_path: The path to the truth data
     # - test_path: The path to the test data
+    # - update_truth: Boolean indicating whether truth should be updated, default is False
     # The test inputs JSON will be updated to include these inputs
 
+    with open(inputs_json, 'r') as file:
+        test_inputs = json.load(file)
+
+    # get the sample name from the test inputs JSON, this is needed for tests with multiple inputs
+    sample_name = os.path.splitext(os.path.basename(inputs_json))[0]
+
     # Get the pipeline name from the test inputs JSON
     pipeline_name = next(iter(test_inputs)).split('.')[0]
 
-    #Append "Test" in front of the pipeline name
+    # Append "Test" in front of the pipeline name
     test_name = f"Test{pipeline_name}"
 
     # Update all keys in the json file to replace the pipeline name with the test name
@@ -22,11 +30,13 @@ def update_test_inputs(test_inputs, truth_path, test_path):
         test_inputs[new_key] = test_inputs.pop(key)
 
     # Add the truth_path and test_path to the test inputs JSON
-    test_inputs[f"{test_name}.truth_path"] = truth_path
-    test_inputs[f"{test_name}.test_path"] = test_path
+    test_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
+    test_inputs[f"{test_name}.test_path"] = f"{test_path}/{sample_name}/"
+    test_inputs[f"{test_name}.update_truth"] = update_truth
 
     # Save the updated test inputs JSON
-    with open("updated_test_inputs.json", 'w') as file:
+    output_name = f"updated_{sample_name}.json"
+    with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
     print("Test inputs JSON updated with truth_path and test_path")
@@ -71,12 +81,9 @@ def main():
 
     args = parser.parse_args()
 
-    # Load the JSON file containing the test inputs
-    with open(args.inputs_json, 'r') as file:
-        test_inputs = json.load(file)
-
     # Update the test inputs to work with the test wrapper WDL
-    update_test_inputs(test_inputs, args.truth_path, args.test_path)
+    update_test_inputs(args.inputs_json, args.truth_path, args.test_path, args.update_truth)
+
 
 if __name__ == "__main__":
     main()
\ No newline at end of file
diff --git a/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
new file mode 100644
index 0000000000..05415985ee
--- /dev/null
+++ b/tasks/broad/TerraCopyFilesFromCloudToCloud.wdl
@@ -0,0 +1,44 @@
+version 1.0
+
+## Copyright Broad Institute, 2024
+##
+## This WDL defines tasks used for moving files from place to place on Terra Platform.
+##
+## Runtime parameters are often optimized for Broad's Google Cloud Platform implementation.
+## For program versions, see docker containers.
+##
+## LICENSING :
+## This script is released under the WDL source code license (BSD-3) (see LICENSE in
+## https://github.com/broadinstitute/wdl). Note however that the programs it calls may
+## be subject to different licenses. Users are responsible for checking that they are
+## authorized to run all programs before running this script. Please see the docker
+## page at https://hub.docker.com/r/broadinstitute/genomes-in-the-cloud/ for detailed
+## licensing information pertaining to the included programs.
+
+task TerraCopyFilesFromCloudToCloud {
+  input {
+    Array[String] files_to_copy
+    String destination_cloud_path
+  }
+
+  command {
+    set -euo pipefail
+
+    gcloud config set storage/process_count 16
+    gcloud config set storage/thread_count  2
+
+    gcloud storage cp ~{sep=' ' files_to_copy} ~{destination_cloud_path}
+  }
+
+  output {
+    Boolean done = true
+  }
+
+  runtime {
+    memory: "16 GiB"
+    cpu: "1"
+    disks: "local-disk 32 HDD"
+    docker: "gcr.io/google.com/cloudsdktool/google-cloud-cli:499.0.0-slim"
+    preemptible: 3
+  }
+}
diff --git a/verification/test-wdls/TestIlluminaGenotypingArray.wdl b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
index f70710653f..1fb3ad419f 100644
--- a/verification/test-wdls/TestIlluminaGenotypingArray.wdl
+++ b/verification/test-wdls/TestIlluminaGenotypingArray.wdl
@@ -4,7 +4,7 @@ version 1.0
 import "../../pipelines/broad/genotyping/illumina/IlluminaGenotypingArray.wdl" as IlluminaGenotypingArray
 import "../../verification/VerifyIlluminaGenotypingArray.wdl" as VerifyIlluminaGenotypingArray
 import "../../tasks/broad/Utilities.wdl" as Utilities
-import "../../tasks/broad/CopyFilesFromCloudToCloud.wdl" as Copy
+import "../../tasks/broad/TerraCopyFilesFromCloudToCloud.wdl" as Copy
 
 workflow TestIlluminaGenotypingArray {
 
@@ -46,8 +46,6 @@ workflow TestIlluminaGenotypingArray {
       String truth_path
       String results_path
       Boolean update_truth
-      String vault_token_path
-      String google_account_vault_path
     }
 
     meta {
@@ -127,21 +125,17 @@ workflow TestIlluminaGenotypingArray {
     ])
 
     # Copy results of pipeline to test results bucket
-    call Copy.CopyFilesFromCloudToCloud as CopyToTestResults {
+    call Copy.TerraCopyFilesFromCloudToCloud as CopyToTestResults {
       input:
         files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-        vault_token_path          = vault_token_path,
-        google_account_vault_path = google_account_vault_path,
         destination_cloud_path    = results_path
     }
   
     # If updating truth then copy output to truth bucket
     if (update_truth){
-      call Copy.CopyFilesFromCloudToCloud as CopyToTruth {
+      call Copy.TerraCopyFilesFromCloudToCloud as CopyToTruth {
         input: 
           files_to_copy             = flatten([pipeline_outputs, pipeline_metrics]),
-          vault_token_path          = vault_token_path,
-          google_account_vault_path = google_account_vault_path,
           destination_cloud_path    = truth_path
       }
     }

From f9e373ec3b6c3685ef16173b64c224112cdbf3b8 Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Thu, 14 Nov 2024 15:11:14 -0700
Subject: [PATCH 004/165] More updates - still untested

---
 .../test_illumina_genotyping_array.yml        | 60 ++++++++++++++++---
 scripts/firecloud_api/UpdateTestInputs.py     | 27 +++++----
 scripts/firecloud_api/firecloud_api.py        | 37 +++++++++++-
 3 files changed, 104 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e1774240bb..cbe056f6db 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -10,11 +10,16 @@ on:
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
     paths:
       - 'pipelines/broad/genotyping/illumina/**'
       - 'tasks/**'
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
+
+
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
@@ -22,6 +27,10 @@ on:
         description: 'Use call cache (default: true)'
         required: false
         default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
 env:
   PROJECT_NAME: WARP
   # Github repo name
@@ -62,33 +71,48 @@ jobs:
     - name: Submit job, poll status, and get outputs
       id: pipeline_run
       run: |
-          # Set these environment variables
+          # Set common environment variables
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
-          PIPELINE_NAME="IlluminaGenotypingArray"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
+          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
+          TEST_TYPE="Plumbing"  # Placeholder for now
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          
+          ########################################
+          # SET PIPELINE SPECIFIC VARIABLES HERE #
+          ########################################
+          PIPELINE_NAME="IlluminaGenotypingArray"
+          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          # TODO: Need to set the truth and result paths appropriately
+          TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
+          RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
+        
 
           # Function to call the Firecloud API using the firecloud_api.py script
           firecloud_action() {
               python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
           }
-
-          # Create the submission_data.json file
-          SUBMISSION_DATA_FILE="submission_data.json"
+        
           # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$USE_CALL_CACHE" = "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
+        
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+        
           # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
             "methodConfigurationName": "$PIPELINE_NAME",
             "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": true,
+            "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,
             "memoryRetryMultiplier": 1.2,
             "workflowFailureMode": "NoNewCalls",
@@ -96,8 +120,30 @@ jobs:
             "ignoreEmptyOutputs": false
           }
           EOF
-
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
+        
+        
+          # Loop through each file in the appropriate test inputs directory
+          INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
+          for input_file in $INPUTS_DIR/*; do
+              # Update input_file as needed for test
+              test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
+                                                                                  --results_path $RESULTS_PATH \
+                                                                                  --inputs_json $input_file \
+                                                                                  --update_truth UPDATE_TRUTH)
+              # Upload the input file to the workspace
+              firecloud_action upload_file --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
+              echo "Submitting job for input file: $input_file"
+              # Submit job for the input file
+              # 1. Create a new submission
+              # 2. Poll submission status and get workflow IDs and statuses
+              # 3. Iterate over the Workflow IDs to get outputs
+          done
+
+        ##################################### LEFT OFF HERE #######################################
+        # TODO: move the submission and monitoring into the for loop to occur for each input file #
+        ###########################################################################################
+
           
           # 1. Submit a new workflow using the generated submission_data.json
           SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 576f0fb370..6d0dffbf26 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -1,14 +1,15 @@
 import argparse
 import json
 import os
+import datetime
 
 
-def update_test_inputs(inputs_json, truth_path, test_path, update_truth):
+def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
     # Update the test inputs JSON to work with the test wrapper WDL
     # The test wrapper WDL runs the pipeline WDL and verifies the results
     # The test wrapper WDL requires the following inputs:
     # - truth_path: The path to the truth data
-    # - test_path: The path to the test data
+    # - results_path: The path to the test data
     # - update_truth: Boolean indicating whether truth should be updated, default is False
     # The test inputs JSON will be updated to include these inputs
 
@@ -29,9 +30,10 @@ def update_test_inputs(inputs_json, truth_path, test_path, update_truth):
         new_key = key.replace(pipeline_name, test_name)
         test_inputs[new_key] = test_inputs.pop(key)
 
-    # Add the truth_path and test_path to the test inputs JSON
+    # Add the truth_path and results_path to the test inputs JSON
+    current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
+    test_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"
     test_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
-    test_inputs[f"{test_name}.test_path"] = f"{test_path}/{sample_name}/"
     test_inputs[f"{test_name}.update_truth"] = update_truth
 
     # Save the updated test inputs JSON
@@ -39,7 +41,8 @@ def update_test_inputs(inputs_json, truth_path, test_path, update_truth):
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
-    print("Test inputs JSON updated with truth_path and test_path")
+    print("Test inputs JSON updated with truth_path and results_path and saved as:", output_name)
+    return output_name
 
 
 def main():
@@ -56,8 +59,8 @@ def main():
     )
 
     parser.add_argument(
-        "--test_path",
-        dest="test_path",
+        "--results_path",
+        dest="results_path",
         required=True,
         help="The base path where the test data will be stored",
     )
@@ -73,16 +76,18 @@ def main():
     parser.add_argument(
         "--update_truth",
         dest="update_truth",
-        default=False,
+        default="false",
         required=False,
-        choices=[True, False],
-        help="Boolean flag to update the truth data. If True, the truth data will be updated with the test data. ",
+        choices=["true", "false"],
+        help="Boolean flag to update the truth data. If true, the truth data will be updated with the test data. ",
     )
 
     args = parser.parse_args()
+    # convert the update_truth flag to a boolean
+    update_truth_bool = args.update_truth.lower() == "true"
 
     # Update the test inputs to work with the test wrapper WDL
-    update_test_inputs(args.inputs_json, args.truth_path, args.test_path, args.update_truth)
+    update_test_inputs(args.inputs_json, args.truth_path, args.results_path, update_truth_bool)
 
 
 if __name__ == "__main__":
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 95d5e42b29..ace878c854 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -129,6 +129,33 @@ def poll_submission_status(self, submission_id):
 
         return workflow_status_map
 
+    def upload_test_inputs(self, pipeline_name, test_inputs):
+        """
+        Uploads test inputs to the workspace via Firecloud API.
+
+        :param test_inputs: JSON data containing test inputs
+        :return: True if successful, False otherwise
+        """
+        # Construct the API endpoint URL for the method configuration
+        url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/method_configs/{self.namespace}/{pipeline_name}"
+        # get the current method configuration
+        response = requests.get(url, headers=self.headers)
+        config = response.json()
+        # update the config with the new inputs
+        with open(test_inputs, 'r') as file:
+            inputs_json = json.load(file)
+            config["inputs"] = inputs_json
+        # post the updated method config to the workspace
+        response = requests.post(url, headers=self.headers, json=config)
+
+        # Check if the test inputs were uploaded successfully
+        if response.status_code == 200:
+            print("Test inputs uploaded successfully.")
+            return True
+        else:
+            print(f"Failed to upload test inputs. Status code: {response.status_code}")
+            return False
+
 
 # Bash Script Interaction
 if __name__ == "__main__":
@@ -144,8 +171,9 @@ def poll_submission_status(self, submission_id):
     parser.add_argument('--workflow_id', help='Workflow ID (required for get_outputs)')
     parser.add_argument('--pipeline_name', help='Pipeline name (required for get_outputs)')
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
+    parser.add_argument('--test_input_file', help='Path to test inputs JSON file (required for upload_test_inputs)')
 
-    args = parser.parse_args()
+args = parser.parse_args()
 
     # Initialize the FirecloudAPI instance with provided arguments
     firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace)
@@ -183,4 +211,9 @@ def poll_submission_status(self, submission_id):
           if workflow_status_map:
               print(json.dumps(workflow_status_map))  # Output the dictionary as a JSON string for bash parsing
           else:
-              print("No workflows found or an error occurred.", file=sys.stderr)
\ No newline at end of file
+              print("No workflows found or an error occurred.", file=sys.stderr)
+
+    elif args.action == 'upload_test_inputs':
+        if not all([args.pipeline_name, args.test_input_file]):
+            print("For 'upload_test_inputs', --pipeline_name and --test_input_file are required.", file=sys.stderr)
+        else:

From cc0f636c89d2b4b579bea157d6be3d1e4889be44 Mon Sep 17 00:00:00 2001
From: jessicaway <jway@broadinstitute.org>
Date: Thu, 14 Nov 2024 15:14:30 -0700
Subject: [PATCH 005/165] More updates - still untested

---
 scripts/firecloud_api/UpdateTestInputs.py | 2 --
 scripts/firecloud_api/firecloud_api.py    | 4 +++-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 6d0dffbf26..040906ae26 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -1,7 +1,6 @@
 import argparse
 import json
 import os
-import datetime
 
 
 def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
@@ -31,7 +30,6 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
         test_inputs[new_key] = test_inputs.pop(key)
 
     # Add the truth_path and results_path to the test inputs JSON
-    current_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
     test_inputs[f"{test_name}.results_path"] = f"{results_path}/{sample_name}/"
     test_inputs[f"{test_name}.truth_path"] = f"{truth_path}/{sample_name}/"
     test_inputs[f"{test_name}.update_truth"] = update_truth
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index ace878c854..322f32d64e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -173,7 +173,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
     parser.add_argument('--test_input_file', help='Path to test inputs JSON file (required for upload_test_inputs)')
 
-args = parser.parse_args()
+    args = parser.parse_args()
 
     # Initialize the FirecloudAPI instance with provided arguments
     firecloud_api = FirecloudAPI(args.token, args.namespace, args.workspace)
@@ -217,3 +217,5 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         if not all([args.pipeline_name, args.test_input_file]):
             print("For 'upload_test_inputs', --pipeline_name and --test_input_file are required.", file=sys.stderr)
         else:
+            success = firecloud_api.upload_test_inputs(args.pipeline_name, args.test_input_file)
+            print(success)
\ No newline at end of file

From 32a0fc614d34b4f1fa18974c532768f4f45424d4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:10:03 -0500
Subject: [PATCH 006/165] testing

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index cbe056f6db..9a0ecac538 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -143,7 +143,6 @@ jobs:
         ##################################### LEFT OFF HERE #######################################
         # TODO: move the submission and monitoring into the for loop to occur for each input file #
         ###########################################################################################
-
           
           # 1. Submit a new workflow using the generated submission_data.json
           SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")

From 0c82ed00b11ee3e9802e7a8fd755cf6c6732b8ff Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:25:45 -0500
Subject: [PATCH 007/165] convert UPDATE_TRUTH to bool friendly format

---
 .github/workflows/test_illumina_genotyping_array.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9a0ecac538..141e0ba644 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -103,6 +103,13 @@ jobs:
               USE_CALL_CACHE_BOOL=false
           fi
         
+          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH=true
+          else
+              UPDATE_TRUTH=false
+          fi
+        
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
         

From ce9be073e3d116360877d1ab8317777885ca557f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:27:07 -0500
Subject: [PATCH 008/165] hard code update truth to false for now

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 141e0ba644..11244d6e66 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -137,7 +137,7 @@ jobs:
               test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
                                                                                   --results_path $RESULTS_PATH \
                                                                                   --inputs_json $input_file \
-                                                                                  --update_truth UPDATE_TRUTH)
+                                                                                  --update_truth false)
               # Upload the input file to the workspace
               firecloud_action upload_file --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
               echo "Submitting job for input file: $input_file"

From 5f3a4650a1ff992bb828e79865e23cbbd6af8948 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:33:50 -0500
Subject: [PATCH 009/165] hard code update truth to false for now

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 11244d6e66..a77dce108e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -10,16 +10,11 @@ on:
   pull_request:
     branches: [ "develop", "staging", "master" ]
     # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml
-    ####################################
-    # SET PIPELINE SPECIFIC PATHS HERE #
-    ####################################
     paths:
       - 'pipelines/broad/genotyping/illumina/**'
       - 'tasks/**'
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
-
-
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:

From f34e163a2884eb785c6da188f5347a56dd26cd73 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:38:28 -0500
Subject: [PATCH 010/165] spacing?

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a77dce108e..8e7bb195c1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -62,7 +62,7 @@ jobs:
         echo "Current directory:"
         pwd
         ls -lht
-  
+
     - name: Submit job, poll status, and get outputs
       id: pipeline_run
       run: |

From 05e2ccb3ce59f37340e70b82c8db97e7d51a1727 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 09:39:49 -0500
Subject: [PATCH 011/165] no more hard coding

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8e7bb195c1..b074a7cff6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -132,7 +132,7 @@ jobs:
               test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
                                                                                   --results_path $RESULTS_PATH \
                                                                                   --inputs_json $input_file \
-                                                                                  --update_truth false)
+                                                                                  --update_truth $UPDATE_TRUTH)
               # Upload the input file to the workspace
               firecloud_action upload_file --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
               echo "Submitting job for input file: $input_file"

From dff5fb41bf93906e59a014c2d7e932fc40ff3f23 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 10:01:31 -0500
Subject: [PATCH 012/165] no more hard coding

---
 .../test_illumina_genotyping_array.yml        | 259 ++++++------------
 1 file changed, 81 insertions(+), 178 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b074a7cff6..221defbf69 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,21 +1,15 @@
-
 name: Test Illumina Genotyping Array
 
-# Controls when the workflow will run
+# Workflow triggers
 on:
-  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
-  # push:
-  #   branches:
-  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
-    # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml
     paths:
       - 'pipelines/broad/genotyping/illumina/**'
       - 'tasks/**'
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
-  # Allows you to run this workflow manually from the Actions tab
+
   workflow_dispatch:
     inputs:
       useCallCache:
@@ -26,94 +20,64 @@ on:
         description: 'Update truth files (default: false)'
         required: false
         default: "false"
+
 env:
   PROJECT_NAME: WARP
-  # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
 
 jobs:
   run_pipeline:
     runs-on: ubuntu-latest
-    # Add "id-token" with the intended permissions.
     permissions:
       contents: 'read'
       id-token: 'write'
 
     steps:
-    # actions/checkout MUST come before auth
-    - uses: 'actions/checkout@v3'
-
-    - id: 'auth'
-      name: 'Authenticate to Google Cloud'
-      uses: 'google-github-actions/auth@v2'
-      with:
-        token_format: 'access_token'
-        # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-        # This is provided by the DevOps team - do not change!
-        workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-        # This is our tester service account
-        service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-        access_token_lifetime: '3600' #seconds, default is 3600
-        access_token_scopes: 'profile, email, openid'
-
-    # ... further steps are automatically authenticated
-    - name: Check working directory
-      run: |
-        echo "Current directory:"
-        pwd
-        ls -lht
-
-    - name: Submit job, poll status, and get outputs
-      id: pipeline_run
-      run: |
-          # Set common environment variables
+      - uses: 'actions/checkout@v3'
+
+      - name: Check working directory
+        run: |
+          echo "Current working directory:"
+          pwd
+          echo "Listing files in the working directory:"
+          ls -alh
+
+      - id: 'auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          token_format: 'access_token'
+          workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+          service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
+          access_token_lifetime: '3600'
+          access_token_scopes: 'profile, email, openid'
+
+      - name: Submit job, poll status, and get outputs
+        id: pipeline_run
+        run: |
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
-          TEST_TYPE="Plumbing"  # Placeholder for now
+          TEST_TYPE="Plumbing"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-          
-          ########################################
-          # SET PIPELINE SPECIFIC VARIABLES HERE #
-          ########################################
           PIPELINE_NAME="IlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-          # TODO: Need to set the truth and result paths appropriately
           TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
           RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
-        
-
-          # Function to call the Firecloud API using the firecloud_api.py script
-          firecloud_action() {
-              python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
-          }
-        
-          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$USE_CALL_CACHE" = "true" ]; then
-              USE_CALL_CACHE_BOOL=true
-          else
-              USE_CALL_CACHE_BOOL=false
-          fi
-        
-          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH=true
-          else
-              UPDATE_TRUTH=false
-          fi
-        
-          # Create the submission_data.json file which will be the same for all inputs
+          
+          # Create JSON for submission data
           SUBMISSION_DATA_FILE="submission_data.json"
-        
-          # Use a heredoc to generate the JSON file content dynamically
+          CALL_CACHE_VALUE="true"
+          if [ "$USE_CALL_CACHE" = "false" ]; then
+            CALL_CACHE_VALUE="false"
+          fi
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
             "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "useCallCache": $CALL_CACHE_VALUE,
             "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,
             "memoryRetryMultiplier": 1.2,
@@ -122,117 +86,56 @@ jobs:
             "ignoreEmptyOutputs": false
           }
           EOF
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-        
-        
-          # Loop through each file in the appropriate test inputs directory
+          
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           for input_file in $INPUTS_DIR/*; do
-              # Update input_file as needed for test
-              test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
-                                                                                  --results_path $RESULTS_PATH \
-                                                                                  --inputs_json $input_file \
-                                                                                  --update_truth $UPDATE_TRUTH)
-              # Upload the input file to the workspace
-              firecloud_action upload_file --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
-              echo "Submitting job for input file: $input_file"
-              # Submit job for the input file
-              # 1. Create a new submission
-              # 2. Poll submission status and get workflow IDs and statuses
-              # 3. Iterate over the Workflow IDs to get outputs
-          done
-
-        ##################################### LEFT OFF HERE #######################################
-        # TODO: move the submission and monitoring into the for loop to occur for each input file #
-        ###########################################################################################
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
+              --truth_path $TRUTH_PATH \
+              --results_path $RESULTS_PATH \
+              --inputs_json $input_file \
+              --update_truth $UPDATE_TRUTH)
           
-          # 1. Submit a new workflow using the generated submission_data.json
-          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-
-          # Check if submission was successful
-          if [ -z "$SUBMISSION_ID" ]; then
-              echo "Submission failed."  # Log failure to stdout
-              echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
-              exit 1
-          fi
-
-          echo "Submission ID: $SUBMISSION_ID"
-          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
-
-          # 2. Poll submission status and get workflow IDs and statuses
-          echo "Polling submission status..."
-          RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-
-          # Parse the JSON response to get the workflow ID and statuses
-          echo "Workflows and their statuses:"
-          echo "$RESPONSE" | jq
-
-          # Check if RESPONSE is empty
-          if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
-              exit 1
-          fi
-
-          # Extract workflows and their statuses
-          WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
-          echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
-
-          # Generate markdown summary table for workflows and statuses
-          WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
+            python3 scripts/firecloud_api/firecloud_api.py \
+              --token "$TOKEN" \
+              --namespace "$NAMESPACE" \
+              --workspace "$WORKSPACE" \
+              --action upload_file \
+              --pipeline_name "$PIPELINE_NAME" \
+              --test_input_file "$test_input_file"
           
-          # Print workflow table to stdout
-          echo "$WORKFLOW_TABLE"
-
-          # 3. Iterate over the Workflow IDs to get outputs
-          OUTPUTS=""
-          echo "Retrieving workflow outputs..."
-          for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-              OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+            SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py \
+              --token "$TOKEN" \
+              --namespace "$NAMESPACE" \
+              --workspace "$WORKSPACE" \
+              --action submit \
+              --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+            if [ -z "$SUBMISSION_ID" ]; then
+              echo "Submission failed for input: $input_file"
+              continue
+            fi
+          
+            echo "Submission ID for input $input_file: $SUBMISSION_ID"
+          
+            RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py \
+              --token "$TOKEN" \
+              --namespace "$NAMESPACE" \
+              --workspace "$WORKSPACE" \
+              --action poll_status \
+              --submission_id "$SUBMISSION_ID")
+          
+            echo "Workflows for input $input_file and their statuses:"
+            echo "$RESPONSE" | jq
+          
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py \
+                --token "$TOKEN" \
+                --namespace "$NAMESPACE" \
+                --workspace "$WORKSPACE" \
+                --action get_outputs \
+                --submission_id "$SUBMISSION_ID" \
+                --workflow_id "$WORKFLOW_ID" \
+                --pipeline_name "$PIPELINE_NAME")
+              echo "Output for workflow $WORKFLOW_ID: $OUTPUT"
+            done
           done
-          echo "Workflow outputs retrieved successfully."
-          echo "Raw output before jq:"
-          echo "$OUTPUTS"
-          echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
-
-          # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
-          OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
-          #print outputs table to stdout
-          echo "$OUTPUTS_TABLE"
-
-    - name: Print Summary on Success
-      if: success()
-      run: |
-          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
-          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo " :shipit: " >> $GITHUB_STEP_SUMMARY
-
-    - name: Print Summary on Failure
-      if: failure()
-      run: |
-          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
-          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
\ No newline at end of file

From 91270db9579f80548b77ae6ebfdf623ff2586af6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 10:05:52 -0500
Subject: [PATCH 013/165] handle UPDATE_TRUTH

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 221defbf69..bd699423d9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -73,6 +73,11 @@ jobs:
           if [ "$USE_CALL_CACHE" = "false" ]; then
             CALL_CACHE_VALUE="false"
           fi
+          if [ "UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH=true
+          else
+              UPDATE_TRUTH=false
+          fi
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",

From 36f41cb474c05b3b8d3285ba7c8d70f519d986e0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 10:10:43 -0500
Subject: [PATCH 014/165] handle UPDATE_TRUTH

---
 .../test_illumina_genotyping_array.yml        | 36 ++++++++++---------
 1 file changed, 19 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index bd699423d9..daf836a0b0 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -66,14 +66,14 @@ jobs:
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
           RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
-          
+
           # Create JSON for submission data
           SUBMISSION_DATA_FILE="submission_data.json"
           CALL_CACHE_VALUE="true"
           if [ "$USE_CALL_CACHE" = "false" ]; then
             CALL_CACHE_VALUE="false"
           fi
-          if [ "UPDATE_TRUTH" = "true" ]; then
+          if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH=true
           else
               UPDATE_TRUTH=false
@@ -91,7 +91,7 @@ jobs:
             "ignoreEmptyOutputs": false
           }
           EOF
-          
+
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           for input_file in $INPUTS_DIR/*; do
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
@@ -99,39 +99,40 @@ jobs:
               --results_path $RESULTS_PATH \
               --inputs_json $input_file \
               --update_truth $UPDATE_TRUTH)
-          
-            python3 scripts/firecloud_api/firecloud_api.py \
-              --token "$TOKEN" \
-              --namespace "$NAMESPACE" \
-              --workspace "$WORKSPACE" \
-              --action upload_file \
-              --pipeline_name "$PIPELINE_NAME" \
-              --test_input_file "$test_input_file"
-          
+
+            # Removed the following lines:
+            # python3 scripts/firecloud_api/firecloud_api.py \
+            #   --token "$TOKEN" \
+            #   --namespace "$NAMESPACE" \
+            #   --workspace "$WORKSPACE" \
+            #   --action upload_file \
+            #   --pipeline_name "$PIPELINE_NAME" \
+            #   --test_input_file "$test_input_file"
+
             SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py \
               --token "$TOKEN" \
               --namespace "$NAMESPACE" \
               --workspace "$WORKSPACE" \
               --action submit \
               --submission_data_file "$SUBMISSION_DATA_FILE")
-          
+
             if [ -z "$SUBMISSION_ID" ]; then
               echo "Submission failed for input: $input_file"
               continue
             fi
-          
+
             echo "Submission ID for input $input_file: $SUBMISSION_ID"
-          
+
             RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py \
               --token "$TOKEN" \
               --namespace "$NAMESPACE" \
               --workspace "$WORKSPACE" \
               --action poll_status \
               --submission_id "$SUBMISSION_ID")
-          
+
             echo "Workflows for input $input_file and their statuses:"
             echo "$RESPONSE" | jq
-          
+
             for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
               OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py \
                 --token "$TOKEN" \
@@ -144,3 +145,4 @@ jobs:
               echo "Output for workflow $WORKFLOW_ID: $OUTPUT"
             done
           done
+      

From 817704ced16ae8ce6a4f5d01df9634a28c3c91b5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 10:41:09 -0500
Subject: [PATCH 015/165] handle UPDATE_TRUTH

---
 .../test_illumina_genotyping_array.yml        | 63 ++++++++++++++-----
 1 file changed, 47 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index daf836a0b0..c9740a5931 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,15 +1,25 @@
 name: Test Illumina Genotyping Array
 
-# Workflow triggers
+# Controls when the workflow will run
 on:
+  #run on push to feature branch "kp_GHA_Terra_auth_PD-2682" - REMOVE WHEN DONE TESTING
+  # push:
+  #   branches:
+  #     - kp_GHA_Terra_auth_PD-2682
   pull_request:
     branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed: pipelines/broad/genotyping/illumina, tasks, verification, .github/workflows/test_illumina_genotyping_array.yml
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
     paths:
       - 'pipelines/broad/genotyping/illumina/**'
       - 'tasks/**'
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
 
+
+  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
       useCallCache:
@@ -20,64 +30,81 @@ on:
         description: 'Update truth files (default: false)'
         required: false
         default: "false"
-
 env:
   PROJECT_NAME: WARP
+  # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
 
 jobs:
   run_pipeline:
     runs-on: ubuntu-latest
+    # Add "id-token" with the intended permissions.
     permissions:
       contents: 'read'
       id-token: 'write'
 
     steps:
+      # actions/checkout MUST come before auth
       - uses: 'actions/checkout@v3'
 
-      - name: Check working directory
-        run: |
-          echo "Current working directory:"
-          pwd
-          echo "Listing files in the working directory:"
-          ls -alh
-
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
         uses: 'google-github-actions/auth@v2'
         with:
           token_format: 'access_token'
+          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
+          # This is provided by the DevOps team - do not change!
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+          # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600'
+          access_token_lifetime: '3600' #seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
+    # ... further steps are automatically authenticated
+      - name: Check working directory
+        run: |
+          echo "Current directory:"
+          pwd
+          ls -lht
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
+          # Set common environment variables
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          TEST_TYPE="Plumbing"
+          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch
+          TEST_TYPE="Plumbing"  # Placeholder for now
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          
+          ########################################
+          # SET PIPELINE SPECIFIC VARIABLES HERE #
+          ########################################
           PIPELINE_NAME="IlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          # TODO: Need to set the truth and result paths appropriately
           TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
           RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
+        
 
-          # Create JSON for submission data
+          # Create JSON for submission data# Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
           CALL_CACHE_VALUE="true"
-          if [ "$USE_CALL_CACHE" = "false" ]; then
-            CALL_CACHE_VALUE="false"
+          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$USE_CALL_CACHE" = "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
           fi
           if [ "$UPDATE_TRUTH" = "true" ]; then
               UPDATE_TRUTH=true
           else
               UPDATE_TRUTH=false
           fi
+          # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
@@ -91,10 +118,14 @@ jobs:
             "ignoreEmptyOutputs": false
           }
           EOF
-
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+        
+        
+          # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           for input_file in $INPUTS_DIR/*; do
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
+            # Update input_file as needed for test
+          test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
               --truth_path $TRUTH_PATH \
               --results_path $RESULTS_PATH \
               --inputs_json $input_file \

From b27c130be496f86bdea87e139520418251660e48 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 10:42:55 -0500
Subject: [PATCH 016/165] debugging

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c9740a5931..add0be23a9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -119,6 +119,7 @@ jobs:
           }
           EOF
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          cat "$SUBMISSION_DATA_FILE"
         
         
           # Loop through each file in the appropriate test inputs directory

From e107dc77f92abb694ed3488c763ed0629501e830 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:17:39 -0500
Subject: [PATCH 017/165] debugging

---
 .../test_illumina_genotyping_array.yml        | 238 +++++++++++-------
 scripts/firecloud_api/firecloud_api.py        |   2 +-
 2 files changed, 152 insertions(+), 88 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index add0be23a9..d55a2df6ae 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,3 +1,4 @@
+
 name: Test Illumina Genotyping Array
 
 # Controls when the workflow will run
@@ -44,39 +45,39 @@ jobs:
       id-token: 'write'
 
     steps:
-      # actions/checkout MUST come before auth
-      - uses: 'actions/checkout@v3'
-
-      - id: 'auth'
-        name: 'Authenticate to Google Cloud'
-        uses: 'google-github-actions/auth@v2'
-        with:
-          token_format: 'access_token'
-          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-          # This is provided by the DevOps team - do not change!
-          workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-          # This is our tester service account
-          service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' #seconds, default is 3600
-          access_token_scopes: 'profile, email, openid'
+    # actions/checkout MUST come before auth
+    - uses: 'actions/checkout@v3'
+
+    - id: 'auth'
+      name: 'Authenticate to Google Cloud'
+      uses: 'google-github-actions/auth@v2'
+      with:
+        token_format: 'access_token'
+        # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
+        # This is provided by the DevOps team - do not change!
+        workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+        # This is our tester service account
+        service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
+        access_token_lifetime: '3600' #seconds, default is 3600
+        access_token_scopes: 'profile, email, openid'
 
     # ... further steps are automatically authenticated
-      - name: Check working directory
-        run: |
-          echo "Current directory:"
-          pwd
-          ls -lht
-
-      - name: Submit job, poll status, and get outputs
-        id: pipeline_run
-        run: |
+    - name: Check working directory
+      run: |
+        echo "Current directory:"
+        pwd
+        ls -lht
+
+    - name: Submit job, poll status, and get outputs
+      id: pipeline_run
+      run: |
           # Set common environment variables
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch
+          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
           TEST_TYPE="Plumbing"  # Placeholder for now
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
@@ -90,26 +91,34 @@ jobs:
           RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
         
 
-          # Create JSON for submission data# Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-          CALL_CACHE_VALUE="true"
+          # Function to call the Firecloud API using the firecloud_api.py script
+          firecloud_action() {
+              python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+          }
+        
           # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$USE_CALL_CACHE" = "true" ]; then
               USE_CALL_CACHE_BOOL=true
           else
               USE_CALL_CACHE_BOOL=false
           fi
-          if [ "$UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH=true
-          else
-              UPDATE_TRUTH=false
-          fi
+        
+        # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+         if [ "UPDATE_TRUTH" = "true" ]; then
+             UPDATE_TRUTH=true
+         else
+             UPDATE_TRUTH=false
+         fi
+        
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+        
           # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
             "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $CALL_CACHE_VALUE,
+            "useCallCache": $USE_CALL_CACHE_BOOL,
             "deleteIntermediateOutputFiles": false,
             "useReferenceDisks": true,
             "memoryRetryMultiplier": 1.2,
@@ -119,62 +128,117 @@ jobs:
           }
           EOF
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
-          cat "$SUBMISSION_DATA_FILE"
         
         
           # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           for input_file in $INPUTS_DIR/*; do
-            # Update input_file as needed for test
-          test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py \
-              --truth_path $TRUTH_PATH \
-              --results_path $RESULTS_PATH \
-              --inputs_json $input_file \
-              --update_truth $UPDATE_TRUTH)
-
-            # Removed the following lines:
-            # python3 scripts/firecloud_api/firecloud_api.py \
-            #   --token "$TOKEN" \
-            #   --namespace "$NAMESPACE" \
-            #   --workspace "$WORKSPACE" \
-            #   --action upload_file \
-            #   --pipeline_name "$PIPELINE_NAME" \
-            #   --test_input_file "$test_input_file"
-
-            SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py \
-              --token "$TOKEN" \
-              --namespace "$NAMESPACE" \
-              --workspace "$WORKSPACE" \
-              --action submit \
-              --submission_data_file "$SUBMISSION_DATA_FILE")
-
-            if [ -z "$SUBMISSION_ID" ]; then
-              echo "Submission failed for input: $input_file"
-              continue
-            fi
-
-            echo "Submission ID for input $input_file: $SUBMISSION_ID"
-
-            RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py \
-              --token "$TOKEN" \
-              --namespace "$NAMESPACE" \
-              --workspace "$WORKSPACE" \
-              --action poll_status \
-              --submission_id "$SUBMISSION_ID")
-
-            echo "Workflows for input $input_file and their statuses:"
-            echo "$RESPONSE" | jq
-
-            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py \
-                --token "$TOKEN" \
-                --namespace "$NAMESPACE" \
-                --workspace "$WORKSPACE" \
-                --action get_outputs \
-                --submission_id "$SUBMISSION_ID" \
-                --workflow_id "$WORKFLOW_ID" \
-                --pipeline_name "$PIPELINE_NAME")
-              echo "Output for workflow $WORKFLOW_ID: $OUTPUT"
-            done
+              # Update input_file as needed for test
+              test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
+                                                                                  --results_path $RESULTS_PATH \
+                                                                                  --inputs_json $input_file \
+                                                                                  --update_truth $UPDATE_TRUTH)
+              # Upload the input file to the workspace
+              firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
+              echo "Submitting job for input file: $input_file"
+              # Submit job for the input file
+              # 1. Create a new submission
+              # 2. Poll submission status and get workflow IDs and statuses
+              # 3. Iterate over the Workflow IDs to get outputs
+          done
+
+        ##################################### LEFT OFF HERE #######################################
+        # TODO: move the submission and monitoring into the for loop to occur for each input file #
+        ###########################################################################################
+
+          
+          # 1. Submit a new workflow using the generated submission_data.json
+          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+
+          # Check if submission was successful
+          if [ -z "$SUBMISSION_ID" ]; then
+              echo "Submission failed."  # Log failure to stdout
+              echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
+              exit 1
+          fi
+
+          echo "Submission ID: $SUBMISSION_ID"
+          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
+
+          # 2. Poll submission status and get workflow IDs and statuses
+          echo "Polling submission status..."
+          RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+
+          # Parse the JSON response to get the workflow ID and statuses
+          echo "Workflows and their statuses:"
+          echo "$RESPONSE" | jq
+
+          # Check if RESPONSE is empty
+          if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
+              exit 1
+          fi
+
+          # Extract workflows and their statuses
+          WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
+          echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
+
+          # Generate markdown summary table for workflows and statuses
+          WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
+          
+          # Print workflow table to stdout
+          echo "$WORKFLOW_TABLE"
+
+          # 3. Iterate over the Workflow IDs to get outputs
+          OUTPUTS=""
+          echo "Retrieving workflow outputs..."
+          for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+              OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
           done
-      
+          echo "Workflow outputs retrieved successfully."
+          echo "Raw output before jq:"
+          echo "$OUTPUTS"
+          echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
+
+          # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
+          OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
+          #print outputs table to stdout
+          echo "$OUTPUTS_TABLE"
+
+    - name: Print Summary on Success
+      if: success()
+      run: |
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
+          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo " :shipit: " >> $GITHUB_STEP_SUMMARY
+
+    - name: Print Summary on Failure
+      if: failure()
+      run: |
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
+          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
+          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 322f32d64e..336ad957cb 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -166,7 +166,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
     parser.add_argument('--token', required=True, help='API access token')
     parser.add_argument('--namespace', required=True, help='Workspace namespace')
     parser.add_argument('--workspace', required=True, help='Workspace name')
-    parser.add_argument('--action', required=True, choices=['get_outputs', 'submit', 'poll_status'], help='Action to perform')
+    parser.add_argument('--action', required=True, choices=['get_outputs', 'submit', 'poll_status', 'upload_test_inputs'], help='Action to perform')
     parser.add_argument('--submission_id', help='Submission ID (required for get_outputs and poll_status)')
     parser.add_argument('--workflow_id', help='Workflow ID (required for get_outputs)')
     parser.add_argument('--pipeline_name', help='Pipeline name (required for get_outputs)')

From 5c5993a72f850060b4950d7a8f3e11810c212ac1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:21:57 -0500
Subject: [PATCH 018/165] spacing?

---
 .../test_illumina_genotyping_array.yml        | 120 +++++++++---------
 1 file changed, 60 insertions(+), 60 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d55a2df6ae..8efe9941e1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -45,32 +45,32 @@ jobs:
       id-token: 'write'
 
     steps:
-    # actions/checkout MUST come before auth
-    - uses: 'actions/checkout@v3'
-
-    - id: 'auth'
-      name: 'Authenticate to Google Cloud'
-      uses: 'google-github-actions/auth@v2'
-      with:
-        token_format: 'access_token'
-        # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-        # This is provided by the DevOps team - do not change!
-        workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-        # This is our tester service account
-        service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-        access_token_lifetime: '3600' #seconds, default is 3600
-        access_token_scopes: 'profile, email, openid'
+      # actions/checkout MUST come before auth
+      - uses: 'actions/checkout@v3'
+
+      - id: 'auth'
+        name: 'Authenticate to Google Cloud'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          token_format: 'access_token'
+          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
+          # This is provided by the DevOps team - do not change!
+          workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+          # This is our tester service account
+          service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
+          access_token_lifetime: '3600' #seconds, default is 3600
+          access_token_scopes: 'profile, email, openid'
 
     # ... further steps are automatically authenticated
-    - name: Check working directory
-      run: |
-        echo "Current directory:"
-        pwd
-        ls -lht
-
-    - name: Submit job, poll status, and get outputs
-      id: pipeline_run
-      run: |
+      - name: Check working directory
+        run: |
+          echo "Current directory:"
+          pwd
+          ls -lht
+
+      - name: Submit job, poll status, and get outputs
+        id: pipeline_run
+        run: |
           # Set common environment variables
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
@@ -206,39 +206,39 @@ jobs:
           #print outputs table to stdout
           echo "$OUTPUTS_TABLE"
 
-    - name: Print Summary on Success
-      if: success()
-      run: |
-          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
-          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo " :shipit: " >> $GITHUB_STEP_SUMMARY
-
-    - name: Print Summary on Failure
-      if: failure()
-      run: |
-          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
-          echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-          echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
+      - name: Print Summary on Success
+        if: success()
+        run: |
+            echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
+            echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
+            echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+  
+            echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+  
+            echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo " :shipit: " >> $GITHUB_STEP_SUMMARY
+
+      - name: Print Summary on Failure
+        if: failure()
+        run: |
+            echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
+            echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
+            echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+  
+            echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+  
+            echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+            echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
+            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
\ No newline at end of file

From 14994f8d1c9e3fc92bf62dbc13ee4359b3b54838 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:26:49 -0500
Subject: [PATCH 019/165] spacing?

---
 .github/workflows/test_illumina_genotyping_array.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8efe9941e1..c36d38cf8a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -104,11 +104,11 @@ jobs:
           fi
         
         # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-         if [ "UPDATE_TRUTH" = "true" ]; then
-             UPDATE_TRUTH=true
-         else
-             UPDATE_TRUTH=false
-         fi
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH=true
+          else
+              UPDATE_TRUTH=false
+          fi
         
           # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"

From bf3b6b17f25ad0307e3ca95088dab63ad9186565 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:42:10 -0500
Subject: [PATCH 020/165] movign into loop

---
 .../test_illumina_genotyping_array.yml        | 250 +++++++++---------
 1 file changed, 120 insertions(+), 130 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c36d38cf8a..f8973d45b4 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -71,141 +71,131 @@ jobs:
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
-          # Set common environment variables
-          TOKEN="${{ steps.auth.outputs.access_token }}"
-          NAMESPACE="warp-pipelines"
-          WORKSPACE="WARP Tests"
-          USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
-          UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
-          TEST_TYPE="Plumbing"  # Placeholder for now
-          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-          
-          ########################################
-          # SET PIPELINE SPECIFIC VARIABLES HERE #
-          ########################################
-          PIPELINE_NAME="IlluminaGenotypingArray"
-          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-          # TODO: Need to set the truth and result paths appropriately
-          TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
-          RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
-        
-
-          # Function to call the Firecloud API using the firecloud_api.py script
-          firecloud_action() {
-              python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
-          }
-        
-          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$USE_CALL_CACHE" = "true" ]; then
-              USE_CALL_CACHE_BOOL=true
-          else
-              USE_CALL_CACHE_BOOL=false
-          fi
-        
-        # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH=true
-          else
-              UPDATE_TRUTH=false
-          fi
-        
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-        
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
-        
-        
-          # Loop through each file in the appropriate test inputs directory
-          INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
-          for input_file in $INPUTS_DIR/*; do
-              # Update input_file as needed for test
-              test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
-                                                                                  --results_path $RESULTS_PATH \
-                                                                                  --inputs_json $input_file \
-                                                                                  --update_truth $UPDATE_TRUTH)
-              # Upload the input file to the workspace
-              firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
-              echo "Submitting job for input file: $input_file"
-              # Submit job for the input file
-              # 1. Create a new submission
-              # 2. Poll submission status and get workflow IDs and statuses
-              # 3. Iterate over the Workflow IDs to get outputs
-          done
+               # Set common environment variables
+               TOKEN="${{ steps.auth.outputs.access_token }}"
+               NAMESPACE="warp-pipelines"
+               WORKSPACE="WARP Tests"
+               USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
+               UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
+               TEST_TYPE="Plumbing"  # Placeholder for now
+               CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+               
+               ########################################
+               # SET PIPELINE SPECIFIC VARIABLES HERE #
+               ########################################
+               PIPELINE_NAME="IlluminaGenotypingArray"
+               PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+               TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
+               RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
+               
+               # Function to call the Firecloud API using the firecloud_api.py script
+               firecloud_action() {
+                python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+               }
+               
+               # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+               if [ "$USE_CALL_CACHE" = "true" ]; then
+                USE_CALL_CACHE_BOOL=true
+               else
+                USE_CALL_CACHE_BOOL=false
+               fi
+               
+               # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+               if [ "$UPDATE_TRUTH" = "true" ]; then
+                UPDATE_TRUTH=true
+               else
+                UPDATE_TRUTH=false
+               fi
+               
+               # Create the submission_data.json file which will be the same for all inputs
+               SUBMISSION_DATA_FILE="submission_data.json"
+               
+               # Use a heredoc to generate the JSON file content dynamically
+               cat <<EOF > "$SUBMISSION_DATA_FILE"
+               {
+               "methodConfigurationNamespace": "warp-pipelines",
+               "methodConfigurationName": "$PIPELINE_NAME",
+               "useCallCache": $USE_CALL_CACHE_BOOL,
+               "deleteIntermediateOutputFiles": false,
+               "useReferenceDisks": true,
+               "memoryRetryMultiplier": 1.2,
+               "workflowFailureMode": "NoNewCalls",
+               "userComment": "Automated submission",
+               "ignoreEmptyOutputs": false
+               }
+               EOF
+               echo "Created submission data file: $SUBMISSION_DATA_FILE"
+               
+               # Loop through each file in the appropriate test inputs directory
+               INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
+               for input_file in $INPUTS_DIR/*; do
+                # Update input_file as needed for test
+                test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
+                                                                                    --results_path $RESULTS_PATH \
+                                                                                    --inputs_json $input_file \
+                                                                                    --update_truth $UPDATE_TRUTH)
+                # Upload the input file to the workspace
+                firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
+                echo "Submitting job for input file: $input_file"
+               
+               # 1. Submit a new workflow using the generated submission_data.json
+                SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+               
+               #check if submission was successful
+                if [ -z "$SUBMISSION_ID" ]; then
+                 echo "Submission failed."  # Log failure to stdout
+                 echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
+                 exit 1
+                fi
+               
+                echo "Submission ID: $SUBMISSION_ID"
+                echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
+               
+               # 2. Poll submission status and get workflow IDs and statuses
+                echo "Polling submission status..."
+                RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+               
+                # Parse the JSON response to get the workflow ID and statuses
+                echo "Workflows and their statuses:"
+                echo "$RESPONSE" | jq
+               
+                # Check if RESPONSE is empty
+                if [ -z "$RESPONSE" ]; then
+                 echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
+                 exit 1
+                fi
+               
+                # Extract workflows and their statuses
+                WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
+                echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
+               
+                # Generate markdown summary table for workflows and statuses
+                WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
+               
+                # Print workflow table to stdout
+                echo "$WORKFLOW_TABLE"
+                
+                # 3. Iterate over the Workflow IDs to get outputs
+                OUTPUTS=""
+                echo "Retrieving workflow outputs..."
+                for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+                    WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+                    OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+                done
+                echo "Workflow outputs retrieved successfully."
+                echo "Raw output before jq:"
+                echo "$OUTPUTS"
+                echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
+                
+                # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
+                OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
+                #print outputs table to stdout
+                echo "$OUTPUTS_TABLE"
 
         ##################################### LEFT OFF HERE #######################################
         # TODO: move the submission and monitoring into the for loop to occur for each input file #
         ###########################################################################################
 
-          
-          # 1. Submit a new workflow using the generated submission_data.json
-          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-
-          # Check if submission was successful
-          if [ -z "$SUBMISSION_ID" ]; then
-              echo "Submission failed."  # Log failure to stdout
-              echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
-              exit 1
-          fi
-
-          echo "Submission ID: $SUBMISSION_ID"
-          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
-
-          # 2. Poll submission status and get workflow IDs and statuses
-          echo "Polling submission status..."
-          RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-
-          # Parse the JSON response to get the workflow ID and statuses
-          echo "Workflows and their statuses:"
-          echo "$RESPONSE" | jq
-
-          # Check if RESPONSE is empty
-          if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
-              exit 1
-          fi
-
-          # Extract workflows and their statuses
-          WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
-          echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
-
-          # Generate markdown summary table for workflows and statuses
-          WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
-          
-          # Print workflow table to stdout
-          echo "$WORKFLOW_TABLE"
-
-          # 3. Iterate over the Workflow IDs to get outputs
-          OUTPUTS=""
-          echo "Retrieving workflow outputs..."
-          for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-              OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-          done
-          echo "Workflow outputs retrieved successfully."
-          echo "Raw output before jq:"
-          echo "$OUTPUTS"
-          echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
-
-          # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
-          OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
-          #print outputs table to stdout
-          echo "$OUTPUTS_TABLE"
-
       - name: Print Summary on Success
         if: success()
         run: |

From e745d51b290bed69b1be38f8f9d183a8af424c9e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:43:48 -0500
Subject: [PATCH 021/165] debugging prints

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f8973d45b4..a2518aa879 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -128,8 +128,10 @@ jobs:
                
                # Loop through each file in the appropriate test inputs directory
                INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
+               echo "Test inputs directory: $INPUTS_DIR"
                for input_file in $INPUTS_DIR/*; do
                 # Update input_file as needed for test
+                echo "Processing input file: $input_file"
                 test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
                                                                                     --results_path $RESULTS_PATH \
                                                                                     --inputs_json $input_file \

From 52b8c45e022de2f41bab7761bb8b9f656152e514 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:51:34 -0500
Subject: [PATCH 022/165] space

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a2518aa879..750fe29aa2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -193,6 +193,7 @@ jobs:
                 OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
                 #print outputs table to stdout
                 echo "$OUTPUTS_TABLE"
+               done
 
         ##################################### LEFT OFF HERE #######################################
         # TODO: move the submission and monitoring into the for loop to occur for each input file #

From de2edb7e119518207e9bce2957ba411f90b1ea71 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 11:58:24 -0500
Subject: [PATCH 023/165] what is the test input file?????

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 750fe29aa2..abafb24672 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -137,6 +137,7 @@ jobs:
                                                                                     --inputs_json $input_file \
                                                                                     --update_truth $UPDATE_TRUTH)
                 # Upload the input file to the workspace
+                echo "Uploading test input file: $test_input_file"
                 firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
                 echo "Submitting job for input file: $input_file"
                

From 6340b02560d14d7ff15ff0b7aec6894bad9c09a1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:17:48 -0500
Subject: [PATCH 024/165] fix name of test input file

---
 scripts/firecloud_api/UpdateTestInputs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 040906ae26..d939e95212 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -39,7 +39,7 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
-    print("Test inputs JSON updated with truth_path and results_path and saved as:", output_name)
+    print(f"Test inputs JSON updated and saved as: {output_name}")
     return output_name
 
 

From 01b07a0d8cee278fd6f3749ddb9b316a350a7991 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:27:45 -0500
Subject: [PATCH 025/165] remove print bc we arent capturing the right thing

---
 scripts/firecloud_api/UpdateTestInputs.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index d939e95212..27b8305e68 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -39,7 +39,6 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
-    print(f"Test inputs JSON updated and saved as: {output_name}")
     return output_name
 
 

From a8801eb51cf9ca85165bf17936c9411763839abc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:33:11 -0500
Subject: [PATCH 026/165] remove print bc we arent capturing the right thing

---
 scripts/firecloud_api/UpdateTestInputs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 27b8305e68..467dc7c7e3 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -39,6 +39,7 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
     with open(output_name, 'w') as file:
         json.dump(test_inputs, file, indent=4)
 
+    print(f"{output_name}")
     return output_name
 
 

From 0ba99a387ab26827405763e3b7d781cc6d81fd86 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:39:13 -0500
Subject: [PATCH 027/165] check to see if we are even getting the config

---
 scripts/firecloud_api/firecloud_api.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 336ad957cb..2a47e2e741 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -141,6 +141,8 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
+        # print the config
+        print(config)
         # update the config with the new inputs
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)

From 528b88f4b9a503b42a0ac2127fd899abd5777670 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:42:18 -0500
Subject: [PATCH 028/165] check to see if we are even getting the config

---
 scripts/firecloud_api/firecloud_api.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 2a47e2e741..6fbc55cb75 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -141,7 +141,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
-        # print the config
+        # print the config, can delete this line later
         print(config)
         # update the config with the new inputs
         with open(test_inputs, 'r') as file:
@@ -149,6 +149,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             config["inputs"] = inputs_json
         # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
+        print(config)
 
         # Check if the test inputs were uploaded successfully
         if response.status_code == 200:

From 6b25baf4b7099ce0669ff3db34b751ec20659653 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:44:21 -0500
Subject: [PATCH 029/165] print response

---
 scripts/firecloud_api/firecloud_api.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 6fbc55cb75..5b9e9d0786 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -149,6 +149,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             config["inputs"] = inputs_json
         # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
+        # print the updated config, can delete this line later
         print(config)
 
         # Check if the test inputs were uploaded successfully
@@ -156,7 +157,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             print("Test inputs uploaded successfully.")
             return True
         else:
-            print(f"Failed to upload test inputs. Status code: {response.status_code}")
+            print(f"Failed to upload test inputs. Status code: {response}")
             return False
 
 

From 91c78ab5671b70c94d06a1f815d9eb5c71bb4bce Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 12:46:10 -0500
Subject: [PATCH 030/165] revert print response

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 5b9e9d0786..20d538d7ad 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -157,7 +157,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             print("Test inputs uploaded successfully.")
             return True
         else:
-            print(f"Failed to upload test inputs. Status code: {response}")
+            print(f"Failed to upload test inputs. Status code: {response.status_code}")
             return False
 
 

From 6abd1d294970e25cb15feb2831f331ceba586786 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 15:43:16 -0500
Subject: [PATCH 031/165] print the url

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 20d538d7ad..e5e0f3ab30 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -138,6 +138,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         """
         # Construct the API endpoint URL for the method configuration
         url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/method_configs/{self.namespace}/{pipeline_name}"
+        print(url)
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()

From cf24731bcdb51c642224df498388c90958e76562 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 15:47:44 -0500
Subject: [PATCH 032/165] print the url

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index e5e0f3ab30..db516cac4c 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -70,6 +70,7 @@ def create_submission(self, submission_data):
         """
         # Construct the API endpoint URL for creating a new submission
         url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
+        print(url)
         response = requests.post(url, headers=self.headers, json=submission_data)
 
         # Check if the submission was created successfully

From 9db9e21106a072b92027607520e9c072808e4cdf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 15:51:03 -0500
Subject: [PATCH 033/165] print the url

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index db516cac4c..8aa1a80577 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -70,7 +70,6 @@ def create_submission(self, submission_data):
         """
         # Construct the API endpoint URL for creating a new submission
         url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions"
-        print(url)
         response = requests.post(url, headers=self.headers, json=submission_data)
 
         # Check if the submission was created successfully
@@ -92,6 +91,7 @@ def poll_submission_status(self, submission_id):
         """
         # Construct the API endpoint URL for polling submission status
         status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
+        print(status_url)
         workflow_status_map = {}
 
         # Continuously poll the status of the submission until completion

From fd62912b96781a01b4f634156881316e7ee9e653 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 15:52:39 -0500
Subject: [PATCH 034/165] print the url

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 8aa1a80577..288e66bae9 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -91,7 +91,7 @@ def poll_submission_status(self, submission_id):
         """
         # Construct the API endpoint URL for polling submission status
         status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
-        print(status_url)
+        print(f"the status url is  {status_url}")
         workflow_status_map = {}
 
         # Continuously poll the status of the submission until completion

From 05975ca3e5bcbb56c32b5f51604b978c9e113c5c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:03:30 -0500
Subject: [PATCH 035/165] use quote lib

---
 scripts/firecloud_api/firecloud_api.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 288e66bae9..ea4ad6afe1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -18,6 +18,8 @@
 import time
 import json
 import sys
+from urllib.parse import quote
+
 
 class FirecloudAPI:
     def __init__(self, token, namespace, workspace_name):
@@ -91,7 +93,6 @@ def poll_submission_status(self, submission_id):
         """
         # Construct the API endpoint URL for polling submission status
         status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
-        print(f"the status url is  {status_url}")
         workflow_status_map = {}
 
         # Continuously poll the status of the submission until completion
@@ -138,7 +139,8 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         :return: True if successful, False otherwise
         """
         # Construct the API endpoint URL for the method configuration
-        url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/method_configs/{self.namespace}/{pipeline_name}"
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
+
         print(url)
         # get the current method configuration
         response = requests.get(url, headers=self.headers)

From 51fad23a4773daffca43b0a18285d1678c0d855a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:07:10 -0500
Subject: [PATCH 036/165] lots of debugging prints

---
 scripts/firecloud_api/firecloud_api.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index ea4ad6afe1..89e2fbaeb8 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -148,13 +148,23 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # print the config, can delete this line later
         print(config)
         # update the config with the new inputs
+        print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
+            print("Test inputs loaded successfully.")
+            print(f"Test inputs content: {json.dumps(inputs_json, indent=2)}")  # Pretty-print inputs
             inputs_json = json.load(file)
             config["inputs"] = inputs_json
-        # post the updated method config to the workspace
+
+        print(f"Constructed URL: {url}")
+        print(f"Headers: {self.headers}")
+        print(f"Config to be posted: {json.dumps(config, indent=2)}")  # Pretty-print config
+
+    # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
         # print the updated config, can delete this line later
         print(config)
+        print(f"Response status code: {response.status_code}")
+        print(f"Response text: {response.text}")
 
         # Check if the test inputs were uploaded successfully
         if response.status_code == 200:

From e7b5f0261fee34564383b60264ed563b7fe9e8a8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:08:41 -0500
Subject: [PATCH 037/165] lots of debugging prints

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 89e2fbaeb8..4228c42fc1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -150,9 +150,9 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
+            inputs_json = json.load(file)
             print("Test inputs loaded successfully.")
             print(f"Test inputs content: {json.dumps(inputs_json, indent=2)}")  # Pretty-print inputs
-            inputs_json = json.load(file)
             config["inputs"] = inputs_json
 
         print(f"Constructed URL: {url}")

From 7dbd1b3e046e7c558e1b4a8659f99faea39dfdc5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:38:11 -0500
Subject: [PATCH 038/165] add test to pipeline name

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 4228c42fc1..703d7d7ff1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -139,7 +139,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         :return: True if successful, False otherwise
         """
         # Construct the API endpoint URL for the method configuration
-        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/Test{pipeline_name}"
 
         print(url)
         # get the current method configuration

From 86893a0683606fe7591f6df64f8f70ba2e5405df Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:45:25 -0500
Subject: [PATCH 039/165] try to quote the strings

---
 scripts/firecloud_api/firecloud_api.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 703d7d7ff1..3060fc9609 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -131,6 +131,20 @@ def poll_submission_status(self, submission_id):
 
         return workflow_status_map
 
+    def quote_values(self, data):
+        """
+        Recursively quotes values in a dictionary or list to match Firecloud API format.
+        """
+        if isinstance(data, dict):
+            return {key: self.quote_values(value) for key, value in data.items()}
+        elif isinstance(data, list):
+            return [self.quote_values(item) for item in data]
+        elif isinstance(data, (str, int, float, bool)):
+            return f"\"{data}\""
+        else:
+            return data  # Return as-is if it's not a string, int, float, or bool
+
+
     def upload_test_inputs(self, pipeline_name, test_inputs):
         """
         Uploads test inputs to the workspace via Firecloud API.
@@ -152,8 +166,10 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)
             print("Test inputs loaded successfully.")
-            print(f"Test inputs content: {json.dumps(inputs_json, indent=2)}")  # Pretty-print inputs
+            print(f"Original Test inputs content: {json.dumps(inputs_json, indent=2)}")  # Pretty-print inputs
             config["inputs"] = inputs_json
+            inputs_json = self.quote_values(inputs_json)
+            print(f"Quoted Test inputs content: {json.dumps(inputs_json, indent=2)}")
 
         print(f"Constructed URL: {url}")
         print(f"Headers: {self.headers}")

From b8f3a4cf1731d843a0232a0e51768751efacca6c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:48:45 -0500
Subject: [PATCH 040/165] try to quote the strings

---
 scripts/firecloud_api/firecloud_api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 3060fc9609..cc77fdb085 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -166,9 +166,9 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)
             print("Test inputs loaded successfully.")
-            print(f"Original Test inputs content: {json.dumps(inputs_json, indent=2)}")  # Pretty-print inputs
-            config["inputs"] = inputs_json
+            print(f"Original Test inputs content: {json.dumps(inputs_json, indent=2)}")
             inputs_json = self.quote_values(inputs_json)
+            config["inputs"] = inputs_json
             print(f"Quoted Test inputs content: {json.dumps(inputs_json, indent=2)}")
 
         print(f"Constructed URL: {url}")

From 160c4cdfe586e5a3ddfb472c7e5466ae93550e5b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 16:59:55 -0500
Subject: [PATCH 041/165] remove test from url

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 scripts/firecloud_api/firecloud_api.py               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index abafb24672..accee05eb6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -83,7 +83,7 @@ jobs:
                ########################################
                # SET PIPELINE SPECIFIC VARIABLES HERE #
                ########################################
-               PIPELINE_NAME="IlluminaGenotypingArray"
+               PIPELINE_NAME="TestIlluminaGenotypingArray"
                PIPELINE_DIR="pipelines/broad/genotyping/illumina"
                TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
                RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index cc77fdb085..afb03ab208 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -153,7 +153,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         :return: True if successful, False otherwise
         """
         # Construct the API endpoint URL for the method configuration
-        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/Test{pipeline_name}"
+        url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
         print(url)
         # get the current method configuration

From 584b226f7c0ae8f8d8c73c047a0eb00b3ff14291 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 17:41:51 -0500
Subject: [PATCH 042/165] change truth path and results path

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index accee05eb6..c912c5fec8 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -85,8 +85,8 @@ jobs:
                ########################################
                PIPELINE_NAME="TestIlluminaGenotypingArray"
                PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-               TRUTH_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/truth"
-               RESULTS_PATH="gs://broad-warp-test-storage/illumina_genotyping_array/results/$CURRENT_TIME"
+               TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
+               RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
                
                # Function to call the Firecloud API using the firecloud_api.py script
                firecloud_action() {

From ba85f6d002065d765f5ca213ecab7ecc3c7aa857 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 17:53:52 -0500
Subject: [PATCH 043/165] call caching

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c912c5fec8..4a2dcea542 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -94,6 +94,7 @@ jobs:
                }
                
                # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+               CALL_CACHE_VALUE="true"
                if [ "$USE_CALL_CACHE" = "true" ]; then
                 USE_CALL_CACHE_BOOL=true
                else

From f6984ac152a7532802df78c3d91ea995a778ceeb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 17:59:15 -0500
Subject: [PATCH 044/165] cat submission data file

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4a2dcea542..1035a76d2d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -126,6 +126,7 @@ jobs:
                }
                EOF
                echo "Created submission data file: $SUBMISSION_DATA_FILE"
+               cat "$SUBMISSION_DATA_FILE"
                
                # Loop through each file in the appropriate test inputs directory
                INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE

From 4075f0ab9b2348dba9ffae2cf9aed4d7fbaf4b88 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 18:00:39 -0500
Subject: [PATCH 045/165] cat submission data file

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1035a76d2d..c54e5ea12d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -94,7 +94,7 @@ jobs:
                }
                
                # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-               CALL_CACHE_VALUE="true"
+               USE_CALL_CACHE_BOOL="true"
                if [ "$USE_CALL_CACHE" = "true" ]; then
                 USE_CALL_CACHE_BOOL=true
                else

From ed38434e96142f5ba4cb2b9ec2abc03033442aad Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Fri, 15 Nov 2024 18:06:03 -0500
Subject: [PATCH 046/165] fix call caching logic

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c54e5ea12d..9069379f85 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -94,7 +94,7 @@ jobs:
                }
                
                # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-               USE_CALL_CACHE_BOOL="true"
+                 USE_CALL_CACHE="true"
                if [ "$USE_CALL_CACHE" = "true" ]; then
                 USE_CALL_CACHE_BOOL=true
                else

From 180502c71dffe74a51dc2b23649fc320f259aa04 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Sun, 17 Nov 2024 19:22:32 -0500
Subject: [PATCH 047/165] retry permissions

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9069379f85..5e230f05a0 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -88,7 +88,7 @@ jobs:
                TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
                RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
                
-               # Function to call the Firecloud API using the firecloud_api.py script
+               # Function to call the Firecloud API using the firecloud_api.py script 
                firecloud_action() {
                 python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
                }

From 3635e0388e8eee6c551e711fa5c6d473b8f23ee7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 07:25:26 -0500
Subject: [PATCH 048/165] retry permissions

---
 scripts/firecloud_api/UpdateTestInputs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/UpdateTestInputs.py b/scripts/firecloud_api/UpdateTestInputs.py
index 467dc7c7e3..881fc7c6c5 100644
--- a/scripts/firecloud_api/UpdateTestInputs.py
+++ b/scripts/firecloud_api/UpdateTestInputs.py
@@ -44,7 +44,7 @@ def update_test_inputs(inputs_json, truth_path, results_path, update_truth):
 
 
 def main():
-    description = """This script updates the test inputs JSON to work with the test wrapper WDL, 
+    description = """This script updates the test inputs JSON to work with the test wrapper WDL,
     which runs the pipeline and verification"""
 
     parser = argparse.ArgumentParser(description=description)

From 32f6fe0e11b4118595fd50acfaed2ec684e9074a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 09:33:27 -0500
Subject: [PATCH 049/165] add a new plumbing json

---
 .../test_illumina_genotyping_array.yml        |  2 +-
 .../test_inputs/Plumbing/SimpleInputNew.json  | 31 +++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)
 create mode 100644 pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5e230f05a0..2059d3f559 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -170,7 +170,7 @@ jobs:
                  exit 1
                 fi
                
-                # Extract workflows and their statuses
+                # Extract workflows and their statuses 
                 WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
                 echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
                
diff --git a/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json b/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json
new file mode 100644
index 0000000000..10ce49903b
--- /dev/null
+++ b/pipelines/broad/genotyping/illumina/test_inputs/Plumbing/SimpleInputNew.json
@@ -0,0 +1,31 @@
+{
+  "IlluminaGenotypingArray.sample_alias": "NA12878_New",
+  "IlluminaGenotypingArray.analysis_version_number": 1,
+  "IlluminaGenotypingArray.call_rate_threshold": 0.98,
+  "IlluminaGenotypingArray.reported_gender": "Female",
+  "IlluminaGenotypingArray.chip_well_barcode": "7991775143_R01C01",
+
+  "IlluminaGenotypingArray.green_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanExome-12v1-1_A/idats/7991775143_R01C01/7991775143_R01C01_Grn.idat",
+  "IlluminaGenotypingArray.red_idat_cloud_path": "gs://broad-gotc-test-storage/arrays/HumanExome-12v1-1_A/idats/7991775143_R01C01/7991775143_R01C01_Red.idat",
+
+  "IlluminaGenotypingArray.bead_pool_manifest_file": "gs://gcp-public-data--broad-references/arrays/HumanExome-12v1-1_A/HumanExome-12v1-1_A.bpm",
+  "IlluminaGenotypingArray.extended_chip_manifest_file": "gs://gcp-public-data--broad-references/arrays/HumanExome-12v1-1_A/HumanExome-12v1-1_A.1.3.extended.csv",
+  "IlluminaGenotypingArray.cluster_file": "gs://gcp-public-data--broad-references/arrays/HumanExome-12v1-1_A/HumanExomev1_1_CEPH_A.egt",
+  "IlluminaGenotypingArray.gender_cluster_file": "gs://broad-gotc-test-storage/arrays/metadata/HumanExome-12v1-1_A/HumanExomev1_1_gender.egt",
+  "IlluminaGenotypingArray.zcall_thresholds_file": "gs://broad-gotc-test-storage/arrays/metadata/HumanExome-12v1-1_A/IBDPRISM_EX.egt.thresholds.txt",
+
+  "IlluminaGenotypingArray.control_sample_vcf_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz",
+  "IlluminaGenotypingArray.control_sample_vcf_index_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.vcf.gz.tbi",
+  "IlluminaGenotypingArray.control_sample_intervals_file" : "gs://broad-gotc-test-storage/arrays/controldata/NA12878.interval_list",
+  "IlluminaGenotypingArray.control_sample_name" : "NA12878",
+  "IlluminaGenotypingArray.disk_size": 100,
+
+  "IlluminaGenotypingArray.ref_fasta": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta",
+  "IlluminaGenotypingArray.ref_fasta_index": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai",
+  "IlluminaGenotypingArray.ref_dict": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.dict",
+  "IlluminaGenotypingArray.dbSNP_vcf": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz",
+  "IlluminaGenotypingArray.dbSNP_vcf_index": "gs://gcp-public-data--broad-references/hg19/v0/dbsnp_138.b37.vcf.gz.tbi",
+  "IlluminaGenotypingArray.haplotype_database_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.txt",
+  "IlluminaGenotypingArray.variant_rsids_file": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.haplotype_database.snps.list",
+  "IlluminaGenotypingArray.preemptible_tries": 3
+}

From c3660ff53ae47945324976b90cbb78f6c90f9116 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 10:50:11 -0500
Subject: [PATCH 050/165] add a new plumbing json

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index afb03ab208..91134ae4fa 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -153,6 +153,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         :return: True if successful, False otherwise
         """
         # Construct the API endpoint URL for the method configuration
+        # properly encode the space in WARP Tests as %20 using from urllib.parse import quote
         url = f"{self.base_url}/workspaces/{self.namespace}/{quote(self.workspace_name)}/method_configs/{self.namespace}/{pipeline_name}"
 
         print(url)

From 2408b66c3046ea9cbb5e63b049f8e8178450dab8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 11:31:39 -0500
Subject: [PATCH 051/165] try not to overqrite

---
 .../test_illumina_genotyping_array.yml        | 275 +++++++-----------
 1 file changed, 110 insertions(+), 165 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2059d3f559..050c3ce339 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -19,7 +19,6 @@ on:
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
 
-
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
@@ -31,6 +30,7 @@ on:
         description: 'Update truth files (default: false)'
         required: false
         default: "false"
+
 env:
   PROJECT_NAME: WARP
   # Github repo name
@@ -53,15 +53,12 @@ jobs:
         uses: 'google-github-actions/auth@v2'
         with:
           token_format: 'access_token'
-          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
-          # This is provided by the DevOps team - do not change!
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
-          # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' #seconds, default is 3600
+          access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-    # ... further steps are automatically authenticated
+      # ... further steps are automatically authenticated
       - name: Check working directory
         run: |
           echo "Current directory:"
@@ -71,170 +68,118 @@ jobs:
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
-               # Set common environment variables
-               TOKEN="${{ steps.auth.outputs.access_token }}"
-               NAMESPACE="warp-pipelines"
-               WORKSPACE="WARP Tests"
-               USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
-               UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-               TEST_TYPE="Plumbing"  # Placeholder for now
-               CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-               
-               ########################################
-               # SET PIPELINE SPECIFIC VARIABLES HERE #
-               ########################################
-               PIPELINE_NAME="TestIlluminaGenotypingArray"
-               PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-               TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
-               RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
-               
-               # Function to call the Firecloud API using the firecloud_api.py script 
-               firecloud_action() {
-                python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
-               }
-               
-               # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-                 USE_CALL_CACHE="true"
-               if [ "$USE_CALL_CACHE" = "true" ]; then
-                USE_CALL_CACHE_BOOL=true
-               else
-                USE_CALL_CACHE_BOOL=false
-               fi
-               
-               # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-               if [ "$UPDATE_TRUTH" = "true" ]; then
-                UPDATE_TRUTH=true
-               else
-                UPDATE_TRUTH=false
-               fi
-               
-               # Create the submission_data.json file which will be the same for all inputs
-               SUBMISSION_DATA_FILE="submission_data.json"
-               
-               # Use a heredoc to generate the JSON file content dynamically
-               cat <<EOF > "$SUBMISSION_DATA_FILE"
-               {
-               "methodConfigurationNamespace": "warp-pipelines",
-               "methodConfigurationName": "$PIPELINE_NAME",
-               "useCallCache": $USE_CALL_CACHE_BOOL,
-               "deleteIntermediateOutputFiles": false,
-               "useReferenceDisks": true,
-               "memoryRetryMultiplier": 1.2,
-               "workflowFailureMode": "NoNewCalls",
-               "userComment": "Automated submission",
-               "ignoreEmptyOutputs": false
-               }
-               EOF
-               echo "Created submission data file: $SUBMISSION_DATA_FILE"
-               cat "$SUBMISSION_DATA_FILE"
-               
-               # Loop through each file in the appropriate test inputs directory
-               INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
-               echo "Test inputs directory: $INPUTS_DIR"
-               for input_file in $INPUTS_DIR/*; do
-                # Update input_file as needed for test
-                echo "Processing input file: $input_file"
-                test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
-                                                                                    --results_path $RESULTS_PATH \
-                                                                                    --inputs_json $input_file \
-                                                                                    --update_truth $UPDATE_TRUTH)
-                # Upload the input file to the workspace
-                echo "Uploading test input file: $test_input_file"
-                firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" 
-                echo "Submitting job for input file: $input_file"
-               
-               # 1. Submit a new workflow using the generated submission_data.json
-                SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-               
-               #check if submission was successful
-                if [ -z "$SUBMISSION_ID" ]; then
-                 echo "Submission failed."  # Log failure to stdout
-                 echo "submission_id=" >> $GITHUB_OUTPUT  # Set empty submission id
-                 exit 1
-                fi
-               
-                echo "Submission ID: $SUBMISSION_ID"
-                echo "submission_id=$SUBMISSION_ID" >> $GITHUB_OUTPUT  # Write the submission ID to GITHUB_OUTPUT
-               
-               # 2. Poll submission status and get workflow IDs and statuses
-                echo "Polling submission status..."
-                RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-               
-                # Parse the JSON response to get the workflow ID and statuses
-                echo "Workflows and their statuses:"
-                echo "$RESPONSE" | jq
-               
-                # Check if RESPONSE is empty
-                if [ -z "$RESPONSE" ]; then
-                 echo "Failed to retrieve Workflow IDs."  # Log failure to stdout
-                 exit 1
-                fi
-               
-                # Extract workflows and their statuses 
-                WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
-                echo "workflow_statuses=$WORKFLOW_STATUSES" >> $GITHUB_OUTPUT  # Write workflow statuses to GITHUB_OUTPUT
-               
-                # Generate markdown summary table for workflows and statuses
-                WORKFLOW_TABLE=$(echo "$RESPONSE" | jq -r 'to_entries | ["Workflow ID | Status", "--- | ---"] + map(.key + " | " + .value) | .[]')
-               
-                # Print workflow table to stdout
-                echo "$WORKFLOW_TABLE"
-                
-                # 3. Iterate over the Workflow IDs to get outputs
-                OUTPUTS=""
-                echo "Retrieving workflow outputs..."
-                for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-                    WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-                    OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-                done
-                echo "Workflow outputs retrieved successfully."
-                echo "Raw output before jq:"
-                echo "$OUTPUTS"
-                echo "outputs=$OUTPUTS" >> $GITHUB_OUTPUT  # Write the outputs to GITHUB_OUTPUT
-                
-                # Handle null values, strings, and numbers in the outputs by converting everything to a string and replacing null with '-'
-                OUTPUTS_TABLE=$(echo "$OUTPUTS" | jq -r 'to_entries | ["Output | Value", "--- | ---"] + map(.key + " | " + (if .value == null then "-" else (.value | tostring) end)) | .[]')
-                #print outputs table to stdout
-                echo "$OUTPUTS_TABLE"
-               done
-
-        ##################################### LEFT OFF HERE #######################################
-        # TODO: move the submission and monitoring into the for loop to occur for each input file #
-        ###########################################################################################
+          # Set common environment variables
+          TOKEN="${{ steps.auth.outputs.access_token }}"
+          NAMESPACE="warp-pipelines"
+          WORKSPACE="WARP Tests"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
+          TEST_TYPE="Plumbing"  # Placeholder for now
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+
+          ########################################
+          # SET PIPELINE SPECIFIC VARIABLES HERE #
+          ########################################
+          PIPELINE_NAME="TestIlluminaGenotypingArray"
+          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
+          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
+
+          # Function to call the Firecloud API using the firecloud_api.py script 
+          firecloud_action() {
+            python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+          }
+
+          # Convert USE_CALL_CACHE to a boolean-friendly format
+          USE_CALL_CACHE_BOOL=$( [ "$USE_CALL_CACHE" = "true" ] && echo true || echo false )
+          UPDATE_TRUTH_BOOL=$( [ "$UPDATE_TRUTH" = "true" ] && echo true || echo false )
+
+          # Create the submission_data.json file
+          SUBMISSION_DATA_FILE="submission_data.json"
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "warp-pipelines",
+            "methodConfigurationName": "$PIPELINE_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+          cat "$SUBMISSION_DATA_FILE"
+
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES=""
+          ALL_OUTPUTS=""
+
+          # Loop through each file in the test inputs directory
+          INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
+          echo "Test inputs directory: $INPUTS_DIR"
+          for input_file in $INPUTS_DIR/*; do
+            echo "Processing input file: $input_file"
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
+                                                                                --results_path $RESULTS_PATH \
+                                                                                --inputs_json $input_file \
+                                                                                --update_truth $UPDATE_TRUTH_BOOL)
+            echo "Uploading test input file: $test_input_file"
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file"
+
+            echo "Submitting job for input file: $input_file"
+            SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+
+            if [ -z "$SUBMISSION_ID" ]; then
+              echo "Submission failed for input file: $input_file"
+              continue
+            fi
+
+            echo "Submission ID: $SUBMISSION_ID"
+
+            # Poll submission status
+            echo "Polling submission status..."
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+
+            if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+              continue
+            fi
+
+            # Parse workflow statuses and append to aggregate
+            WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
+            ALL_WORKFLOW_STATUSES+="$WORKFLOW_STATUSES"$'\n'
+
+            # Retrieve workflow outputs and append to aggregate
+            echo "Retrieving workflow outputs..."
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+            done
+          done
+
+          # Generate final markdown tables
+          WORKFLOW_TABLE=$(echo "$ALL_WORKFLOW_STATUSES" | jq -R 'split("\n") | map(select(length > 0)) | ["Workflow ID | Status", "--- | ---"] + . | .[]')
+          OUTPUTS_TABLE=$(echo "$ALL_OUTPUTS" | jq -R 'split("\n") | map(select(length > 0)) | ["Output | Value", "--- | ---"] + map(split(": ") | join(" | ")) | .[]')
+
+          # Print final summaries
+          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "$WORKFLOW_TABLE" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+
+          echo "## Combined Workflow Outputs" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+          echo "$OUTPUTS_TABLE" >> $GITHUB_STEP_SUMMARY
+          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
 
       - name: Print Summary on Success
         if: success()
         run: |
-            echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
-            echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-            echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-            echo "" >> $GITHUB_STEP_SUMMARY
-  
-            echo "## Workflows and their statuses" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-  
-            echo "## Workflow Outputs" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo " :shipit: " >> $GITHUB_STEP_SUMMARY
+          echo "# :white_check_mark: Pipeline Execution Summary :white_check_mark:" >> $GITHUB_STEP_SUMMARY
 
       - name: Print Summary on Failure
         if: failure()
         run: |
-            echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
-            echo "- **Pipeline Name**: IlluminaGenotypingArray" >> $GITHUB_STEP_SUMMARY
-            echo "- **Submission ID**: ${{ steps.pipeline_run.outputs.submission_id }}" >> $GITHUB_STEP_SUMMARY
-            echo "" >> $GITHUB_STEP_SUMMARY
-  
-            echo "## Workflows and their statuses (if available)" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo "${{ steps.pipeline_run.outputs.workflow_statuses }}" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-  
-            echo "## Workflow Outputs (if available)" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-            echo "${{ steps.pipeline_run.outputs.outputs }}" >> $GITHUB_STEP_SUMMARY
-            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
\ No newline at end of file
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY

From eeb8eb392205332e3ebf63ed1b5afbc6e00fa8c6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 11:39:42 -0500
Subject: [PATCH 052/165] formatting

---
 .../test_illumina_genotyping_array.yml        | 37 ++++++++++++++-----
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 050c3ce339..2530566354 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -19,6 +19,7 @@ on:
       - 'verification/**'
       - '.github/workflows/test_illumina_genotyping_array.yml'
 
+
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
     inputs:
@@ -30,7 +31,6 @@ on:
         description: 'Update truth files (default: false)'
         required: false
         default: "false"
-
 env:
   PROJECT_NAME: WARP
   # Github repo name
@@ -53,7 +53,10 @@ jobs:
         uses: 'google-github-actions/auth@v2'
         with:
           token_format: 'access_token'
+          # Centralized in dsp-tools-k8s; ask in #dsp-devops-champions for help troubleshooting
+          # This is provided by the DevOps team - do not change!
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
+          # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
@@ -74,28 +77,43 @@ jobs:
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
+          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
           TEST_TYPE="Plumbing"  # Placeholder for now
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-
+          
           ########################################
           # SET PIPELINE SPECIFIC VARIABLES HERE #
           ########################################
           PIPELINE_NAME="TestIlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          # TODO: Need to set the truth and result paths appropriately
           TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
-          # Function to call the Firecloud API using the firecloud_api.py script 
+          # Function to call the Firecloud API using the firecloud_api.py script
           firecloud_action() {
             python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
           }
 
-          # Convert USE_CALL_CACHE to a boolean-friendly format
-          USE_CALL_CACHE_BOOL=$( [ "$USE_CALL_CACHE" = "true" ] && echo true || echo false )
-          UPDATE_TRUTH_BOOL=$( [ "$UPDATE_TRUTH" = "true" ] && echo true || echo false )
-
-          # Create the submission_data.json file
+          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+          USE_CALL_CACHE="true"
+          if [ "$USE_CALL_CACHE" = "true" ]; then
+              USE_CALL_CACHE_BOOL=true
+          else
+              USE_CALL_CACHE_BOOL=false
+          fi
+          
+          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH=true
+          else
+              UPDATE_TRUTH=false
+          fi
+          
+          # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
+        
+          # Use a heredoc to generate the JSON file content dynamically
           cat <<EOF > "$SUBMISSION_DATA_FILE"
           {
             "methodConfigurationNamespace": "warp-pipelines",
@@ -110,13 +128,12 @@ jobs:
           }
           EOF
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
-          cat "$SUBMISSION_DATA_FILE"
 
           # Initialize variables to aggregate statuses and outputs
           ALL_WORKFLOW_STATUSES=""
           ALL_OUTPUTS=""
 
-          # Loop through each file in the test inputs directory
+          # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           echo "Test inputs directory: $INPUTS_DIR"
           for input_file in $INPUTS_DIR/*; do

From 16f4a7cfa384188407a2938798b8b76badbc6423 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 11:43:57 -0500
Subject: [PATCH 053/165] mardkiown formatting

---
 .../test_illumina_genotyping_array.yml        | 20 +++++++------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2530566354..13765a3e32 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -130,12 +130,13 @@ jobs:
           echo "Created submission data file: $SUBMISSION_DATA_FILE"
 
           # Initialize variables to aggregate statuses and outputs
-          ALL_WORKFLOW_STATUSES=""
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
           ALL_OUTPUTS=""
 
           # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           echo "Test inputs directory: $INPUTS_DIR"
+          
           for input_file in $INPUTS_DIR/*; do
             echo "Processing input file: $input_file"
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
@@ -165,8 +166,8 @@ jobs:
             fi
 
             # Parse workflow statuses and append to aggregate
-            WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + ": " + .value) | .[]')
-            ALL_WORKFLOW_STATUSES+="$WORKFLOW_STATUSES"$'\n'
+            WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            ALL_WORKFLOW_STATUSES+=$'\n'"$WORKFLOW_STATUSES"
 
             # Retrieve workflow outputs and append to aggregate
             echo "Retrieving workflow outputs..."
@@ -180,16 +181,9 @@ jobs:
           WORKFLOW_TABLE=$(echo "$ALL_WORKFLOW_STATUSES" | jq -R 'split("\n") | map(select(length > 0)) | ["Workflow ID | Status", "--- | ---"] + . | .[]')
           OUTPUTS_TABLE=$(echo "$ALL_OUTPUTS" | jq -R 'split("\n") | map(select(length > 0)) | ["Output | Value", "--- | ---"] + map(split(": ") | join(" | ")) | .[]')
 
-          # Print final summaries
-          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "$WORKFLOW_TABLE" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-
-          echo "## Combined Workflow Outputs" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
-          echo "$OUTPUTS_TABLE" >> $GITHUB_STEP_SUMMARY
-          echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
+         # Generate final markdown table for workflow statuses
+         echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+         echo "$ALL_WORKFLOW_STATUSES" >> $GITHUB_STEP_SUMMARY
 
       - name: Print Summary on Success
         if: success()

From 13c26f688bf97ba07b068f136093cd27f2b03da5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 11:46:09 -0500
Subject: [PATCH 054/165] mardkiown formatting

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 13765a3e32..640b8891d5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -181,9 +181,9 @@ jobs:
           WORKFLOW_TABLE=$(echo "$ALL_WORKFLOW_STATUSES" | jq -R 'split("\n") | map(select(length > 0)) | ["Workflow ID | Status", "--- | ---"] + . | .[]')
           OUTPUTS_TABLE=$(echo "$ALL_OUTPUTS" | jq -R 'split("\n") | map(select(length > 0)) | ["Output | Value", "--- | ---"] + map(split(": ") | join(" | ")) | .[]')
 
-         # Generate final markdown table for workflow statuses
-         echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-         echo "$ALL_WORKFLOW_STATUSES" >> $GITHUB_STEP_SUMMARY
+          # Generate final markdown table for workflow statuses
+          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+          echo "$ALL_WORKFLOW_STATUSES" >> $GITHUB_STEP_SUMMARY
 
       - name: Print Summary on Success
         if: success()

From 53ea6108134e0f5cb902b8286856ae739841902d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 11:47:38 -0500
Subject: [PATCH 055/165] mardkiown formatting

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 640b8891d5..2844aef01c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -105,9 +105,9 @@ jobs:
           
           # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH=true
+              UPDATE_TRUTH_BOOL=true
           else
-              UPDATE_TRUTH=false
+              UPDATE_TRUTH_BOOL=false
           fi
           
           # Create the submission_data.json file which will be the same for all inputs

From dec0c7b4b0cc113295ebebb830ffd78ce7d3f29d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 12:46:57 -0500
Subject: [PATCH 056/165] make submissions run in parallel

---
 .../test_illumina_genotyping_array.yml        | 50 +++++++++++--------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2844aef01c..16742e0d2d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -132,6 +132,10 @@ jobs:
           # Initialize variables to aggregate statuses and outputs
           ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
           ALL_OUTPUTS=""
+          
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
@@ -155,35 +159,41 @@ jobs:
             fi
 
             echo "Submission ID: $SUBMISSION_ID"
-
-            # Poll submission status
-            echo "Polling submission status..."
-            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+            SUBMISSION_IDS+=("$SUBMISSION_ID")
+          done
+          
+          echo "Monitoring the status of submitted workflows..."
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 
             if [ -z "$RESPONSE" ]; then
               echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
               continue
             fi
-
-            # Parse workflow statuses and append to aggregate
-            WORKFLOW_STATUSES=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            ALL_WORKFLOW_STATUSES+=$'\n'"$WORKFLOW_STATUSES"
-
-            # Retrieve workflow outputs and append to aggregate
-            echo "Retrieving workflow outputs..."
+          
+            # Parse and store workflow statuses
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "Statuses for submission $SUBMISSION_ID:"
+            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+            # Append to aggregate statuses
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+          
+            # Optionally, retrieve workflow outputs
+            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
             for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
               WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
               ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+            done
+          
+            # Generate final summary tables
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              echo "Submission ID: $SUBMISSION_ID" >> $GITHUB_STEP_SUMMARY
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
             done
-          done
-
-          # Generate final markdown tables
-          WORKFLOW_TABLE=$(echo "$ALL_WORKFLOW_STATUSES" | jq -R 'split("\n") | map(select(length > 0)) | ["Workflow ID | Status", "--- | ---"] + . | .[]')
-          OUTPUTS_TABLE=$(echo "$ALL_OUTPUTS" | jq -R 'split("\n") | map(select(length > 0)) | ["Output | Value", "--- | ---"] + map(split(": ") | join(" | ")) | .[]')
-
-          # Generate final markdown table for workflow statuses
-          echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-          echo "$ALL_WORKFLOW_STATUSES" >> $GITHUB_STEP_SUMMARY
 
       - name: Print Summary on Success
         if: success()

From c15085bd48722eaf2892a57c0f9b4a49ed224b72 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 14:37:43 -0500
Subject: [PATCH 057/165] clean up debugging prints

---
 scripts/firecloud_api/firecloud_api.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 91134ae4fa..743b01dadd 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -167,19 +167,14 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         with open(test_inputs, 'r') as file:
             inputs_json = json.load(file)
             print("Test inputs loaded successfully.")
-            print(f"Original Test inputs content: {json.dumps(inputs_json, indent=2)}")
             inputs_json = self.quote_values(inputs_json)
             config["inputs"] = inputs_json
-            print(f"Quoted Test inputs content: {json.dumps(inputs_json, indent=2)}")
 
         print(f"Constructed URL: {url}")
         print(f"Headers: {self.headers}")
-        print(f"Config to be posted: {json.dumps(config, indent=2)}")  # Pretty-print config
 
     # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
-        # print the updated config, can delete this line later
-        print(config)
         print(f"Response status code: {response.status_code}")
         print(f"Response text: {response.text}")
 

From 7bcbc983c035d964906d1a4240b91aa5a3c10984 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 14:54:04 -0500
Subject: [PATCH 058/165] clean up debugging prints

---
 scripts/firecloud_api/firecloud_api.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 743b01dadd..e1c6052b7b 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -160,8 +160,6 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
-        # print the config, can delete this line later
-        print(config)
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
@@ -170,13 +168,11 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             inputs_json = self.quote_values(inputs_json)
             config["inputs"] = inputs_json
 
-        print(f"Constructed URL: {url}")
-        print(f"Headers: {self.headers}")
 
     # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
-        print(f"Response status code: {response.status_code}")
-        print(f"Response text: {response.text}")
+        #print(f"Response status code: {response.status_code}")
+        #print(f"Response text: {response.text}")
 
         # Check if the test inputs were uploaded successfully
         if response.status_code == 200:

From bab0a822883138b2acaf23ab1cc7d150166f42a7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:14:35 -0500
Subject: [PATCH 059/165] hyperlink?

---
 .../workflows/test_illumina_genotyping_array.yml    | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 16742e0d2d..38a2668dee 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -188,11 +188,20 @@ jobs:
               done
             done
           
-            # Generate final summary tables
+            # Generate final summary tables with hyperlinks for Submission IDs
             echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
             for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              echo "Submission ID: $SUBMISSION_ID" >> $GITHUB_STEP_SUMMARY
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+            
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+              
+              # Add the workflows and statuses for this submission
               echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+              
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
             done
 
       - name: Print Summary on Success

From 19433aa735bcb346052ef6e92eabb60367b5d6fc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:32:17 -0500
Subject: [PATCH 060/165] is call cache working?????

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 ++++++++------
 scripts/firecloud_api/firecloud_api.py             |  2 +-
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 38a2668dee..4d6a5f61e7 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -96,12 +96,14 @@ jobs:
           }
 
           # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-          USE_CALL_CACHE="true"
-          if [ "$USE_CALL_CACHE" = "true" ]; then
-              USE_CALL_CACHE_BOOL=true
-          else
-              USE_CALL_CACHE_BOOL=false
-          fi
+          echo "useCallCache input: $USE_CALL_CACHE"  # Debugging output
+            if [ "$USE_CALL_CACHE" == "true" ]; then
+                USE_CALL_CACHE_BOOL=true
+            else
+                USE_CALL_CACHE_BOOL=false
+            fi
+          echo "USE_CALL_CACHE_BOOL: $USE_CALL_CACHE_BOOL"  # Debugging output
+
           
           # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
           if [ "$UPDATE_TRUTH" = "true" ]; then
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index e1c6052b7b..f07ce9ad4c 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -127,7 +127,7 @@ def poll_submission_status(self, submission_id):
                 break
 
             # Wait for 60 seconds before polling again
-            time.sleep(60)
+            time.sleep(20)
 
         return workflow_status_map
 

From 66ed27c46159b54a277d2d5381ce4f390be4ec90 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:44:25 -0500
Subject: [PATCH 061/165] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4d6a5f61e7..78cef8722c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -31,6 +31,10 @@ on:
         description: 'Update truth files (default: false)'
         required: false
         default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific'
+        required: true
+
 env:
   PROJECT_NAME: WARP
   # Github repo name
@@ -77,8 +81,7 @@ jobs:
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          # TODO: Need to get the test type (Plumbing/ Scientific) from inputs or based on the target branch        
-          TEST_TYPE="Plumbing"  # Placeholder for now
+          TEST_TYPE="${{ github.event.inputs.testType }}"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
           ########################################
@@ -102,7 +105,6 @@ jobs:
             else
                 USE_CALL_CACHE_BOOL=false
             fi
-          echo "USE_CALL_CACHE_BOOL: $USE_CALL_CACHE_BOOL"  # Debugging output
 
           
           # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)

From 1f1d7e042f581d0b55fdbd8b137cffe34c490885 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:51:19 -0500
Subject: [PATCH 062/165] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 78cef8722c..55f8590a6c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -145,12 +145,12 @@ jobs:
           INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
           echo "Test inputs directory: $INPUTS_DIR"
           
-          for input_file in $INPUTS_DIR/*; do
+          for input_file in "$INPUTS_DIR"/*; do
             echo "Processing input file: $input_file"
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path $TRUTH_PATH \
-                                                                                --results_path $RESULTS_PATH \
-                                                                                --inputs_json $input_file \
-                                                                                --update_truth $UPDATE_TRUTH_BOOL)
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                                                                                --results_path "$RESULTS_PATH" \
+                                                                                --inputs_json "$input_file" \
+                                                                                --update_truth "$UPDATE_TRUTH_BOOL")
             echo "Uploading test input file: $test_input_file"
             firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file"
 

From 75e138aecc91d48e21061e97facafcd793eb1668 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:56:43 -0500
Subject: [PATCH 063/165] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 55f8590a6c..236ff77da9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -142,11 +142,12 @@ jobs:
           declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
-          INPUTS_DIR=$PIPELINE_DIR/test_inputs/$TEST_TYPE
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Test inputs directory: $INPUTS_DIR"
           
-          for input_file in "$INPUTS_DIR"/*; do
+          for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
+            if [ -f "$input_file" ]; then
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
                                                                                 --results_path "$RESULTS_PATH" \
                                                                                 --inputs_json "$input_file" \

From 4d0cbb531d27cd61be44945925638c4823567f85 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 15:58:40 -0500
Subject: [PATCH 064/165] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 236ff77da9..53d2fefed2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -142,12 +142,12 @@ jobs:
           declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
+          echo "$TEST_TYPE"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Test inputs directory: $INPUTS_DIR"
           
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
-            if [ -f "$input_file" ]; then
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
                                                                                 --results_path "$RESULTS_PATH" \
                                                                                 --inputs_json "$input_file" \

From f7be47cf17192022dfdca0085a5ee1ba41205c0f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:23:30 -0500
Subject: [PATCH 065/165] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 53d2fefed2..6afd8b6107 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -39,6 +39,7 @@ env:
   PROJECT_NAME: WARP
   # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
+  TEST_TYPE: ${{ github.event.inputs.testType || 'Plumbing' }} # Default to 'Plumbing' if not set
 
 jobs:
   run_pipeline:

From 7c4abaca14621218df7ba74e901bb58ffd0f2952 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:26:12 -0500
Subject: [PATCH 066/165] add test type

---
 .../test_illumina_genotyping_array.yml         | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 6afd8b6107..1241def06d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -39,7 +39,7 @@ env:
   PROJECT_NAME: WARP
   # Github repo name
   REPOSITORY_NAME: ${{ github.event.repository.name }}
-  TEST_TYPE: ${{ github.event.inputs.testType || 'Plumbing' }} # Default to 'Plumbing' if not set
+
 
 jobs:
   run_pipeline:
@@ -73,6 +73,22 @@ jobs:
           pwd
           ls -lht
 
+      - name: Set TEST_TYPE
+        run: |
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            echo "TEST_TYPE=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+          else
+            echo "TEST_TYPE=Plumbing" >> $GITHUB_ENV
+          fi
+
+      - name: Debug TEST_TYPE
+        run: |
+          echo "TEST_TYPE: $TEST_TYPE"
+          if [ -z "$TEST_TYPE" ]; then
+              echo "Error: TEST_TYPE is not set."
+              exit 1
+          fi
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |

From 4949be8f0895aaaa3c770bd1d98958c166936e80 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:28:37 -0500
Subject: [PATCH 067/165] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1241def06d..90e35a3ccf 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -79,6 +79,7 @@ jobs:
             echo "TEST_TYPE=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
           else
             echo "TEST_TYPE=Plumbing" >> $GITHUB_ENV
+            cat $GITHUB_ENV
           fi
 
       - name: Debug TEST_TYPE
@@ -159,7 +160,7 @@ jobs:
           declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
-          echo "$TEST_TYPE"
+          cat "$TEST_TYPE"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Test inputs directory: $INPUTS_DIR"
           

From 4df055a96e36a2ebfce63d57ffad9a909f248b01 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:31:09 -0500
Subject: [PATCH 068/165] add test type

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 90e35a3ccf..142a8a405a 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -160,7 +160,7 @@ jobs:
           declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
-          cat "$TEST_TYPE"
+          echo "the test type is set to $TEST_TYPE"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           echo "Test inputs directory: $INPUTS_DIR"
           

From bcf96daae82c0d9be02b70464f7ea17a0e35a6a7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:34:25 -0500
Subject: [PATCH 069/165] add defaults?

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 142a8a405a..59f02a77ed 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -97,9 +97,9 @@ jobs:
           TOKEN="${{ steps.auth.outputs.access_token }}"
           NAMESPACE="warp-pipelines"
           WORKSPACE="WARP Tests"
-          USE_CALL_CACHE="${{ github.event.inputs.useCallCache }}"
-          UPDATE_TRUTH="${{ github.event.inputs.updateTruth }}"
-          TEST_TYPE="${{ github.event.inputs.testType }}"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
           ########################################

From e32de1b5299a2ce58382cf298ae19f8e192ba020 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 18 Nov 2024 16:38:12 -0500
Subject: [PATCH 070/165] remove debugging

---
 .../test_illumina_genotyping_array.yml        | 20 -------------------
 1 file changed, 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 59f02a77ed..4d72d29961 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -73,23 +73,6 @@ jobs:
           pwd
           ls -lht
 
-      - name: Set TEST_TYPE
-        run: |
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            echo "TEST_TYPE=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
-          else
-            echo "TEST_TYPE=Plumbing" >> $GITHUB_ENV
-            cat $GITHUB_ENV
-          fi
-
-      - name: Debug TEST_TYPE
-        run: |
-          echo "TEST_TYPE: $TEST_TYPE"
-          if [ -z "$TEST_TYPE" ]; then
-              echo "Error: TEST_TYPE is not set."
-              exit 1
-          fi
-
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -117,7 +100,6 @@ jobs:
           }
 
           # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-          echo "useCallCache input: $USE_CALL_CACHE"  # Debugging output
             if [ "$USE_CALL_CACHE" == "true" ]; then
                 USE_CALL_CACHE_BOOL=true
             else
@@ -160,9 +142,7 @@ jobs:
           declare -A WORKFLOW_STATUSES
 
           # Loop through each file in the appropriate test inputs directory
-          echo "the test type is set to $TEST_TYPE"
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-          echo "Test inputs directory: $INPUTS_DIR"
           
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"

From c5031f11600828a06e2fd6e365376bb866f671ba Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 19 Nov 2024 13:08:02 -0500
Subject: [PATCH 071/165] remove debugging

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4d72d29961..70ef43d422 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -91,6 +91,7 @@ jobs:
           PIPELINE_NAME="TestIlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           # TODO: Need to set the truth and result paths appropriately
+          # We may want to keep the truth and resuts buckets separate for TTL reasons
           TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 

From e416a3c9f35cfe4447913592639d2f6d783fff8d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 19 Nov 2024 14:12:09 -0500
Subject: [PATCH 072/165] fix truth path

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 70ef43d422..261f10c7b3 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -32,7 +32,7 @@ on:
         required: false
         default: "false"
       testType:
-        description: 'Specify the type of test (Plumbing or Scientific'
+        description: 'Specify the type of test (Plumbing or Scientific)'
         required: true
 
 env:
@@ -92,7 +92,7 @@ jobs:
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           # TODO: Need to set the truth and result paths appropriately
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script

From 3f9746db8e3df0e46e4d91be311e8cbac2ebaf3f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 19 Nov 2024 14:26:42 -0500
Subject: [PATCH 073/165] hard code to master branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 261f10c7b3..7e374c2146 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -91,8 +91,9 @@ jobs:
           PIPELINE_NAME="TestIlluminaGenotypingArray"
           PIPELINE_DIR="pipelines/broad/genotyping/illumina"
           # TODO: Need to set the truth and result paths appropriately
+          # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script

From 7099697f96748c080c61cf98e86e8770edd9db07 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 11:50:41 -0500
Subject: [PATCH 074/165] set pipeline branch-specific variables

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7e374c2146..253ba7a4ce 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -52,6 +52,8 @@ jobs:
     steps:
       # actions/checkout MUST come before auth
       - uses: 'actions/checkout@v3'
+        with:
+          ref: ${{ github.ref }}
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
@@ -73,6 +75,14 @@ jobs:
           pwd
           ls -lht
 
+      - name: Verify checked-out branch
+        run: |
+        echo "Branch: ${{ github.ref }}"
+
+      - name: Set pipeline branch-specific variables
+        run: |
+          echo "CURRENT_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -93,7 +103,7 @@ jobs:
           # TODO: Need to set the truth and result paths appropriately
           # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/${CURRENT_BRANCH}"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script
@@ -185,7 +195,7 @@ jobs:
             # Append to aggregate statuses
             WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
           
-            # Optionally, retrieve workflow outputs
+            # retrieve workflow outputs
             echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
             for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
               WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")

From f22fdab8dc1a80b860f81537811058def1bcc066 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 11:51:35 -0500
Subject: [PATCH 075/165] set pipeline branch-specific variables

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 253ba7a4ce..22ee7a2f2c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -77,7 +77,7 @@ jobs:
 
       - name: Verify checked-out branch
         run: |
-        echo "Branch: ${{ github.ref }}"
+          echo "Branch: ${{ github.ref }}"
 
       - name: Set pipeline branch-specific variables
         run: |

From b211c5163b693419f63398aebafe7a51d9fc8069 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 11:54:32 -0500
Subject: [PATCH 076/165] revert

---
 .github/workflows/test_illumina_genotyping_array.yml | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 22ee7a2f2c..aec6ebcbc5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -52,8 +52,6 @@ jobs:
     steps:
       # actions/checkout MUST come before auth
       - uses: 'actions/checkout@v3'
-        with:
-          ref: ${{ github.ref }}
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
@@ -75,14 +73,6 @@ jobs:
           pwd
           ls -lht
 
-      - name: Verify checked-out branch
-        run: |
-          echo "Branch: ${{ github.ref }}"
-
-      - name: Set pipeline branch-specific variables
-        run: |
-          echo "CURRENT_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV
-
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -103,7 +93,7 @@ jobs:
           # TODO: Need to set the truth and result paths appropriately
           # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/${CURRENT_BRANCH}"
+          "TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script

From 441561eb1463e904a2f2be5e53afd2ec48a763fe Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 11:59:08 -0500
Subject: [PATCH 077/165] revert

---
 .github/workflows/test_illumina_genotyping_array.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index aec6ebcbc5..52977096b7 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,4 +1,3 @@
-
 name: Test Illumina Genotyping Array
 
 # Controls when the workflow will run
@@ -93,7 +92,7 @@ jobs:
           # TODO: Need to set the truth and result paths appropriately
           # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          "TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script
@@ -217,4 +216,4 @@ jobs:
       - name: Print Summary on Failure
         if: failure()
         run: |
-          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
+          echo "# :x: Pipeline Execution Summary (on Failure) :x: " >> $GITHUB_STEP_SUMMARY
\ No newline at end of file

From 93b09be597a1ece87aa304069376892326a3e809 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:06:20 -0500
Subject: [PATCH 078/165] Verify checked-out branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 52977096b7..bbcc635a25 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -50,7 +50,9 @@ jobs:
 
     steps:
       # actions/checkout MUST come before auth
-      - uses: 'actions/checkout@v3'
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.ref_name }}
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
@@ -72,6 +74,11 @@ jobs:
           pwd
           ls -lht
 
+      - name: Verify checked-out branch
+        run: |
+          echo "Branch: ${{ github.ref }}"
+
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |

From a8f2523e775b10491df5787fc60e1daaf4fa2065 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:07:52 -0500
Subject: [PATCH 079/165] Verify checked-out branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index bbcc635a25..5dac15903e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -52,7 +52,7 @@ jobs:
       # actions/checkout MUST come before auth
       - uses: actions/checkout@v3
         with:
-          ref: ${{ github.ref_name }}
+          ref: ${{ github.ref }}
 
       - id: 'auth'
         name: 'Authenticate to Google Cloud'
@@ -76,7 +76,7 @@ jobs:
 
       - name: Verify checked-out branch
         run: |
-          echo "Branch: ${{ github.ref }}"
+          echo "Branch: ${{ github.ref_name }}"
 
 
       - name: Submit job, poll status, and get outputs

From 7ae262a370d84313369f865b1de4632a2816321e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:10:10 -0500
Subject: [PATCH 080/165] Verify checked-out branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5dac15903e..d1a1095ea6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -76,7 +76,7 @@ jobs:
 
       - name: Verify checked-out branch
         run: |
-          echo "Branch: ${{ github.ref_name }}"
+          echo "Branch: ${{ github.head_ref }}"
 
 
       - name: Submit job, poll status, and get outputs

From 23eef10311b1e46b24bc38d26424f145d8ccf0e7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:12:12 -0500
Subject: [PATCH 081/165] look at config

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index f07ce9ad4c..9d1247029e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -160,6 +160,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
+        print(f"Current method configuration: {config}")
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:

From f49229c9a475a5bf6fd1b9346a56964ef95316ef Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:14:28 -0500
Subject: [PATCH 082/165] look at config

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 9d1247029e..9d8ff61822 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -160,7 +160,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
-        print(f"Current method configuration: {config}")
+        print(f"Current method configuration: {response}")
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:

From bebdb1e18435b08b864f7b74d1ab4dfc83ffb8b6 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:21:13 -0500
Subject: [PATCH 083/165] update config methodVersion

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 scripts/firecloud_api/firecloud_api.py               | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d1a1095ea6..3a6e9fae30 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -159,7 +159,7 @@ jobs:
                                                                                 --inputs_json "$input_file" \
                                                                                 --update_truth "$UPDATE_TRUTH_BOOL")
             echo "Uploading test input file: $test_input_file"
-            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file"
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name "${{ github.head_ref }}"
 
             echo "Submitting job for input file: $input_file"
             SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 9d8ff61822..2057191c95 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -145,7 +145,7 @@ def quote_values(self, data):
             return data  # Return as-is if it's not a string, int, float, or bool
 
 
-    def upload_test_inputs(self, pipeline_name, test_inputs):
+    def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         """
         Uploads test inputs to the workspace via Firecloud API.
 
@@ -160,7 +160,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
-        print(f"Current method configuration: {response}")
+        print(f"Current method configuration: {config}")
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:
@@ -169,6 +169,11 @@ def upload_test_inputs(self, pipeline_name, test_inputs):
             inputs_json = self.quote_values(inputs_json)
             config["inputs"] = inputs_json
 
+        # update the config with the new branch name
+        print(f"Updating methodVersion with branch name: {branch_name}")
+        config["methodRepoMethod"]["methodVersion"] = branch_name
+        print(f"Updated method configuration: {config}")
+
 
     # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)

From 78f1e3ef5b46e036a33cfdb7efe3f73aea6b31e1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:22:25 -0500
Subject: [PATCH 084/165] update config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 2057191c95..61c8e270ca 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -204,6 +204,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
     parser.add_argument('--pipeline_name', help='Pipeline name (required for get_outputs)')
     parser.add_argument('--submission_data_file', help='Path to submission data JSON file (required for submit)')
     parser.add_argument('--test_input_file', help='Path to test inputs JSON file (required for upload_test_inputs)')
+    parser.add_argument('--branch_name', help='Branch name for the method configuration (required for upload)')
 
     args = parser.parse_args()
 

From b48084f418d845e4390a8d1ea03d41f64c46d352 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:25:54 -0500
Subject: [PATCH 085/165] update config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 61c8e270ca..884728d333 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -247,8 +247,8 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
               print("No workflows found or an error occurred.", file=sys.stderr)
 
     elif args.action == 'upload_test_inputs':
-        if not all([args.pipeline_name, args.test_input_file]):
-            print("For 'upload_test_inputs', --pipeline_name and --test_input_file are required.", file=sys.stderr)
+        if not all([args.pipeline_name, args.test_input_file, args.branch_name]):
+            print("For 'upload_test_inputs', --pipeline_name, --test_input_file and --branch_name are required.", file=sys.stderr)
         else:
-            success = firecloud_api.upload_test_inputs(args.pipeline_name, args.test_input_file)
+            success = firecloud_api.upload_test_inputs(args.pipeline_name, args.test_input_file, args.branch_name)
             print(success)
\ No newline at end of file

From c4e57a1fdef23e51d59dbd93b0c93389016ac018 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:28:46 -0500
Subject: [PATCH 086/165] update config methodVersion

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3a6e9fae30..1c6e667e47 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -122,7 +122,7 @@ jobs:
               UPDATE_TRUTH_BOOL=false
           fi
           
-          # Create the submission_data.json file which will be the same for all inputs
+          # Create the submission_data.json file which will be the same for all inputs 
           SUBMISSION_DATA_FILE="submission_data.json"
         
           # Use a heredoc to generate the JSON file content dynamically

From d758cbbe21c4d97bb2ac64a9c20f309d30110c53 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 13:45:45 -0500
Subject: [PATCH 087/165] update config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 884728d333..4ba1456a33 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -172,6 +172,9 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         # update the config with the new branch name
         print(f"Updating methodVersion with branch name: {branch_name}")
         config["methodRepoMethod"]["methodVersion"] = branch_name
+
+        # Increment the methodConfigVersion
+        config["methodConfigVersion"] += 1  # Increment version number by  1
         print(f"Updated method configuration: {config}")
 
 

From 1b41e0ce5d347a012f6d4f6d8204c7de71daa565 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:33:44 -0500
Subject: [PATCH 088/165] test out config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 4ba1456a33..3cf729bc56 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -142,7 +142,7 @@ def quote_values(self, data):
         elif isinstance(data, (str, int, float, bool)):
             return f"\"{data}\""
         else:
-            return data  # Return as-is if it's not a string, int, float, or bool
+            return data  # Return as-is if it's not a string, int, float, or boolean
 
 
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):

From 7b97f221dd07b403ceedb49117f964207dafe532 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:36:33 -0500
Subject: [PATCH 089/165] test out config methodVersion

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 scripts/firecloud_api/firecloud_api.py               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1c6e667e47..3a6e9fae30 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -122,7 +122,7 @@ jobs:
               UPDATE_TRUTH_BOOL=false
           fi
           
-          # Create the submission_data.json file which will be the same for all inputs 
+          # Create the submission_data.json file which will be the same for all inputs
           SUBMISSION_DATA_FILE="submission_data.json"
         
           # Use a heredoc to generate the JSON file content dynamically
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 3cf729bc56..dff53dfb37 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -175,7 +175,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
 
         # Increment the methodConfigVersion
         config["methodConfigVersion"] += 1  # Increment version number by  1
-        print(f"Updated method configuration: {config}")
+        print(f"Updated method configuration: {json.dumps(config, indent=2)}")
 
 
     # post the updated method config to the workspace

From 5269367c48a0eee159c91fafa72c727727b81bfa Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:37:40 -0500
Subject: [PATCH 090/165] test out config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index dff53dfb37..5c5584c64d 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -160,7 +160,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         # get the current method configuration
         response = requests.get(url, headers=self.headers)
         config = response.json()
-        print(f"Current method configuration: {config}")
+        print(f"Current method configuration: {json.dumps(config, indent=2)}")
         # update the config with the new inputs
         print(f"Opening test inputs file: {test_inputs}")
         with open(test_inputs, 'r') as file:

From 1da10584a924be68d9c913a13fe56dd33d2c73b2 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:39:11 -0500
Subject: [PATCH 091/165] test out config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 5c5584c64d..cd0ded721a 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -180,8 +180,8 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
 
     # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
-        #print(f"Response status code: {response.status_code}")
-        #print(f"Response text: {response.text}")
+        print(f"Response status code: {response.status_code}")
+        print(f"Response text: {response.text}")
 
         # Check if the test inputs were uploaded successfully
         if response.status_code == 200:

From 2748295a1cae16fbcf45d263bcda2578b65462e4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:44:40 -0500
Subject: [PATCH 092/165] test out config methodVersion

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3a6e9fae30..47f4def20e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -162,6 +162,7 @@ jobs:
             firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name "${{ github.head_ref }}"
 
             echo "Submitting job for input file: $input_file"
+            cat "$SUBMISSION_DATA_FILE"
             SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
 
             if [ -z "$SUBMISSION_ID" ]; then

From 0ff43177f9a4558fdeb8d63b2662c2048855e7ac Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 14:48:29 -0500
Subject: [PATCH 093/165] test out config methodVersion

---
 scripts/firecloud_api/firecloud_api.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index cd0ded721a..622b22a8e1 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -81,6 +81,7 @@ def create_submission(self, submission_data):
             return submission_id
         else:
             print(f"Failed to create submission. Status code: {response.status_code}")
+            print(f"Response content: {response.text}")
             return None
 
     

From d9a21a0dc66102801fcac6a7ffa3fc48c15c9b3c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 15:00:43 -0500
Subject: [PATCH 094/165] methoduri

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 scripts/firecloud_api/firecloud_api.py               | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 47f4def20e..8346626ed1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -159,6 +159,7 @@ jobs:
                                                                                 --inputs_json "$input_file" \
                                                                                 --update_truth "$UPDATE_TRUTH_BOOL")
             echo "Uploading test input file: $test_input_file"
+          
             firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name "${{ github.head_ref }}"
 
             echo "Submitting job for input file: $input_file"
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 622b22a8e1..0ea03e6e5d 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -171,6 +171,13 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             config["inputs"] = inputs_json
 
         # update the config with the new branch name
+
+        # Construct the methodUri with the branch name
+        base_url = "github.com/broadinstitute/warp/TestIlluminaGenotypingArray"
+        method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
+        print(f"Updating methodUri with branch name: {method_uri}")
+        config["methodRepoMethod"]["methodUri"] = method_uri
+
         print(f"Updating methodVersion with branch name: {branch_name}")
         config["methodRepoMethod"]["methodVersion"] = branch_name
 

From c01f95aef041c85ca9c5bffde07a78a54746500b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 15:02:30 -0500
Subject: [PATCH 095/165] methoduri

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 0ea03e6e5d..fe1f037630 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -173,7 +173,7 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         # update the config with the new branch name
 
         # Construct the methodUri with the branch name
-        base_url = "github.com/broadinstitute/warp/TestIlluminaGenotypingArray"
+        base_url = "github.com/broadinstitute/warp/{pipeline_name}"
         method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
         print(f"Updating methodUri with branch name: {method_uri}")
         config["methodRepoMethod"]["methodUri"] = method_uri

From 9c1655a88cc9b0c15012bdbebff9ec3722dd8e06 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 15:04:31 -0500
Subject: [PATCH 096/165] methoduri

---
 scripts/firecloud_api/firecloud_api.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index fe1f037630..0b837a5d33 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -170,8 +170,6 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
             inputs_json = self.quote_values(inputs_json)
             config["inputs"] = inputs_json
 
-        # update the config with the new branch name
-
         # Construct the methodUri with the branch name
         base_url = "github.com/broadinstitute/warp/{pipeline_name}"
         method_uri = f"dockstore://{quote(base_url)}/{branch_name}"
@@ -181,12 +179,12 @@ def upload_test_inputs(self, pipeline_name, test_inputs, branch_name):
         print(f"Updating methodVersion with branch name: {branch_name}")
         config["methodRepoMethod"]["methodVersion"] = branch_name
 
-        # Increment the methodConfigVersion
+        # We need to increment the methodConfigVersion by 1 every time we update the method configuration
         config["methodConfigVersion"] += 1  # Increment version number by  1
         print(f"Updated method configuration: {json.dumps(config, indent=2)}")
 
 
-    # post the updated method config to the workspace
+       # post the updated method config to the workspace
         response = requests.post(url, headers=self.headers, json=config)
         print(f"Response status code: {response.status_code}")
         print(f"Response text: {response.text}")

From 38116146bfd1be91fbaff3f6795629d2ec9ad487 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 15:35:50 -0500
Subject: [PATCH 097/165] add comments to code

---
 .../test_illumina_genotyping_array.yml        | 31 ++++++++++++-------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8346626ed1..96ecb179e7 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -34,6 +34,7 @@ on:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: true
 
+
 env:
   PROJECT_NAME: WARP
   # Github repo name
@@ -67,16 +68,21 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      # ... further steps are automatically authenticated
-      - name: Check working directory
-        run: |
-          echo "Current directory:"
-          pwd
-          ls -lht
-
-      - name: Verify checked-out branch
+      # Set the branch name.
+      # github.head_ref contains the name of the branch in the context of a pull request
+      # if github.head_ref is empty, it implies the workflow was triggered manually
+      # ${GITHUB_REF##*/} extracts the branch name from GITHUB_REF.
+      # The ##*/ is a parameter expansion that removes the refs/heads/ prefix, leaving just the branch name.
+      - name: Set Branch Name
+        id: set_branch
         run: |
-          echo "Branch: ${{ github.head_ref }}"
+          if [ -z "${{ github.head_ref }}" ]; then
+            echo "Branch name is missing, using ${GITHUB_REF##*/}"
+            echo "branch_name=${GITHUB_REF##*/}" >> $GITHUB_ENV
+          else
+            echo "Branch name from PR: ${{ github.head_ref }}"
+            echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
+          fi
 
 
       - name: Submit job, poll status, and get outputs
@@ -158,9 +164,10 @@ jobs:
                                                                                 --results_path "$RESULTS_PATH" \
                                                                                 --inputs_json "$input_file" \
                                                                                 --update_truth "$UPDATE_TRUTH_BOOL")
-            echo "Uploading test input file: $test_input_file"
-          
-            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name "${{ github.head_ref }}"
+            echo "Uploading the test input file: $test_input_file"
+            echo "Branch name: $branch_name"
+            
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
 
             echo "Submitting job for input file: $input_file"
             cat "$SUBMISSION_DATA_FILE"

From 6ba8a907def2deea1d0426f288ba8b8146fe80e8 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 16:12:53 -0500
Subject: [PATCH 098/165] try to add delay on first push to dockstore

---
 .../test_illumina_genotyping_array.yml        | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 96ecb179e7..7431432b00 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -84,6 +84,25 @@ jobs:
             echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      - name: Check for Previous Commits in GitHub
+        id: check_commits
+        run: |
+          BRANCH_NAME="${{ env.branch_name }}"
+          REPO_OWNER="${{ github.repository_owner }}"
+          REPO_NAME="${{ github.event.repository.name }}"
+          
+          # Get commit history for the branch
+          COMMITS=$(curl -s -f "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/commits?sha=${BRANCH_NAME}")
+
+          # Check the return code of the curl command
+          if [ $? -eq 0 ]; then
+            echo "Branch has previous commits. Assuming this is not the first push to the branch."
+            echo "skip_pipeline=false" >> $GITHUB_ENV
+          else
+            echo "Branch is new or has no commits. Assuming first push. Delaying test by 5 minutes."
+            # Delay the test by 5 minutes (300 seconds)
+            sleep 300
+          fi
 
       - name: Submit job, poll status, and get outputs
         id: pipeline_run

From c84c2d53379c8a5896a4126127a82e6b9d954bad Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 20 Nov 2024 16:34:05 -0500
Subject: [PATCH 099/165] remove delays

---
 .../test_illumina_genotyping_array.yml        | 20 -------------------
 1 file changed, 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7431432b00..329c7404fe 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -84,26 +84,6 @@ jobs:
             echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
-      - name: Check for Previous Commits in GitHub
-        id: check_commits
-        run: |
-          BRANCH_NAME="${{ env.branch_name }}"
-          REPO_OWNER="${{ github.repository_owner }}"
-          REPO_NAME="${{ github.event.repository.name }}"
-          
-          # Get commit history for the branch
-          COMMITS=$(curl -s -f "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/commits?sha=${BRANCH_NAME}")
-
-          # Check the return code of the curl command
-          if [ $? -eq 0 ]; then
-            echo "Branch has previous commits. Assuming this is not the first push to the branch."
-            echo "skip_pipeline=false" >> $GITHUB_ENV
-          else
-            echo "Branch is new or has no commits. Assuming first push. Delaying test by 5 minutes."
-            # Delay the test by 5 minutes (300 seconds)
-            sleep 300
-          fi
-
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |

From 5cd0c2ae0d8a3324ecb3ae4e8072de3d2843c7c0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 10:39:45 -0500
Subject: [PATCH 100/165] test truth branch

---
 .github/workflows/test_illumina_genotyping_array.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 329c7404fe..d5290e4038 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -33,6 +33,10 @@ on:
       testType:
         description: 'Specify the type of test (Plumbing or Scientific)'
         required: true
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
 
 
 env:
@@ -94,8 +98,11 @@ jobs:
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
           TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
+          echo "truth branch: $TRUTH_BRANCH"
+          
           ########################################
           # SET PIPELINE SPECIFIC VARIABLES HERE #
           ########################################
@@ -104,7 +111,7 @@ jobs:
           # TODO: Need to set the truth and result paths appropriately
           # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
           # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/master"
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
           RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
 
           # Function to call the Firecloud API using the firecloud_api.py script

From 3a923be1c9dade3d7baee1b202428d5d75cbc58b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 11:54:52 -0500
Subject: [PATCH 101/165] test truth branch

---
 .../test_illumina_genotyping_array.yml        | 23 ++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d5290e4038..fca6e91740 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -88,6 +88,24 @@ jobs:
             echo "branch_name=${{ github.head_ref }}" >> $GITHUB_ENV
           fi
 
+      - name: Set Test Type for PRs
+        if: ${{ github.event_name == 'pull_request' }}
+        id: set_test_type
+        run: |
+          # Default to "Scientific" if targeting master
+          if [ "${{ github.base_ref }}" == "master" ]; then
+            echo "testType=Scientific" >> $GITHUB_ENV
+          else
+            echo "testType=Plumbing" >> $GITHUB_ENV
+          fi
+
+      - name: Use Provided Test Type
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        id: use_provided_test_type
+        run: |
+          # Use the testType provided by the user
+          echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -97,7 +115,8 @@ jobs:
           WORKSPACE="WARP Tests"
           USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
           UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
-          TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
+          #TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
+          TEST_TYPE="${{ env.testType }}"
           TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
           CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
           
@@ -164,6 +183,8 @@ jobs:
           # Loop through each file in the appropriate test inputs directory
           INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
           
+          echo "Running tests with test type: $TEST_TYPE"
+          
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \

From 91a5f8c2ca4eec3bfb5d6a991db11f1373875c19 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 11:58:27 -0500
Subject: [PATCH 102/165] def to develop for the moment

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index fca6e91740..5056cade47 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -93,7 +93,7 @@ jobs:
         id: set_test_type
         run: |
           # Default to "Scientific" if targeting master
-          if [ "${{ github.base_ref }}" == "master" ]; then
+          if [ "${{ github.base_ref }}" == "develop" ]; then
             echo "testType=Scientific" >> $GITHUB_ENV
           else
             echo "testType=Plumbing" >> $GITHUB_ENV

From 08fcd81c5ba59c98935daec4c3afb58b44962ce4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 11:59:12 -0500
Subject: [PATCH 103/165] def to back to master for the moment

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5056cade47..fca6e91740 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -93,7 +93,7 @@ jobs:
         id: set_test_type
         run: |
           # Default to "Scientific" if targeting master
-          if [ "${{ github.base_ref }}" == "develop" ]; then
+          if [ "${{ github.base_ref }}" == "master" ]; then
             echo "testType=Scientific" >> $GITHUB_ENV
           else
             echo "testType=Plumbing" >> $GITHUB_ENV

From a9748316b3dc8d92904a78cd68c31ef669f652fc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 13:23:08 -0500
Subject: [PATCH 104/165] look for 404 and attempt to delay

---
 .../test_illumina_genotyping_array.yml        | 37 ++++++++++++++-----
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index fca6e91740..12269c394b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -185,6 +185,9 @@ jobs:
           
           echo "Running tests with test type: $TEST_TYPE"
           
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
@@ -195,15 +198,31 @@ jobs:
             echo "Branch name: $branch_name"
             
             firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
-
-            echo "Submitting job for input file: $input_file"
-            cat "$SUBMISSION_DATA_FILE"
-            SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-
-            if [ -z "$SUBMISSION_ID" ]; then
-              echo "Submission failed for input file: $input_file"
-              continue
-            fi
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Attempt $attempt: Submitting job for input file: $input_file"
+              #echo "Submitting job for input file: $input_file"
+              cat "$SUBMISSION_DATA_FILE"
+              SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
+                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
+                sleep $RETRY_DELAY
+                ((attempt++))
+              elif [ -z "$SUBMISSION_ID" ]; then
+                echo "Submission failed for input file: $input_file. No submission ID received."
+                break
+              else
+                echo "Submission successful. Submission ID: $SUBMISSION_ID"
+                SUBMISSION_IDS+=("$SUBMISSION_ID")
+                break
+              fi
+          
+              if [ $attempt -gt $MAX_RETRIES ]; then
+                echo "Max retries reached. Exiting..."
+              fi
+            done
+          done
 
             echo "Submission ID: $SUBMISSION_ID"
             SUBMISSION_IDS+=("$SUBMISSION_ID")

From fa613b5fccf3f661e8b00a6b4efe0bdd9ab2b838 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 13:29:16 -0500
Subject: [PATCH 105/165] look for 404 and attempt to delay

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 12269c394b..a5bf154dd2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -226,7 +226,7 @@ jobs:
 
             echo "Submission ID: $SUBMISSION_ID"
             SUBMISSION_IDS+=("$SUBMISSION_ID")
-          done
+          
           
           echo "Monitoring the status of submitted workflows..."
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do

From f3c231cc8262bc06cc6f39cca3baca2c9dd358bf Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 26 Nov 2024 13:29:48 -0500
Subject: [PATCH 106/165] look for 404 and attempt to delay

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a5bf154dd2..8761d1a965 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -224,8 +224,8 @@ jobs:
             done
           done
 
-            echo "Submission ID: $SUBMISSION_ID"
-            SUBMISSION_IDS+=("$SUBMISSION_ID")
+            #echo "Submission ID: $SUBMISSION_ID"
+            #SUBMISSION_IDS+=("$SUBMISSION_ID")
           
           
           echo "Monitoring the status of submitted workflows..."

From e5fa16c16c9bc020c16b0ec0b5ff45b7ed715756 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 27 Nov 2024 10:14:24 -0500
Subject: [PATCH 107/165] more access token lifetime?

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 8761d1a965..7cd215b145 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' # seconds, default is 3600
+          access_token_lifetime: '7200' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.

From 059368a0b147e0f66eb03b66b8c606597eb602a5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:07:18 -0500
Subject: [PATCH 108/165] see if token is close to expiring

---
 .../test_illumina_genotyping_array.yml        | 23 ++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7cd215b145..d1fe9a4a86 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -227,12 +227,33 @@ jobs:
             #echo "Submission ID: $SUBMISSION_ID"
             #SUBMISSION_IDS+=("$SUBMISSION_ID")
           
+          # Function to refresh token
+          refresh_token() {
+            echo "Refreshing Google Cloud authentication token..."
+            # Re-authenticate and get a new token
+            TOKEN=$(gcloud auth application-default print-access-token)
+            echo "New token retrieved"
+          }          
           
+      
           echo "Monitoring the status of submitted workflows..."
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
+            
+            # Check if the token is expired or close to expiration and refresh it if necessary
+            CURRENT_TIME=$(date +%s)
+            TOKEN_EXPIRATION_TIME=$(gcloud auth application-default get-access-token --format='value(expiry)')
+            EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
+            TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
+
+            # If token is about to expire, refresh it
+            if (( EXPIRATION_TIME_IN_SECONDS - CURRENT_TIME <= TOKEN_LIFETIME_THRESHOLD )); then
+              refresh_token
+            fi
 
+            # Poll the status using the fresh token
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
+  
             if [ -z "$RESPONSE" ]; then
               echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
               continue

From 8bdf3846ccea074365518a1632fa31250f92dd86 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:08:04 -0500
Subject: [PATCH 109/165] see if token is close to expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d1fe9a4a86..4dc7614c74 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '7200' # seconds, default is 3600
+          access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.

From a7388dbcd495dcc1885b438addf8432a88a749a7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:16:29 -0500
Subject: [PATCH 110/165] try differnt gcloud

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 4dc7614c74..570e024c62 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -232,7 +232,7 @@ jobs:
             echo "Refreshing Google Cloud authentication token..."
             # Re-authenticate and get a new token
             TOKEN=$(gcloud auth application-default print-access-token)
-            echo "New token retrieved"
+            echo "New token retrieved: $TOKEN"
           }          
           
       

From de221bd57dc6b1d665030084fd0c3dd8acff24ec Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:19:17 -0500
Subject: [PATCH 111/165] try differnt gcloud

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 570e024c62..a959d636e8 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -242,7 +242,7 @@ jobs:
             
             # Check if the token is expired or close to expiration and refresh it if necessary
             CURRENT_TIME=$(date +%s)
-            TOKEN_EXPIRATION_TIME=$(gcloud auth application-default get-access-token --format='value(expiry)')
+            TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
             EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
             TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
 

From 03d5f61614b4481aed4d5dd12461844d3ac0fb10 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:23:03 -0500
Subject: [PATCH 112/165] timestamp

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a959d636e8..97181dffcd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -241,7 +241,7 @@ jobs:
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             
             # Check if the token is expired or close to expiration and refresh it if necessary
-            CURRENT_TIME=$(date +%s)
+            CURRENT_TIME=$(date +"%Y-%m-%d %H:%M:%S")
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
             EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
             TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration

From ec1364e7c32af4fda0e5f7859af5d1f43882b93f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:27:17 -0500
Subject: [PATCH 113/165] echo the time

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 97181dffcd..0317890f8e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -242,6 +242,7 @@ jobs:
             
             # Check if the token is expired or close to expiration and refresh it if necessary
             CURRENT_TIME=$(date +"%Y-%m-%d %H:%M:%S")
+            echo "Current time: $CURRENT_TIME"
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
             EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
             TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration

From f5d9d93817bbe4bf96057e691ebab6f4d1e198aa Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:30:55 -0500
Subject: [PATCH 114/165] echo the time

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0317890f8e..aae42776b9 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -241,7 +241,7 @@ jobs:
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             
             # Check if the token is expired or close to expiration and refresh it if necessary
-            CURRENT_TIME=$(date +"%Y-%m-%d %H:%M:%S")
+            CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
             echo "Current time: $CURRENT_TIME"
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
             EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)

From b9a1cb6449299709a1627a19779df8dad61bdd17 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:39:02 -0500
Subject: [PATCH 115/165] echo the time

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index aae42776b9..54042e916e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -241,15 +241,17 @@ jobs:
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             
             # Check if the token is expired or close to expiration and refresh it if necessary
-            CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-            echo "Current time: $CURRENT_TIME"
+            CURRENT_TIME_EPOCH=$(date +%s)
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
-            EXPIRATION_TIME_IN_SECONDS=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
+            EXPIRATION_TIME_EPOCH=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
             TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
-
-            # If token is about to expire, refresh it
-            if (( EXPIRATION_TIME_IN_SECONDS - CURRENT_TIME <= TOKEN_LIFETIME_THRESHOLD )); then
+            
+            # Check and refresh token if necessary
+            if (( EXPIRATION_TIME_EPOCH - CURRENT_TIME_EPOCH <= TOKEN_LIFETIME_THRESHOLD )); then
+              echo "Token is nearing expiration or expired. Refreshing token..."
               refresh_token
+            else
+              echo "Token is valid. No refresh needed."
             fi
 
             # Poll the status using the fresh token

From 0393d9f71bda43434458ae78779751158aac04bd Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:40:29 -0500
Subject: [PATCH 116/165] echo the time

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 54042e916e..b0af51ebf7 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -243,6 +243,7 @@ jobs:
             # Check if the token is expired or close to expiration and refresh it if necessary
             CURRENT_TIME_EPOCH=$(date +%s)
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
+            echo "Token expiration time: $TOKEN_EXPIRATION_TIME"
             EXPIRATION_TIME_EPOCH=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
             TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
             

From 6c38f122b956cbaa34a2c56cec2bd565a5061df3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:43:51 -0500
Subject: [PATCH 117/165] echo the time

---
 .github/workflows/test_illumina_genotyping_array.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b0af51ebf7..43d23805cb 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -243,10 +243,16 @@ jobs:
             # Check if the token is expired or close to expiration and refresh it if necessary
             CURRENT_TIME_EPOCH=$(date +%s)
             TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
-            echo "Token expiration time: $TOKEN_EXPIRATION_TIME"
-            EXPIRATION_TIME_EPOCH=$(date -d "$TOKEN_EXPIRATION_TIME" +%s)
-            TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
+            echo "Raw token expiration time: $TOKEN_EXPIRATION_TIME"
+            
+            # Extract the valid datetime portion (first part before the semicolon)
+            TOKEN_EXPIRATION_DATETIME=$(echo "$TOKEN_EXPIRATION_TIME" | awk -F';' '{print $1}' | awk -F'=' '{print $2}')
+            echo "Parsed token expiration datetime: $TOKEN_EXPIRATION_DATETIME"
             
+            # Convert the parsed datetime to epoch time
+            EXPIRATION_TIME_EPOCH=$(date -d "$TOKEN_EXPIRATION_DATETIME" +%s)
+            TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
+              
             # Check and refresh token if necessary
             if (( EXPIRATION_TIME_EPOCH - CURRENT_TIME_EPOCH <= TOKEN_LIFETIME_THRESHOLD )); then
               echo "Token is nearing expiration or expired. Refreshing token..."

From bc71bf1ca7c90517bfae6c5ea76486879a926c4e Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:47:40 -0500
Subject: [PATCH 118/165] make token expire after 2 mins

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 43d23805cb..2851c40462 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' # seconds, default is 3600
+          access_token_lifetime: '120' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.

From 65dcec70535dd6958869cf163b334809cbb6b3a5 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 14:50:41 -0500
Subject: [PATCH 119/165] make token expire after 2 mins

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2851c40462..43d23805cb 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '120' # seconds, default is 3600
+          access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.

From a9a35089f71a3e3c2c2ce05458aede53c04d2fd4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:09:50 -0500
Subject: [PATCH 120/165] break things down

---
 .../test_illumina_genotyping_array.yml        | 227 ++++--------------
 1 file changed, 41 insertions(+), 186 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 43d23805cb..525e373941 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -106,200 +106,55 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      - name: Submit job, poll status, and get outputs
-        id: pipeline_run
+      - name: Create Submission Data File
         run: |
-          # Set common environment variables
-          TOKEN="${{ steps.auth.outputs.access_token }}"
-          NAMESPACE="warp-pipelines"
-          WORKSPACE="WARP Tests"
-          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
-          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
-          #TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
-          TEST_TYPE="${{ env.testType }}"
-          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
-          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
-          
-          echo "truth branch: $TRUTH_BRANCH"
-          
-          ########################################
-          # SET PIPELINE SPECIFIC VARIABLES HERE #
-          ########################################
-          PIPELINE_NAME="TestIlluminaGenotypingArray"
-          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
-          # TODO: Need to set the truth and result paths appropriately
-          # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
-          # We may want to keep the truth and resuts buckets separate for TTL reasons
-          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
-          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
+          # Creating the submission data file for job submission
+          echo "Creating submission data file..."
+          # All necessary data preparation steps here
 
-          # Function to call the Firecloud API using the firecloud_api.py script
-          firecloud_action() {
-            python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
-          }
-
-          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
-            if [ "$USE_CALL_CACHE" == "true" ]; then
-                USE_CALL_CACHE_BOOL=true
-            else
-                USE_CALL_CACHE_BOOL=false
-            fi
+      - name: Submit Job
+        id: submit_job
+        run: |
+          echo "Submitting job..."
+          # Submit the job here and store the submission ID
+          SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action submit_job "$@")
+          echo "Submission ID: $SUBMISSION_ID"
+          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_ENV
+
+      - name: Poll Status
+        id: poll_status
+        run: |
+          echo "Polling status for submission ID: ${{ env.submission_id }}"
+          RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action poll_status --submission_id "${{ env.submission_id }}")
 
-          
-          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
-          if [ "$UPDATE_TRUTH" = "true" ]; then
-              UPDATE_TRUTH_BOOL=true
-          else
-              UPDATE_TRUTH_BOOL=false
+          # Check if polling returned any data or an error
+          if [ -z "$RESPONSE" ]; then
+            echo "Failed to retrieve Workflow IDs for submission: ${{ env.submission_id }}"
+            exit 1
           fi
-          
-          # Create the submission_data.json file which will be the same for all inputs
-          SUBMISSION_DATA_FILE="submission_data.json"
-        
-          # Use a heredoc to generate the JSON file content dynamically
-          cat <<EOF > "$SUBMISSION_DATA_FILE"
-          {
-            "methodConfigurationNamespace": "warp-pipelines",
-            "methodConfigurationName": "$PIPELINE_NAME",
-            "useCallCache": $USE_CALL_CACHE_BOOL,
-            "deleteIntermediateOutputFiles": false,
-            "useReferenceDisks": true,
-            "memoryRetryMultiplier": 1.2,
-            "workflowFailureMode": "NoNewCalls",
-            "userComment": "Automated submission",
-            "ignoreEmptyOutputs": false
-          }
-          EOF
-          echo "Created submission data file: $SUBMISSION_DATA_FILE"
 
-          # Initialize variables to aggregate statuses and outputs
-          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
-          ALL_OUTPUTS=""
-          
-          # Initialize arrays to track submission and workflow statuses
-          declare -a SUBMISSION_IDS
-          declare -A WORKFLOW_STATUSES
+          # Store workflow statuses
+          echo "$RESPONSE" > workflow_statuses.json
 
-          # Loop through each file in the appropriate test inputs directory
-          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
-          
-          echo "Running tests with test type: $TEST_TYPE"
-          
-          MAX_RETRIES=2
-          RETRY_DELAY=300  # 300 seconds = 5 minutes
-          
-          for input_file in "$INPUTS_DIR"/*.json; do
-            echo "Processing input file: $input_file"
-            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
-                                                                                --results_path "$RESULTS_PATH" \
-                                                                                --inputs_json "$input_file" \
-                                                                                --update_truth "$UPDATE_TRUTH_BOOL")
-            echo "Uploading the test input file: $test_input_file"
-            echo "Branch name: $branch_name"
-            
-            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
-            attempt=1
-            while [ $attempt -le $MAX_RETRIES ]; do
-              echo "Attempt $attempt: Submitting job for input file: $input_file"
-              #echo "Submitting job for input file: $input_file"
-              cat "$SUBMISSION_DATA_FILE"
-              SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-          
-              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
-                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
-                sleep $RETRY_DELAY
-                ((attempt++))
-              elif [ -z "$SUBMISSION_ID" ]; then
-                echo "Submission failed for input file: $input_file. No submission ID received."
-                break
-              else
-                echo "Submission successful. Submission ID: $SUBMISSION_ID"
-                SUBMISSION_IDS+=("$SUBMISSION_ID")
-                break
-              fi
-          
-              if [ $attempt -gt $MAX_RETRIES ]; then
-                echo "Max retries reached. Exiting..."
-              fi
-            done
-          done
+      - name: Get Outputs
+        id: get_outputs
+        run: |
+          echo "Retrieving outputs for submission ID: ${{ env.submission_id }}"
+          WORKFLOW_IDS=$(jq -r 'keys[]' workflow_statuses.json)
 
-            #echo "Submission ID: $SUBMISSION_ID"
-            #SUBMISSION_IDS+=("$SUBMISSION_ID")
-          
-          # Function to refresh token
-          refresh_token() {
-            echo "Refreshing Google Cloud authentication token..."
-            # Re-authenticate and get a new token
-            TOKEN=$(gcloud auth application-default print-access-token)
-            echo "New token retrieved: $TOKEN"
-          }          
-          
-      
-          echo "Monitoring the status of submitted workflows..."
-          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            
-            # Check if the token is expired or close to expiration and refresh it if necessary
-            CURRENT_TIME_EPOCH=$(date +%s)
-            TOKEN_EXPIRATION_TIME=$(gcloud auth application-default print-access-token --format='value(expiry)')
-            echo "Raw token expiration time: $TOKEN_EXPIRATION_TIME"
-            
-            # Extract the valid datetime portion (first part before the semicolon)
-            TOKEN_EXPIRATION_DATETIME=$(echo "$TOKEN_EXPIRATION_TIME" | awk -F';' '{print $1}' | awk -F'=' '{print $2}')
-            echo "Parsed token expiration datetime: $TOKEN_EXPIRATION_DATETIME"
-            
-            # Convert the parsed datetime to epoch time
-            EXPIRATION_TIME_EPOCH=$(date -d "$TOKEN_EXPIRATION_DATETIME" +%s)
-            TOKEN_LIFETIME_THRESHOLD=300  # Set the threshold to 5 minutes before expiration
-              
-            # Check and refresh token if necessary
-            if (( EXPIRATION_TIME_EPOCH - CURRENT_TIME_EPOCH <= TOKEN_LIFETIME_THRESHOLD )); then
-              echo "Token is nearing expiration or expired. Refreshing token..."
-              refresh_token
-            else
-              echo "Token is valid. No refresh needed."
-            fi
+          for WORKFLOW_ID in $WORKFLOW_IDS; do
+            OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action get_outputs --submission_id "${{ env.submission_id }}" --workflow_id "$WORKFLOW_ID")
+            echo "Workflow Output for $WORKFLOW_ID: $OUTPUT"
+            echo "$OUTPUT" >> final_outputs.json
+          done
 
-            # Poll the status using the fresh token
-            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-  
-            if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-              continue
-            fi
-          
-            # Parse and store workflow statuses
-            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            echo "Statuses for submission $SUBMISSION_ID:"
-            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-          
-            # Append to aggregate statuses
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
-          
-            # retrieve workflow outputs
-            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-              done
-            done
-          
-            # Generate final summary tables with hyperlinks for Submission IDs
-            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
-            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
-              # Generate the Terra URL for the submission
-              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
-            
-              # Add the Submission ID as a hyperlink
-              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
-              
-              # Add the workflows and statuses for this submission
-              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
-              
-              # Add a blank line for separation
-              echo "" >> $GITHUB_STEP_SUMMARY
-            done
+      - name: Summarize and Print Results
+        id: summarize_results
+        run: |
+          echo "Summarizing the final results..."
+          # Process and print the results (outputs, statuses, etc.)
+          cat final_outputs.json
+          echo "Pipeline run complete!"
 
       - name: Print Summary on Success
         if: success()

From 60a68ea3577260719874155143aaadf78c73caff Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:11:16 -0500
Subject: [PATCH 121/165] break things down

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 525e373941..212b6d395e 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -117,7 +117,7 @@ jobs:
         run: |
           echo "Submitting job..."
           # Submit the job here and store the submission ID
-          SUBMISSION_ID=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action submit_job "$@")
+          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
           echo "Submission ID: $SUBMISSION_ID"
           echo "submission_id=$SUBMISSION_ID" >> $GITHUB_ENV
 

From fb91ad28f513ed2506e4720b05ce0c67dabd0474 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:12:28 -0500
Subject: [PATCH 122/165] break things down

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 212b6d395e..d8b48e13c3 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -117,6 +117,12 @@ jobs:
         run: |
           echo "Submitting job..."
           # Submit the job here and store the submission ID
+          
+          # Function to call the Firecloud API using the firecloud_api.py script
+          firecloud_action() {
+            python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
+          }
+          
           SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
           echo "Submission ID: $SUBMISSION_ID"
           echo "submission_id=$SUBMISSION_ID" >> $GITHUB_ENV

From 96137dca54cd29afd798a6ec01cad4eb4661e168 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:34:28 -0500
Subject: [PATCH 123/165] Poll for Status and Refresh Token if Expiring

---
 .../test_illumina_genotyping_array.yml        | 240 +++++++++++++++---
 1 file changed, 201 insertions(+), 39 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d8b48e13c3..dbe50f83d6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,3 +1,4 @@
+
 name: Test Illumina Genotyping Array
 
 # Controls when the workflow will run
@@ -72,6 +73,14 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
+      - name: Get Token Expiry Time
+        id: token_expiry
+        run: |
+          # Get the expiration time of the token (it expires in 3600 seconds = 1 hour by default)
+          EXPIRATION_TIME=$(date -d "$(date +%Y-%m-%dT%H:%M:%S) + 1 hour" +%s)
+          echo "Expiration Time: $EXPIRATION_TIME"
+          echo "expiration_time=$EXPIRATION_TIME" >> $GITHUB_ENV
+
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
       # if github.head_ref is empty, it implies the workflow was triggered manually
@@ -106,61 +115,214 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      - name: Create Submission Data File
+      - name: Submit job, poll status, and get outputs
+        id: pipeline_run
         run: |
-          # Creating the submission data file for job submission
-          echo "Creating submission data file..."
-          # All necessary data preparation steps here
-
-      - name: Submit Job
-        id: submit_job
-        run: |
-          echo "Submitting job..."
-          # Submit the job here and store the submission ID
+          # Set common environment variables
+          TOKEN="${{ steps.auth.outputs.access_token }}"
+          NAMESPACE="warp-pipelines"
+          WORKSPACE="WARP Tests"
+          USE_CALL_CACHE="${{ github.event.inputs.useCallCache || 'true' }}"
+          UPDATE_TRUTH="${{ github.event.inputs.updateTruth || 'false' }}"
+          #TEST_TYPE="${{ github.event.inputs.testType || 'Plumbing' }}"
+          TEST_TYPE="${{ env.testType }}"
+          TRUTH_BRANCH="${{ github.event.inputs.truthBranch || 'master' }}"
+          CURRENT_TIME=$(date +"%Y-%m-%d-%H-%M-%S")
+          
+          echo "truth branch: $TRUTH_BRANCH"
           
+          ########################################
+          # SET PIPELINE SPECIFIC VARIABLES HERE #
+          ########################################
+          PIPELINE_NAME="TestIlluminaGenotypingArray"
+          PIPELINE_DIR="pipelines/broad/genotyping/illumina"
+          # TODO: Need to set the truth and result paths appropriately
+          # TODO: Need to dynamically set the truth branch, for now it is hardcoded to master branch
+          # We may want to keep the truth and resuts buckets separate for TTL reasons
+          TRUTH_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/truth/$(echo "$TEST_TYPE" | tr '[:upper:]' '[:lower:]')/$TRUTH_BRANCH"
+          RESULTS_PATH="gs://broad-gotc-test-storage/IlluminaGenotypingArray/results/$CURRENT_TIME"
+
           # Function to call the Firecloud API using the firecloud_api.py script
           firecloud_action() {
             python3 scripts/firecloud_api/firecloud_api.py --token "$TOKEN" --namespace "$NAMESPACE" --workspace "$WORKSPACE" --action "$1" "${@:2}"
           }
+
+          # Convert USE_CALL_CACHE to a boolean-friendly format ("true" -> true, "false" -> false)
+            if [ "$USE_CALL_CACHE" == "true" ]; then
+                USE_CALL_CACHE_BOOL=true
+            else
+                USE_CALL_CACHE_BOOL=false
+            fi
+          
           
-          SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
-          echo "Submission ID: $SUBMISSION_ID"
-          echo "submission_id=$SUBMISSION_ID" >> $GITHUB_ENV
+          # Convert UPDATE_TRUTH to a boolean-friendly format ("true" -> true, "false" -> false)
+          if [ "$UPDATE_TRUTH" = "true" ]; then
+              UPDATE_TRUTH_BOOL=true
+          else
+              UPDATE_TRUTH_BOOL=false
+          fi
+          
+          # Create the submission_data.json file which will be the same for all inputs
+          SUBMISSION_DATA_FILE="submission_data.json"
+          
+          # Use a heredoc to generate the JSON file content dynamically
+          cat <<EOF > "$SUBMISSION_DATA_FILE"
+          {
+            "methodConfigurationNamespace": "warp-pipelines",
+            "methodConfigurationName": "$PIPELINE_NAME",
+            "useCallCache": $USE_CALL_CACHE_BOOL,
+            "deleteIntermediateOutputFiles": false,
+            "useReferenceDisks": true,
+            "memoryRetryMultiplier": 1.2,
+            "workflowFailureMode": "NoNewCalls",
+            "userComment": "Automated submission",
+            "ignoreEmptyOutputs": false
+          }
+          EOF
+          echo "Created submission data file: $SUBMISSION_DATA_FILE"
+
+          # Initialize variables to aggregate statuses and outputs
+          ALL_WORKFLOW_STATUSES="Workflow ID | Status"$'\n'"--- | ---"
+          ALL_OUTPUTS=""
+          
+          # Initialize arrays to track submission and workflow statuses
+          declare -a SUBMISSION_IDS
+          declare -A WORKFLOW_STATUSES
 
-      - name: Poll Status
+          # Loop through each file in the appropriate test inputs directory
+          INPUTS_DIR="$PIPELINE_DIR/test_inputs/$TEST_TYPE"
+          
+          echo "Running tests with test type: $TEST_TYPE"
+          
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          
+          for input_file in "$INPUTS_DIR"/*.json; do
+            echo "Processing input file: $input_file"
+            test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \
+                                                                                --results_path "$RESULTS_PATH" \
+                                                                                --inputs_json "$input_file" \
+                                                                                --update_truth "$UPDATE_TRUTH_BOOL")
+            echo "Uploading the test input file: $test_input_file"
+            echo "Branch name: $branch_name"
+          
+            firecloud_action upload_test_inputs --pipeline_name $PIPELINE_NAME --test_input_file "$test_input_file" --branch_name $branch_name
+            attempt=1
+            while [ $attempt -le $MAX_RETRIES ]; do
+              echo "Attempt $attempt: Submitting job for input file: $input_file"
+              #echo "Submitting job for input file: $input_file"
+              cat "$SUBMISSION_DATA_FILE"
+              SUBMISSION_ID=$(firecloud_action submit --submission_data_file "$SUBMISSION_DATA_FILE")
+          
+              if [[ "$SUBMISSION_ID" == *"404"* ]]; then
+                echo "Error: Dockstore method not found. Retrying in $RETRY_DELAY seconds..."
+                sleep $RETRY_DELAY
+                ((attempt++))
+              elif [ -z "$SUBMISSION_ID" ]; then
+                echo "Submission failed for input file: $input_file. No submission ID received."
+                break
+              else
+                echo "Submission successful. Submission ID: $SUBMISSION_ID"
+                SUBMISSION_IDS+=("$SUBMISSION_ID")
+                break
+              fi
+          
+              if [ $attempt -gt $MAX_RETRIES ]; then
+                echo "Max retries reached. Exiting..."
+              fi
+            done
+          done
+
+            #echo "Submission ID: $SUBMISSION_ID"
+            #SUBMISSION_IDS+=("$SUBMISSION_ID") 
+          
+          
+          echo "Monitoring the status of submitted workflows..."
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
+
+            if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+              continue
+            fi
+          
+            # Parse and store workflow statuses
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "Statuses for submission $SUBMISSION_ID:"
+            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
+          
+            # Append to aggregate statuses
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+          
+            # retrieve workflow outputs
+            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+              done
+            done
+          
+            # Generate final summary tables with hyperlinks for Submission IDs
+            echo "## Combined Workflow Statuses" >> $GITHUB_STEP_SUMMARY
+            for SUBMISSION_ID in "${!WORKFLOW_STATUSES[@]}"; do
+              # Generate the Terra URL for the submission
+              SUBMISSION_URL="https://app.terra.bio/#workspaces/$NAMESPACE/${WORKSPACE// /%20}/job_history/$SUBMISSION_ID"
+          
+              # Add the Submission ID as a hyperlink
+              echo "[Submission ID: $SUBMISSION_ID]($SUBMISSION_URL)" >> $GITHUB_STEP_SUMMARY
+          
+              # Add the workflows and statuses for this submission
+              echo "${WORKFLOW_STATUSES[$SUBMISSION_ID]}" >> $GITHUB_STEP_SUMMARY
+          
+              # Add a blank line for separation
+              echo "" >> $GITHUB_STEP_SUMMARY
+            done
+
+      - name: Poll for Status and Refresh Token if Expiring
         id: poll_status
         run: |
-          echo "Polling status for submission ID: ${{ env.submission_id }}"
-          RESPONSE=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action poll_status --submission_id "${{ env.submission_id }}")
+          MAX_RETRIES=2
+          RETRY_DELAY=300  # 300 seconds = 5 minutes
+          CURRENT_TIME=$(date +%s) # current timestamp in seconds
+          TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
 
-          # Check if polling returned any data or an error
-          if [ -z "$RESPONSE" ]; then
-            echo "Failed to retrieve Workflow IDs for submission: ${{ env.submission_id }}"
-            exit 1
+          # If token expires in less than 5 minutes, refresh the token
+          if [ $TIME_LEFT -le 300 ]; then
+            echo "Token is about to expire in $TIME_LEFT seconds, refreshing..."
+            # Refresh the token
+            TOKEN=$(gcloud auth application-default print-access-token)
+            echo "Token refreshed"
+          else
+            echo "Token valid, no need to refresh. Time left: $TIME_LEFT seconds"
           fi
 
-          # Store workflow statuses
-          echo "$RESPONSE" > workflow_statuses.json
+          # Proceed with polling workflow status
+          echo "Polling for workflow status..."
+          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
 
-      - name: Get Outputs
-        id: get_outputs
-        run: |
-          echo "Retrieving outputs for submission ID: ${{ env.submission_id }}"
-          WORKFLOW_IDS=$(jq -r 'keys[]' workflow_statuses.json)
+            if [ -z "$RESPONSE" ]; then
+              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
+              continue
+            fi
 
-          for WORKFLOW_ID in $WORKFLOW_IDS; do
-            OUTPUT=$(python3 scripts/firecloud_api/firecloud_api.py --token "${{ steps.auth.outputs.access_token }}" --action get_outputs --submission_id "${{ env.submission_id }}" --workflow_id "$WORKFLOW_ID")
-            echo "Workflow Output for $WORKFLOW_ID: $OUTPUT"
-            echo "$OUTPUT" >> final_outputs.json
-          done
+            # Parse and store workflow statuses
+            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
+            echo "Statuses for submission $SUBMISSION_ID:"
+            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
 
-      - name: Summarize and Print Results
-        id: summarize_results
-        run: |
-          echo "Summarizing the final results..."
-          # Process and print the results (outputs, statuses, etc.)
-          cat final_outputs.json
-          echo "Pipeline run complete!"
+            # Append to aggregate statuses
+            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
+
+            # retrieve workflow outputs
+            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
+            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
+              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
+              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
+            done
+          done
 
       - name: Print Summary on Success
         if: success()

From 6f8b7e80dfb8b5d8b4b709cd67bab41d1db879cb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:38:06 -0500
Subject: [PATCH 124/165] Poll for Status and Refresh Token if Expiring

---
 .../workflows/test_illumina_genotyping_array.yml   | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index dbe50f83d6..9ca86031ac 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -239,6 +239,20 @@ jobs:
           
           echo "Monitoring the status of submitted workflows..."
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
+            # Before polling, check if the token is about to expire
+            CURRENT_TIME=$(date +%s)
+            TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
+          
+            # If token expires in less than 5 minutes, refresh it
+            if [ $TIME_LEFT -le 300 ]; then
+              echo "Token is about to expire in $TIME_LEFT seconds, refreshing..."
+              # Refresh the token
+              TOKEN=$(gcloud auth application-default print-access-token)
+              echo "Token refreshed"
+            else
+              echo "Token valid, no need to refresh. Time left: $TIME_LEFT seconds"
+            fi
+
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 

From 9b8fbf035f843ab582f28a5e8a280416dbd43e5b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:39:59 -0500
Subject: [PATCH 125/165] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 9ca86031ac..e6345ec8bd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -300,6 +300,9 @@ jobs:
           RETRY_DELAY=300  # 300 seconds = 5 minutes
           CURRENT_TIME=$(date +%s) # current timestamp in seconds
           TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
+          echo "Time left for token expiry: $TIME_LEFT seconds"
+          echo "The current time is: $CURRENT_TIME"
+          echo "The expiration time is: $EXPIRATION_TIME"
 
           # If token expires in less than 5 minutes, refresh the token
           if [ $TIME_LEFT -le 300 ]; then

From 51c94d96cb3da051264adf599375a290c1d2d758 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 15:45:20 -0500
Subject: [PATCH 126/165] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e6345ec8bd..7fc1326ab5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -73,12 +73,14 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      - name: Get Token Expiry Time
-        id: token_expiry
+      - name: Set Token Expiry Time
+        id: set_token_expiry
         run: |
-          # Get the expiration time of the token (it expires in 3600 seconds = 1 hour by default)
-          EXPIRATION_TIME=$(date -d "$(date +%Y-%m-%dT%H:%M:%S) + 1 hour" +%s)
-          echo "Expiration Time: $EXPIRATION_TIME"
+          # Get the current time and set expiration time to 1 hour from now (3600 seconds)
+          CURRENT_TIME=$(date +%s)
+          EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from current time
+          echo "Current time: $CURRENT_TIME"
+          echo "Expiration time: $EXPIRATION_TIME"
           echo "expiration_time=$EXPIRATION_TIME" >> $GITHUB_ENV
 
       # Set the branch name.

From 7ffba179cc0e2fb92a91e8641e8b1e0cfbe75620 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 16:00:03 -0500
Subject: [PATCH 127/165] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7fc1326ab5..26fb9dd09d 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -251,6 +251,10 @@ jobs:
               # Refresh the token
               TOKEN=$(gcloud auth application-default print-access-token)
               echo "Token refreshed"
+              # Update expiration time after refreshing
+              CURRENT_TIME=$(date +%s)
+              EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from now
+              echo "New expiration time: $EXPIRATION_TIME"
             else
               echo "Token valid, no need to refresh. Time left: $TIME_LEFT seconds"
             fi

From c25fe37072b5ef00b4c52cc5e125991cb3926f89 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 16:01:18 -0500
Subject: [PATCH 128/165] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 26fb9dd09d..aff0ea6724 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -243,7 +243,9 @@ jobs:
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
             # Before polling, check if the token is about to expire
             CURRENT_TIME=$(date +%s)
+            echo "Current time: $CURRENT_TIME"
             TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
+            echo "Time left for token expiry: $TIME_LEFT seconds"
           
             # If token expires in less than 5 minutes, refresh it
             if [ $TIME_LEFT -le 300 ]; then

From 7f783e1aef9dfcd3c65bd2283868c77905d6a8ec Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 16:02:25 -0500
Subject: [PATCH 129/165] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index aff0ea6724..43b2242bbd 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -245,7 +245,7 @@ jobs:
             CURRENT_TIME=$(date +%s)
             echo "Current time: $CURRENT_TIME"
             TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
-            echo "Time left for token expiry: $TIME_LEFT seconds"
+            echo "Time left for token expiry: $TIME_LEFT  seconds"
           
             # If token expires in less than 5 minutes, refresh it
             if [ $TIME_LEFT -le 300 ]; then

From 7e1c2a50fa43b4097e0469e739878ebfe1fe4539 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Mon, 2 Dec 2024 16:05:17 -0500
Subject: [PATCH 130/165] Poll for Status and Refresh Token if Expiring

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 43b2242bbd..5f343ca4f5 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -244,6 +244,7 @@ jobs:
             # Before polling, check if the token is about to expire
             CURRENT_TIME=$(date +%s)
             echo "Current time: $CURRENT_TIME"
+            echo "Expiration time: $EXPIRATION_TIME"
             TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
             echo "Time left for token expiry: $TIME_LEFT  seconds"
           

From d02d408e14a39f8be9cff0ecfafe385b52a011bc Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 10:31:02 -0500
Subject: [PATCH 131/165] remove a step

---
 .../test_illumina_genotyping_array.yml        | 48 -------------------
 1 file changed, 48 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5f343ca4f5..43c218d3ce 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -302,54 +302,6 @@ jobs:
               echo "" >> $GITHUB_STEP_SUMMARY
             done
 
-      - name: Poll for Status and Refresh Token if Expiring
-        id: poll_status
-        run: |
-          MAX_RETRIES=2
-          RETRY_DELAY=300  # 300 seconds = 5 minutes
-          CURRENT_TIME=$(date +%s) # current timestamp in seconds
-          TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
-          echo "Time left for token expiry: $TIME_LEFT seconds"
-          echo "The current time is: $CURRENT_TIME"
-          echo "The expiration time is: $EXPIRATION_TIME"
-
-          # If token expires in less than 5 minutes, refresh the token
-          if [ $TIME_LEFT -le 300 ]; then
-            echo "Token is about to expire in $TIME_LEFT seconds, refreshing..."
-            # Refresh the token
-            TOKEN=$(gcloud auth application-default print-access-token)
-            echo "Token refreshed"
-          else
-            echo "Token valid, no need to refresh. Time left: $TIME_LEFT seconds"
-          fi
-
-          # Proceed with polling workflow status
-          echo "Polling for workflow status..."
-          for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-            echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID")
-
-            if [ -z "$RESPONSE" ]; then
-              echo "Failed to retrieve Workflow IDs for submission: $SUBMISSION_ID"
-              continue
-            fi
-
-            # Parse and store workflow statuses
-            WORKFLOW_STATUSES_FOR_SUBMISSION=$(echo "$RESPONSE" | jq -r 'to_entries | map(.key + " | " + .value) | .[]')
-            echo "Statuses for submission $SUBMISSION_ID:"
-            echo "$WORKFLOW_STATUSES_FOR_SUBMISSION"
-
-            # Append to aggregate statuses
-            WORKFLOW_STATUSES["$SUBMISSION_ID"]=$WORKFLOW_STATUSES_FOR_SUBMISSION
-
-            # retrieve workflow outputs
-            echo "Retrieving workflow outputs for Submission ID: $SUBMISSION_ID..."
-            for WORKFLOW_ID in $(echo "$RESPONSE" | jq -r 'keys[]'); do
-              WORKFLOW_OUTPUT=$(firecloud_action get_outputs --submission_id "$SUBMISSION_ID" --workflow_id "$WORKFLOW_ID" --pipeline_name "$PIPELINE_NAME")
-              ALL_OUTPUTS+="$WORKFLOW_OUTPUT"$'\n'
-            done
-          done
-
       - name: Print Summary on Success
         if: success()
         run: |

From 5ef130b1142db69f6e0599e120c5e1a3cec438b3 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 10:39:06 -0500
Subject: [PATCH 132/165] use $expiration_time

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 43c218d3ce..90925758d6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -244,8 +244,8 @@ jobs:
             # Before polling, check if the token is about to expire
             CURRENT_TIME=$(date +%s)
             echo "Current time: $CURRENT_TIME"
-            echo "Expiration time: $EXPIRATION_TIME"
-            TIME_LEFT=$(( $EXPIRATION_TIME - $CURRENT_TIME ))
+            echo "Expiration time: $expiration_time"
+            TIME_LEFT=$(( $expiration_time - $CURRENT_TIME ))
             echo "Time left for token expiry: $TIME_LEFT  seconds"
           
             # If token expires in less than 5 minutes, refresh it

From bad4ce26b9df36505e25dd68bef30fceda7a732a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 11:27:52 -0500
Subject: [PATCH 133/165] move expiration to pythong sciprt

---
 scripts/firecloud_api/firecloud_api.py | 35 +++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 0b837a5d33..c7defd86fa 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -18,6 +18,7 @@
 import time
 import json
 import sys
+import subprocess
 from urllib.parse import quote
 
 
@@ -63,6 +64,22 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             print(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
+    def refresh_token(self):
+        """
+        Refreshes the API token using gcloud's application default credentials.
+        :return: The new token as a string
+        """
+        try:
+            # Execute the gcloud command to get the new access token
+            result = subprocess.run(
+                ["gcloud", "auth", "application-default", "print-access-token"],
+                capture_output=True, text=True, check=True
+            )
+            return result.stdout.strip()  # Return the new token
+        except subprocess.CalledProcessError as e:
+            print(f"Error refreshing token: {e.stderr}", file=sys.stderr)
+            return None
+
     def create_submission(self, submission_data):
         """
         Submits a workflow to the Firecloud API.
@@ -88,11 +105,9 @@ def create_submission(self, submission_data):
     def poll_submission_status(self, submission_id):
         """
         Polls the status of a submission until it is complete and returns a dictionary of workflow IDs and their statuses.
-
         :param submission_id: The ID of the submission to poll
         :return: Dictionary with workflow IDs as keys and their statuses as values
         """
-        # Construct the API endpoint URL for polling submission status
         status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
         workflow_status_map = {}
 
@@ -100,6 +115,17 @@ def poll_submission_status(self, submission_id):
         while True:
             status_response = requests.get(status_url, headers=self.headers)
 
+            if status_response.status_code == 401:
+                print("Token expired, refreshing token...")
+                new_token = self.refresh_token()  # Get the new token
+                if new_token:
+                    self.token = new_token
+                    self.headers["Authorization"] = f"Bearer {self.token}"
+                    status_response = requests.get(status_url, headers=self.headers)
+                else:
+                    print("Failed to refresh token", file=sys.stderr)
+                    return {}
+
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:
                 print(f"Error: Received status code {status_response.status_code}", file=sys.stderr)
@@ -107,14 +133,12 @@ def poll_submission_status(self, submission_id):
                 return {}
 
             try:
-                # Parse the response as JSON
                 status_data = status_response.json()
             except json.JSONDecodeError:
                 print("Error decoding JSON response.", file=sys.stderr)
                 print(f"Response content: {status_response.text}", file=sys.stderr)
                 return {}
 
-            # Retrieve workflows and their statuses
             workflows = status_data.get("workflows", [])
             for workflow in workflows:
                 workflow_id = workflow.get("workflowId")
@@ -122,12 +146,11 @@ def poll_submission_status(self, submission_id):
                 if workflow_id and workflow_status:
                     workflow_status_map[workflow_id] = workflow_status
 
-            # Check if the submission is complete
             submission_status = status_data.get("status", "")
             if submission_status == "Done":
                 break
 
-            # Wait for 60 seconds before polling again
+            # Wait for 20 seconds before polling again
             time.sleep(20)
 
         return workflow_status_map

From 7f81b681dc20c32e86ec021d407600a7446d0a15 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 11:29:42 -0500
Subject: [PATCH 134/165] testing with 200

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index c7defd86fa..c2ea74db9a 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -115,7 +115,7 @@ def poll_submission_status(self, submission_id):
         while True:
             status_response = requests.get(status_url, headers=self.headers)
 
-            if status_response.status_code == 401:
+            if status_response.status_code == 200:
                 print("Token expired, refreshing token...")
                 new_token = self.refresh_token()  # Get the new token
                 if new_token:

From 79e7cdab5769e7feb6fd8bae08f8db6dcb124f72 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 11:31:24 -0500
Subject: [PATCH 135/165] testing with 200

---
 scripts/firecloud_api/firecloud_api.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index c2ea74db9a..86d3507054 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -117,7 +117,8 @@ def poll_submission_status(self, submission_id):
 
             if status_response.status_code == 200:
                 print("Token expired, refreshing token...")
-                new_token = self.refresh_token()  # Get the new token
+                new_token = self.refresh_token()
+                print(f"New token: {new_token}")
                 if new_token:
                     self.token = new_token
                     self.headers["Authorization"] = f"Bearer {self.token}"

From e8bf934da7d23f0f33a6e3907252a0972201cad1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 11:41:12 -0500
Subject: [PATCH 136/165] print status code

---
 scripts/firecloud_api/firecloud_api.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 86d3507054..c9824a7b6a 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -114,6 +114,9 @@ def poll_submission_status(self, submission_id):
         # Continuously poll the status of the submission until completion
         while True:
             status_response = requests.get(status_url, headers=self.headers)
+            print(f"Polling status for submission {submission_id}...")
+            print(f"Status response: {status_response.text}")
+            print(f"Status code: {status_response.status_code}")
 
             if status_response.status_code == 200:
                 print("Token expired, refreshing token...")

From 9e3b3129c0c9404bf895d01857281860a2ef5e00 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 11:43:36 -0500
Subject: [PATCH 137/165] print status code

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index c9824a7b6a..5b11dc962e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -118,7 +118,7 @@ def poll_submission_status(self, submission_id):
             print(f"Status response: {status_response.text}")
             print(f"Status code: {status_response.status_code}")
 
-            if status_response.status_code == 200:
+            if status_response.status_code == 401:
                 print("Token expired, refreshing token...")
                 new_token = self.refresh_token()
                 print(f"New token: {new_token}")

From d1dbfd1d9dce426c8b011a1ad334c03216a0ca02 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 12:10:08 -0500
Subject: [PATCH 138/165] print status code

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 90925758d6..70d8ec9b13 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -263,6 +263,7 @@ jobs:
             fi
 
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
+            firecloud_action poll_status --submission_id "$SUBMISSION_ID"
             RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 
             if [ -z "$RESPONSE" ]; then

From fb26921a489670941044fdbc8eba2d205f4d5e74 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 12:12:07 -0500
Subject: [PATCH 139/165] print status code

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 5b11dc962e..c9824a7b6a 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -118,7 +118,7 @@ def poll_submission_status(self, submission_id):
             print(f"Status response: {status_response.text}")
             print(f"Status code: {status_response.status_code}")
 
-            if status_response.status_code == 401:
+            if status_response.status_code == 200:
                 print("Token expired, refreshing token...")
                 new_token = self.refresh_token()
                 print(f"New token: {new_token}")

From 7867d84e8c3d46d35cc568b53ccf97baf04a0a5c Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 12:17:47 -0500
Subject: [PATCH 140/165] print status code

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 70d8ec9b13..98309835aa 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -249,7 +249,7 @@ jobs:
             echo "Time left for token expiry: $TIME_LEFT  seconds"
           
             # If token expires in less than 5 minutes, refresh it
-            if [ $TIME_LEFT -le 300 ]; then
+            if [ $TIME_LEFT -le 3600 ]; then
               echo "Token is about to expire in $TIME_LEFT seconds, refreshing..."
               # Refresh the token
               TOKEN=$(gcloud auth application-default print-access-token)

From 0b43690185fff6f9ff9eaadebc075fe26cc34dae Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 12:19:16 -0500
Subject: [PATCH 141/165] print status code

---
 scripts/firecloud_api/firecloud_api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index c9824a7b6a..5b11dc962e 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -118,7 +118,7 @@ def poll_submission_status(self, submission_id):
             print(f"Status response: {status_response.text}")
             print(f"Status code: {status_response.status_code}")
 
-            if status_response.status_code == 200:
+            if status_response.status_code == 401:
                 print("Token expired, refreshing token...")
                 new_token = self.refresh_token()
                 print(f"New token: {new_token}")

From 5df0ecf988da8f462fa7695d56bb261ef449fab1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 12:20:52 -0500
Subject: [PATCH 142/165] remove logic for inside while loop

---
 scripts/firecloud_api/firecloud_api.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 5b11dc962e..7cc2f92f68 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -118,17 +118,17 @@ def poll_submission_status(self, submission_id):
             print(f"Status response: {status_response.text}")
             print(f"Status code: {status_response.status_code}")
 
-            if status_response.status_code == 401:
-                print("Token expired, refreshing token...")
-                new_token = self.refresh_token()
-                print(f"New token: {new_token}")
-                if new_token:
-                    self.token = new_token
-                    self.headers["Authorization"] = f"Bearer {self.token}"
-                    status_response = requests.get(status_url, headers=self.headers)
-                else:
-                    print("Failed to refresh token", file=sys.stderr)
-                    return {}
+            #if status_response.status_code == 401:
+            #    print("Token expired, refreshing token...")
+            #    new_token = self.refresh_token()
+            #    print(f"New token: {new_token}")
+            #    if new_token:
+            #        self.token = new_token
+            #        self.headers["Authorization"] = f"Bearer {self.token}"
+            #        status_response = requests.get(status_url, headers=self.headers)
+            #    else:
+            #        print("Failed to refresh token", file=sys.stderr)
+            #        return {}
 
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:

From 889c46a6dc38d9bd6246394a8d3f9a32d121cded Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:07:42 -0500
Subject: [PATCH 143/165] remove logic for inside while loop

---
 .../test_illumina_genotyping_array.yml        | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 98309835aa..e6ac3eb829 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -73,15 +73,25 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      - name: Set Token Expiry Time
-        id: set_token_expiry
+      - name: Periodically Refresh Token
+        id: refresh_token
         run: |
-          # Get the current time and set expiration time to 1 hour from now (3600 seconds)
-          CURRENT_TIME=$(date +%s)
-          EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from current time
-          echo "Current time: $CURRENT_TIME"
-          echo "Expiration time: $EXPIRATION_TIME"
-          echo "expiration_time=$EXPIRATION_TIME" >> $GITHUB_ENV
+          while true; do
+            echo "Refreshing token..."
+            TOKEN=$(gcloud auth print-access-token)
+            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
+            sleep 1800  # Sleep for 30 minutes
+          done &
+
+      #- name: Set Token Expiry Time
+      #  id: set_token_expiry
+      #  run: |
+      #    # Get the current time and set expiration time to 1 hour from now (3600 seconds)
+      #    CURRENT_TIME=$(date +%s)
+      #    EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from current time
+      #    echo "Current time: $CURRENT_TIME"
+      #    echo "Expiration time: $EXPIRATION_TIME"
+      #    echo "expiration_time=$EXPIRATION_TIME" >> $GITHUB_ENV
 
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
@@ -263,7 +273,6 @@ jobs:
             fi
 
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
-            firecloud_action poll_status --submission_id "$SUBMISSION_ID"
             RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 
             if [ -z "$RESPONSE" ]; then

From a1460546095bd81208d1d7a47be06a5715f37933 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:10:52 -0500
Subject: [PATCH 144/165] start over

---
 .../test_illumina_genotyping_array.yml        | 52 ++++---------------
 1 file changed, 10 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index e6ac3eb829..adebd0ca28 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -1,4 +1,3 @@
-
 name: Test Illumina Genotyping Array
 
 # Controls when the workflow will run
@@ -73,26 +72,6 @@ jobs:
           access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      - name: Periodically Refresh Token
-        id: refresh_token
-        run: |
-          while true; do
-            echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token)
-            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 1800  # Sleep for 30 minutes
-          done &
-
-      #- name: Set Token Expiry Time
-      #  id: set_token_expiry
-      #  run: |
-      #    # Get the current time and set expiration time to 1 hour from now (3600 seconds)
-      #    CURRENT_TIME=$(date +%s)
-      #    EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from current time
-      #    echo "Current time: $CURRENT_TIME"
-      #    echo "Expiration time: $EXPIRATION_TIME"
-      #    echo "expiration_time=$EXPIRATION_TIME" >> $GITHUB_ENV
-
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
       # if github.head_ref is empty, it implies the workflow was triggered manually
@@ -127,6 +106,16 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
+      - name: Periodically Refresh Token
+        id: refresh_token
+        run: |
+          while true; do
+            echo "Refreshing token..."
+            TOKEN=$(gcloud auth print-access-token)
+            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
+            sleep 1800  # Sleep for 30 minutes
+          done &
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -251,27 +240,6 @@ jobs:
           
           echo "Monitoring the status of submitted workflows..."
           for SUBMISSION_ID in "${SUBMISSION_IDS[@]}"; do
-            # Before polling, check if the token is about to expire
-            CURRENT_TIME=$(date +%s)
-            echo "Current time: $CURRENT_TIME"
-            echo "Expiration time: $expiration_time"
-            TIME_LEFT=$(( $expiration_time - $CURRENT_TIME ))
-            echo "Time left for token expiry: $TIME_LEFT  seconds"
-          
-            # If token expires in less than 5 minutes, refresh it
-            if [ $TIME_LEFT -le 3600 ]; then
-              echo "Token is about to expire in $TIME_LEFT seconds, refreshing..."
-              # Refresh the token
-              TOKEN=$(gcloud auth application-default print-access-token)
-              echo "Token refreshed"
-              # Update expiration time after refreshing
-              CURRENT_TIME=$(date +%s)
-              EXPIRATION_TIME=$((CURRENT_TIME + 3600))  # 1 hour from now
-              echo "New expiration time: $EXPIRATION_TIME"
-            else
-              echo "Token valid, no need to refresh. Time left: $TIME_LEFT seconds"
-            fi
-
             echo "Polling submission status for Submission ID: $SUBMISSION_ID"
             RESPONSE=$(firecloud_action poll_status --submission_id "$SUBMISSION_ID") 
 

From 0929f95f9cbcce93ac4f1e5edbf0fa112b02e838 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:13:42 -0500
Subject: [PATCH 145/165] change token lifetime

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index adebd0ca28..d7cf241a19 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3600' # seconds, default is 3600
+          access_token_lifetime: '60' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.
@@ -113,7 +113,7 @@ jobs:
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 1800  # Sleep for 30 minutes
+            sleep 30  # Sleep for 30 seconds
           done &
 
       - name: Submit job, poll status, and get outputs

From 3de8468990cfe0a3649cce4b2667821ad486946f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:18:50 -0500
Subject: [PATCH 146/165] change token lifetime

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index d7cf241a19..2ef2337b85 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -113,7 +113,7 @@ jobs:
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 30  # Sleep for 30 seconds
+            sleep 5  # Sleep for 30 seconds
           done &
 
       - name: Submit job, poll status, and get outputs

From f0a4149464828bf31c7e3130c839c19cb3eef522 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:24:13 -0500
Subject: [PATCH 147/165] remove &

---
 .github/workflows/test_illumina_genotyping_array.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 2ef2337b85..cc6fdda6e1 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -113,8 +113,8 @@ jobs:
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 5  # Sleep for 30 seconds
-          done &
+            sleep 5  # Sleep for 30 seconds 
+          done
 
       - name: Submit job, poll status, and get outputs
         id: pipeline_run

From 11b7817aa3379d3e450c66c669f3a35884f28a4d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:27:44 -0500
Subject: [PATCH 148/165] wait

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index cc6fdda6e1..3877f4924b 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -114,7 +114,8 @@ jobs:
             TOKEN=$(gcloud auth print-access-token)
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
             sleep 5  # Sleep for 30 seconds 
-          done
+          done &
+          wait 
 
       - name: Submit job, poll status, and get outputs
         id: pipeline_run

From 8a88ef56ae8a02db77283b0b304e0683beecc24d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:33:27 -0500
Subject: [PATCH 149/165] move while loop

---
 .../test_illumina_genotyping_array.yml        | 27 ++++++++++++-------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 3877f4924b..952d8ac894 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -106,16 +106,16 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      - name: Periodically Refresh Token
-        id: refresh_token
-        run: |
-          while true; do
-            echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token)
-            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 5  # Sleep for 30 seconds 
-          done &
-          wait 
+      #- name: Periodically Refresh Token
+      #  id: refresh_token
+      #  run: |
+      #    while true; do
+      #      echo "Refreshing token..."
+      #      TOKEN=$(gcloud auth print-access-token)
+      #      echo "TOKEN=$TOKEN" >> $GITHUB_ENV
+      #      sleep 5  # Sleep for 30 seconds
+      #    done &
+      #    wait
 
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
@@ -199,6 +199,13 @@ jobs:
           MAX_RETRIES=2
           RETRY_DELAY=300  # 300 seconds = 5 minutes
           
+          while true; do
+            echo "Refreshing token..."
+            TOKEN=$(gcloud auth print-access-token)
+            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
+            sleep 5  # Sleep for 30 seconds
+          done &
+          
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
             test_input_file=$(python3 scripts/firecloud_api/UpdateTestInputs.py --truth_path "$TRUTH_PATH" \

From a748e92f3919b2eab100f3bad6f297e65e81d267 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:36:12 -0500
Subject: [PATCH 150/165] move while loop

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 952d8ac894..ab4cb53077 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -203,7 +203,7 @@ jobs:
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-            sleep 5  # Sleep for 30 seconds
+            sleep 30  # Sleep for 30 seconds
           done &
           
           for input_file in "$INPUTS_DIR"/*.json; do

From df37f337dda9f88c7e67c041ea28e3e65bf3ba4f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 13:43:13 -0500
Subject: [PATCH 151/165] move while loop

---
 .github/workflows/test_illumina_genotyping_array.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index ab4cb53077..c8b2e9add6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -202,6 +202,7 @@ jobs:
           while true; do
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
+            echo "TOKEN=$TOKEN"
             echo "TOKEN=$TOKEN" >> $GITHUB_ENV
             sleep 30  # Sleep for 30 seconds
           done &

From dcd42351d8b7f118b841d45b6f11297018ea09fb Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:05:00 -0500
Subject: [PATCH 152/165] --lifetime 60

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c8b2e9add6..1244a97144 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -201,9 +201,8 @@ jobs:
           
           while true; do
             echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token)
+            TOKEN=$(gcloud auth print-access-token --lifetime 60)
             echo "TOKEN=$TOKEN"
-            echo "TOKEN=$TOKEN" >> $GITHUB_ENV
             sleep 30  # Sleep for 30 seconds
           done &
           

From 1c5dae4f47a1784e3a5c7104df0adc7d789c1d2f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:08:49 -0500
Subject: [PATCH 153/165] --lifetime 60

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 1244a97144..f2748d6174 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -201,7 +201,7 @@ jobs:
           
           while true; do
             echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token --lifetime 60)
+            TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account = pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
             echo "TOKEN=$TOKEN"
             sleep 30  # Sleep for 30 seconds
           done &

From 132c639f7632e93accca11a15a4a068b38200d3a Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:09:58 -0500
Subject: [PATCH 154/165] --lifetime 60

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index f2748d6174..a421e6f8b2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -201,7 +201,7 @@ jobs:
           
           while true; do
             echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account = pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
+            TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account=pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
             echo "TOKEN=$TOKEN"
             sleep 30  # Sleep for 30 seconds
           done &

From 10c5dfb591652d0adbc7c1e5d762aabd8d3f4957 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:12:43 -0500
Subject: [PATCH 155/165] --lifetime 60

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index a421e6f8b2..7dddab7c93 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -199,6 +199,8 @@ jobs:
           MAX_RETRIES=2
           RETRY_DELAY=300  # 300 seconds = 5 minutes
           
+          gcloud auth list
+          
           while true; do
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account=pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)

From 69b8e4f658409f2c83938637fc08ce2812f500d7 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:34:26 -0500
Subject: [PATCH 156/165] echo acct

---
 .../test_illumina_genotyping_array.yml          | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 7dddab7c93..b6c5831d8c 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -106,17 +106,6 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
-      #- name: Periodically Refresh Token
-      #  id: refresh_token
-      #  run: |
-      #    while true; do
-      #      echo "Refreshing token..."
-      #      TOKEN=$(gcloud auth print-access-token)
-      #      echo "TOKEN=$TOKEN" >> $GITHUB_ENV
-      #      sleep 5  # Sleep for 30 seconds
-      #    done &
-      #    wait
-
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -198,11 +187,11 @@ jobs:
           
           MAX_RETRIES=2
           RETRY_DELAY=300  # 300 seconds = 5 minutes
-          
-          gcloud auth list
-          
+                    
           while true; do
             echo "Refreshing token..."
+            ACCOUNT=$(gcloud auth list --filter=status:ACTIVE --format="value(account)")
+            echo "Current account: $ACCOUNT"
             TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account=pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
             echo "TOKEN=$TOKEN"
             sleep 30  # Sleep for 30 seconds

From 50e300edd23438d2df3396b0669a4440a4b9664d Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:37:22 -0500
Subject: [PATCH 157/165] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index b6c5831d8c..14ae0eb7fc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -190,8 +190,7 @@ jobs:
                     
           while true; do
             echo "Refreshing token..."
-            ACCOUNT=$(gcloud auth list --filter=status:ACTIVE --format="value(account)")
-            echo "Current account: $ACCOUNT"
+            gcloud auth list
             TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account=pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
             echo "TOKEN=$TOKEN"
             sleep 30  # Sleep for 30 seconds

From 087d5d6eb38a25e6714ca10eb58954e8f1466cc4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:40:29 -0500
Subject: [PATCH 158/165] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 14ae0eb7fc..c4e61750e6 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -72,6 +72,12 @@ jobs:
           access_token_lifetime: '60' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
+      - name: Debug Authentication
+        run: |
+          echo "Checking gcloud authentication..."
+          gcloud auth list
+          gcloud auth print-access-token
+
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
       # if github.head_ref is empty, it implies the workflow was triggered manually

From a23da82621127b8bba870467174b71080abd74b1 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:42:55 -0500
Subject: [PATCH 159/165] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c4e61750e6..639afcebfc 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -72,11 +72,9 @@ jobs:
           access_token_lifetime: '60' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      - name: Debug Authentication
+      - name: Debug Auth Outputs
         run: |
-          echo "Checking gcloud authentication..."
-          gcloud auth list
-          gcloud auth print-access-token
+          echo "Access Token: ${{ steps.auth.outputs.access_token }}"
 
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request

From 666758056c4ca459acff58c596a2026628d8fe7f Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:44:55 -0500
Subject: [PATCH 160/165] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 639afcebfc..c925378323 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -75,6 +75,12 @@ jobs:
       - name: Debug Auth Outputs
         run: |
           echo "Access Token: ${{ steps.auth.outputs.access_token }}"
+      - name: Authenticate gcloud with token
+        run: |
+          echo "${{ steps.auth.outputs.access_token }}" | gcloud auth activate-service-account --key-file=-
+          gcloud config set project warp-pipeline-dev
+          gcloud auth list
+
 
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request

From 363da5b272b84a9badf50d6e585a79b2cf0046d4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:48:11 -0500
Subject: [PATCH 161/165] echo acct

---
 .../test_illumina_genotyping_array.yml         | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index c925378323..23c84ee896 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,19 +69,9 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '60' # seconds, default is 3600
+          access_token_lifetime: '3660' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
-      - name: Debug Auth Outputs
-        run: |
-          echo "Access Token: ${{ steps.auth.outputs.access_token }}"
-      - name: Authenticate gcloud with token
-        run: |
-          echo "${{ steps.auth.outputs.access_token }}" | gcloud auth activate-service-account --key-file=-
-          gcloud config set project warp-pipeline-dev
-          gcloud auth list
-
-
       # Set the branch name.
       # github.head_ref contains the name of the branch in the context of a pull request
       # if github.head_ref is empty, it implies the workflow was triggered manually
@@ -200,10 +190,8 @@ jobs:
                     
           while true; do
             echo "Refreshing token..."
-            gcloud auth list
-            TOKEN=$(gcloud auth print-access-token --lifetime 60 --impersonate-service-account=pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com)
-            echo "TOKEN=$TOKEN"
-            sleep 30  # Sleep for 30 seconds
+            TOKEN=$(gcloud auth print-access-token)
+            sleep 50  # Sleep for 30 seconds
           done &
           
           for input_file in "$INPUTS_DIR"/*.json; do

From 6ca3eb27a3822d049cd27fbfd5b77afba0130195 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:49:10 -0500
Subject: [PATCH 162/165] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 23c84ee896..0eeafffe04 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -191,7 +191,7 @@ jobs:
           while true; do
             echo "Refreshing token..."
             TOKEN=$(gcloud auth print-access-token)
-            sleep 50  # Sleep for 30 seconds
+            sleep 3000 # sleep for 50 minutes
           done &
           
           for input_file in "$INPUTS_DIR"/*.json; do

From 514b8dd1c8faea309fd54c9e90d92584bcc431d0 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 14:51:49 -0500
Subject: [PATCH 163/165] echo acct

---
 .github/workflows/test_illumina_genotyping_array.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 0eeafffe04..5b993b87a2 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -69,7 +69,7 @@ jobs:
           workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider'
           # This is our tester service account
           service_account: 'pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com'
-          access_token_lifetime: '3660' # seconds, default is 3600
+          access_token_lifetime: '3600' # seconds, default is 3600
           access_token_scopes: 'profile, email, openid'
 
       # Set the branch name.

From a8568d441f55776420492f4cdd7c6da5f7d47cb4 Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Tue, 3 Dec 2024 15:07:41 -0500
Subject: [PATCH 164/165] echo acct

---
 scripts/firecloud_api/firecloud_api.py | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index 7cc2f92f68..f01c784129 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -105,30 +105,17 @@ def create_submission(self, submission_data):
     def poll_submission_status(self, submission_id):
         """
         Polls the status of a submission until it is complete and returns a dictionary of workflow IDs and their statuses.
+
         :param submission_id: The ID of the submission to poll
         :return: Dictionary with workflow IDs as keys and their statuses as values
         """
+        # Construct the API endpoint URL for polling submission status
         status_url = f"{self.base_url}/workspaces/{self.namespace}/{self.workspace_name}/submissions/{submission_id}"
         workflow_status_map = {}
 
         # Continuously poll the status of the submission until completion
         while True:
             status_response = requests.get(status_url, headers=self.headers)
-            print(f"Polling status for submission {submission_id}...")
-            print(f"Status response: {status_response.text}")
-            print(f"Status code: {status_response.status_code}")
-
-            #if status_response.status_code == 401:
-            #    print("Token expired, refreshing token...")
-            #    new_token = self.refresh_token()
-            #    print(f"New token: {new_token}")
-            #    if new_token:
-            #        self.token = new_token
-            #        self.headers["Authorization"] = f"Bearer {self.token}"
-            #        status_response = requests.get(status_url, headers=self.headers)
-            #    else:
-            #        print("Failed to refresh token", file=sys.stderr)
-            #        return {}
 
             # Check if the response status code is successful (200)
             if status_response.status_code != 200:
@@ -137,12 +124,14 @@ def poll_submission_status(self, submission_id):
                 return {}
 
             try:
+                # Parse the response as JSON
                 status_data = status_response.json()
             except json.JSONDecodeError:
                 print("Error decoding JSON response.", file=sys.stderr)
                 print(f"Response content: {status_response.text}", file=sys.stderr)
                 return {}
 
+            # Retrieve workflows and their statuses
             workflows = status_data.get("workflows", [])
             for workflow in workflows:
                 workflow_id = workflow.get("workflowId")
@@ -150,11 +139,12 @@ def poll_submission_status(self, submission_id):
                 if workflow_id and workflow_status:
                     workflow_status_map[workflow_id] = workflow_status
 
+            # Check if the submission is complete
             submission_status = status_data.get("status", "")
             if submission_status == "Done":
                 break
 
-            # Wait for 20 seconds before polling again
+            # Wait for 60 seconds before polling again
             time.sleep(20)
 
         return workflow_status_map

From 50897cfe851920188b8479a596bc954c6164ce5b Mon Sep 17 00:00:00 2001
From: npetrill <npetrill@broadinstitute.org>
Date: Wed, 4 Dec 2024 13:18:47 -0500
Subject: [PATCH 165/165] wait before submitting any workflows

---
 .../test_illumina_genotyping_array.yml        | 15 ++++++----
 scripts/firecloud_api/firecloud_api.py        | 30 +++++++++----------
 2 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/test_illumina_genotyping_array.yml b/.github/workflows/test_illumina_genotyping_array.yml
index 5b993b87a2..9a9e878e7f 100644
--- a/.github/workflows/test_illumina_genotyping_array.yml
+++ b/.github/workflows/test_illumina_genotyping_array.yml
@@ -106,6 +106,11 @@ jobs:
           # Use the testType provided by the user
           echo "testType=${{ github.event.inputs.testType }}" >> $GITHUB_ENV
 
+      - name: Delay before submitting workflows
+        run: |
+          echo "Waiting for 5 minutes before submitting workflows..."
+          sleep 300  # 300 seconds = 5 minutes
+
       - name: Submit job, poll status, and get outputs
         id: pipeline_run
         run: |
@@ -188,11 +193,11 @@ jobs:
           MAX_RETRIES=2
           RETRY_DELAY=300  # 300 seconds = 5 minutes
                     
-          while true; do
-            echo "Refreshing token..."
-            TOKEN=$(gcloud auth print-access-token)
-            sleep 3000 # sleep for 50 minutes
-          done &
+          #while true; do
+          #  echo "Refreshing token..."
+          #  TOKEN=$(gcloud auth print-access-token)
+          #  sleep 3000 # sleep for 50 minutes
+          #done &
           
           for input_file in "$INPUTS_DIR"/*.json; do
             echo "Processing input file: $input_file"
diff --git a/scripts/firecloud_api/firecloud_api.py b/scripts/firecloud_api/firecloud_api.py
index f01c784129..9961af4e10 100644
--- a/scripts/firecloud_api/firecloud_api.py
+++ b/scripts/firecloud_api/firecloud_api.py
@@ -64,21 +64,21 @@ def get_workflow_outputs(self, submission_id, workflow_id, pipeline_name):
             print(f"Failed to retrieve workflow outputs. Status code: {response.status_code}")
             return None, None
 
-    def refresh_token(self):
-        """
-        Refreshes the API token using gcloud's application default credentials.
-        :return: The new token as a string
-        """
-        try:
-            # Execute the gcloud command to get the new access token
-            result = subprocess.run(
-                ["gcloud", "auth", "application-default", "print-access-token"],
-                capture_output=True, text=True, check=True
-            )
-            return result.stdout.strip()  # Return the new token
-        except subprocess.CalledProcessError as e:
-            print(f"Error refreshing token: {e.stderr}", file=sys.stderr)
-            return None
+    #def refresh_token(self):
+    #    """
+    #    Refreshes the API token using gcloud's application default credentials.
+    #    :return: The new token as a string
+    #    """
+    #    try:
+    #        # Execute the gcloud command to get the new access token
+    #        result = subprocess.run(
+    #            ["gcloud", "auth", "application-default", "print-access-token"],
+    #            capture_output=True, text=True, check=True
+    #        )
+    #        return result.stdout.strip()  # Return the new token
+    #    except subprocess.CalledProcessError as e:
+    #        print(f"Error refreshing token: {e.stderr}", file=sys.stderr)
+    #        return None
 
     def create_submission(self, submission_data):
         """