Enable traditional threading as an option (#3149)

In a previous PR, stubs were added to enable traditional threading in the UFS. This PR takes the next step to execute the model with traditional threading. Specifically; 1. Do not use the `_esmf` `ufs_configure` file for traditional threading. This sets the `globalResourceControl` to `false` (traditional threading) ... this was done in the previous PR 2. when calculating components pet bounds do not multiply with number of threads also in `ufs_configure` ... this PR 3. `export OMP_NUM_THREADS=$UFS_THREADS` ... this was done in the previous PR The default behaviour to use ESMF-managed threading is retained. Traditional threading can be achieved by toggling `USE_ESMF_THREADING` flag in `config.fcst` Closes #3122 - as in allows the model to be configured to run w/ traditional threading. Testing on C768 S2SW was successful on Hercules (This was previously reported to be failing) The use of traditional threading will be inefficient for model components that are not thread-enabled and supported e.g. MOM6 and CICE6, but since they require smaller number of tasks compared to ATM or WAV, the CPU 'waste' is not huge.
NOAA-EMC · Dec 13, 2024 · 0db859e · 0db859e
1 parent 2b11995
commit 0db859e
Show file tree

Hide file tree

Showing 3 changed files with 95 additions and 33 deletions.
diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources
@@ -110,29 +110,41 @@ case ${step} in
       [[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1
     fi
 
-    # PETS for the atmosphere dycore
-    (( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
-    echo "FV3 using (nthreads, PETS) = (${nthreads_fv3}, ${FV3PETS})"
+    # FV3
+    if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+      (( FV3THREADS = nthreads_fv3 ))
+      (( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
+    else
+      (( FV3THREADS = UFS_THREADS ))
+      (( FV3PETS = ntasks_fv3 ))
+    fi
+    echo "FV3 using (nthreads, PETS) = (${FV3THREADS}, ${FV3PETS})"
 
-    # PETS for quilting
+    # Write grid component
+    QUILTPETS=0; QUILTTHREADS=0
     if [[ "${QUILTING:-}" == ".true." ]]; then
-      (( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
+      if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+        (( QUILTTHREADS = nthreads_fv3 ))
+        (( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
+      else
+        (( QUILTTHREADS = UFS_THREADS ))
+        (( QUILTPETS = ntasks_quilt ))
+      fi
       (( WRTTASK_PER_GROUP = WRTTASK_PER_GROUP_PER_THREAD ))
       export WRTTASK_PER_GROUP
-    else
-      QUILTPETS=0
     fi
-    echo "QUILT using (nthreads, PETS) = (${nthreads_fv3}, ${QUILTPETS})"
+    echo "QUILT using (nthreads, PETS) = (${QUILTTHREADS}, ${QUILTPETS})"
 
     # Total PETS for the atmosphere component
-    ATMTHREADS=${nthreads_fv3}
+    ATMTHREADS=${FV3THREADS}
     (( ATMPETS = FV3PETS + QUILTPETS ))
     export ATMPETS ATMTHREADS
     echo "FV3ATM using (nthreads, PETS) = (${ATMTHREADS}, ${ATMPETS})"
 
     # Total PETS for the coupled model (starting w/ the atmosphere)
     NTASKS_TOT=${ATMPETS}
 
+    # Mediator
     # The mediator PETS can overlap with other components, usually it lands on the atmosphere tasks.
     # However, it is suggested limiting mediator PETS to 300, as it may cause the slow performance.
     # See https://docs.google.com/document/d/1bKpi-52t5jIfv2tuNHmQkYUe3hkKsiG_DG_s6Mnukog/edit
@@ -143,6 +155,7 @@ case ${step} in
     export MEDPETS MEDTHREADS
     echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})"
 
+    # GOCART
     CHMPETS=0; CHMTHREADS=0
     if [[ "${DO_AERO_FCST}" == "YES" ]]; then
       # GOCART shares the same grid and forecast tasks as FV3 (do not add write grid component tasks).
@@ -153,28 +166,46 @@ case ${step} in
     fi
     export CHMPETS CHMTHREADS
 
+    # Waves
     WAVPETS=0; WAVTHREADS=0
     if [[ "${DO_WAVE}" == "YES" ]]; then
-      (( WAVPETS = ntasks_ww3 * nthreads_ww3 ))
-      (( WAVTHREADS = nthreads_ww3 ))
+      if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+        (( WAVTHREADS = nthreads_ww3 ))
+        (( WAVPETS = ntasks_ww3 * nthreads_ww3 ))
+      else
+        (( WAVTHREADS = UFS_THREADS ))
+        (( WAVPETS = ntasks_ww3 ))
+      fi
       echo "WW3 using (threads, PETS) = (${WAVTHREADS}, ${WAVPETS})"
       (( NTASKS_TOT = NTASKS_TOT + WAVPETS ))
     fi
     export WAVPETS WAVTHREADS
 
+    # Ocean
     OCNPETS=0; OCNTHREADS=0
     if [[ "${DO_OCN}" == "YES" ]]; then
-      (( OCNPETS = ntasks_mom6 * nthreads_mom6 ))
-      (( OCNTHREADS = nthreads_mom6 ))
+      if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+        (( OCNTHREADS = nthreads_mom6 ))
+        (( OCNPETS = ntasks_mom6 * nthreads_mom6 ))
+      else
+        (( OCNTHREADS = UFS_THREADS ))
+        (( OCNPETS = ntasks_mom6 ))
+      fi
       echo "MOM6 using (threads, PETS) = (${OCNTHREADS}, ${OCNPETS})"
       (( NTASKS_TOT = NTASKS_TOT + OCNPETS ))
     fi
     export OCNPETS OCNTHREADS
 
+    # Ice
     ICEPETS=0; ICETHREADS=0
     if [[ "${DO_ICE}" == "YES" ]]; then
-      (( ICEPETS = ntasks_cice6 * nthreads_cice6 ))
-      (( ICETHREADS = nthreads_cice6 ))
+      if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+        (( ICETHREADS = nthreads_cice6 ))
+        (( ICEPETS = ntasks_cice6 * nthreads_cice6 ))
+      else
+        (( ICETHREADS = UFS_THREADS ))
+        (( ICEPETS = ntasks_cice6 ))
+      fi
       echo "CICE6 using (threads, PETS) = (${ICETHREADS}, ${ICEPETS})"
       (( NTASKS_TOT = NTASKS_TOT + ICEPETS ))
     fi
@@ -184,7 +215,7 @@ case ${step} in
 
     declare -x "ntasks"="${NTASKS_TOT}"
     declare -x "threads_per_task"="${UFS_THREADS}"
-    declare -x "tasks_per_node"="${max_tasks_per_node}"
+    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
 
     case "${CASE}" in
       "C48" | "C96" | "C192")

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -775,29 +775,41 @@ case ${step} in
       export npy_nest=${npy_nest:-961}
     fi
 
-    # PETS for the atmosphere dycore
-    (( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
-    echo "FV3 using (nthreads, PETS) = (${nthreads_fv3}, ${FV3PETS})"
+    # FV3
+    if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+      (( FV3THREADS = nthreads_fv3 ))
+      (( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
+    else
+      (( FV3THREADS = UFS_THREADS ))
+      (( FV3PETS = ntasks_fv3 ))
+    fi
+    echo "FV3 using (nthreads, PETS) = (${FV3THREADS}, ${FV3PETS})"
 
-    # PETS for quilting
+    # Write grid component
+    QUILTPETS=0; QUILTTHREADS=0
     if [[ "${QUILTING:-}" == ".true." ]]; then
-      (( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
+      if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+        (( QUILTTHREADS = nthreads_fv3 ))
+        (( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
+      else
+        (( QUILTTHREADS = UFS_THREADS ))
+        (( QUILTPETS = ntasks_quilt ))
+      fi
       (( WRTTASK_PER_GROUP = WRTTASK_PER_GROUP_PER_THREAD ))
       export WRTTASK_PER_GROUP
-    else
-      QUILTPETS=0
     fi
-    echo "QUILT using (nthreads, PETS) = (${nthreads_fv3}, ${QUILTPETS})"
+    echo "QUILT using (nthreads, PETS) = (${QUILTTHREADS}, ${QUILTPETS})"
 
     # Total PETS for the atmosphere component
-    ATMTHREADS=${nthreads_fv3}
+    ATMTHREADS=${FV3THREADS}
     (( ATMPETS = FV3PETS + QUILTPETS ))
     export ATMPETS ATMTHREADS
     echo "FV3ATM using (nthreads, PETS) = (${ATMTHREADS}, ${ATMPETS})"
 
     # Total PETS for the coupled model (starting w/ the atmosphere)
     NTASKS_TOT=${ATMPETS}
 
+    # Mediator
     # The mediator PETS can overlap with other components, usually it lands on the atmosphere tasks.
     # However, it is suggested limiting mediator PETS to 300, as it may cause the slow performance.
     # See https://docs.google.com/document/d/1bKpi-52t5jIfv2tuNHmQkYUe3hkKsiG_DG_s6Mnukog/edit
@@ -808,6 +820,7 @@ case ${step} in
     export MEDPETS MEDTHREADS
     echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})"
 
+    # GOCART
     CHMPETS=0; CHMTHREADS=0
     if [[ "${DO_AERO_FCST}" == "YES" ]]; then
       # GOCART shares the same grid and forecast tasks as FV3 (do not add write grid component tasks).
@@ -818,28 +831,46 @@ case ${step} in
     fi
     export CHMPETS CHMTHREADS
 
+    # Waves
     WAVPETS=0; WAVTHREADS=0
     if [[ "${DO_WAVE}" == "YES" ]]; then
-      (( WAVPETS = ntasks_ww3 * nthreads_ww3 ))
-      (( WAVTHREADS = nthreads_ww3 ))
+      if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+        (( WAVTHREADS = nthreads_ww3 ))
+        (( WAVPETS = ntasks_ww3 * nthreads_ww3 ))
+      else
+        (( WAVTHREADS = UFS_THREADS ))
+        (( WAVPETS = ntasks_ww3 ))
+      fi
       echo "WW3 using (threads, PETS) = (${WAVTHREADS}, ${WAVPETS})"
       (( NTASKS_TOT = NTASKS_TOT + WAVPETS ))
     fi
     export WAVPETS WAVTHREADS
 
+    # Ocean
     OCNPETS=0; OCNTHREADS=0
     if [[ "${DO_OCN}" == "YES" ]]; then
-      (( OCNPETS = ntasks_mom6 * nthreads_mom6 ))
-      (( OCNTHREADS = nthreads_mom6 ))
+      if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+        (( OCNTHREADS = nthreads_mom6 ))
+        (( OCNPETS = ntasks_mom6 * nthreads_mom6 ))
+      else
+        (( OCNTHREADS = UFS_THREADS ))
+        (( OCNPETS = ntasks_mom6 ))
+      fi
       echo "MOM6 using (threads, PETS) = (${OCNTHREADS}, ${OCNPETS})"
       (( NTASKS_TOT = NTASKS_TOT + OCNPETS ))
     fi
     export OCNPETS OCNTHREADS
 
+    # Ice
     ICEPETS=0; ICETHREADS=0
     if [[ "${DO_ICE}" == "YES" ]]; then
-      (( ICEPETS = ntasks_cice6 * nthreads_cice6 ))
-      (( ICETHREADS = nthreads_cice6 ))
+      if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
+        (( ICETHREADS = nthreads_cice6 ))
+        (( ICEPETS = ntasks_cice6 * nthreads_cice6 ))
+      else
+        (( ICETHREADS = UFS_THREADS ))
+        (( ICEPETS = ntasks_cice6 ))
+      fi
       echo "CICE6 using (threads, PETS) = (${ICETHREADS}, ${ICEPETS})"
       (( NTASKS_TOT = NTASKS_TOT + ICEPETS ))
     fi
@@ -849,7 +880,7 @@ case ${step} in
 
     declare -x "ntasks"="${NTASKS_TOT}"
     declare -x "threads_per_task"="${UFS_THREADS}"
-    declare -x "tasks_per_node"="${max_tasks_per_node}"
+    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
 
     case "${CASE}" in
       "C48" | "C96" | "C192")

diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh
@@ -26,7 +26,7 @@ local FHROT=${IAU_FHROT:-0}
 local DT_ATMOS=${DELTIM}
 local RESTART_INTERVAL="${FV3_RESTART_FH[*]}"
 # QUILTING
-local QUILTING_RESTART=".true."
+local QUILTING_RESTART="${QUILTING_RESTART:-${QUILTING}}"
 local WRITE_GROUP=${WRITE_GROUP:-1}
 local WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP:-24}
 local ITASKS=1