From 22ae3676e622653b537cdfdedff2b66fd20468b1 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 29 Aug 2024 10:04:13 -0500
Subject: [PATCH 01/21] Update model hash, reenable Orion cycling support

---
 .gitignore                         | 32 +++++++++++++++---------------
 jobs/rocoto/upp.sh                 |  5 ++++-
 parm/config/gefs/config.base       |  4 ++--
 parm/post/upp.yaml                 |  8 ++++----
 sorc/build_all.sh                  |  8 ++++----
 sorc/link_workflow.sh              | 21 +++++++++++++-------
 sorc/ufs_model.fd                  |  2 +-
 ush/forecast_predet.sh             |  4 ++--
 ush/parsing_model_configure_FV3.sh |  2 +-
 9 files changed, 48 insertions(+), 38 deletions(-)

diff --git a/.gitignore b/.gitignore
index f5e33f4f61..f732eeb19e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -87,22 +87,22 @@ parm/post/postcntrl_gfs_goes.xml-new
 parm/post/postcntrl_gfs_two.xml
 parm/post/postcntrl_gfs_wafs.xml
 parm/post/postcntrl_gfs_wafs_anl.xml
-parm/post/postxconfig-NT-GEFS-ANL.txt
-parm/post/postxconfig-NT-GEFS-F00.txt
-parm/post/postxconfig-NT-GEFS-F00-aerosol.txt
-parm/post/postxconfig-NT-GEFS-WAFS.txt
-parm/post/postxconfig-NT-GEFS-aerosol.txt
-parm/post/postxconfig-NT-GEFS.txt
-parm/post/postxconfig-NT-GFS-ANL.txt
-parm/post/postxconfig-NT-GFS-F00-TWO.txt
-parm/post/postxconfig-NT-GFS-F00.txt
-parm/post/postxconfig-NT-GFS-FLUX-F00.txt
-parm/post/postxconfig-NT-GFS-FLUX.txt
-parm/post/postxconfig-NT-GFS-GOES.txt
-parm/post/postxconfig-NT-GFS-TWO.txt
-parm/post/postxconfig-NT-GFS-WAFS-ANL.txt
-parm/post/postxconfig-NT-GFS-WAFS.txt
-parm/post/postxconfig-NT-GFS.txt
+parm/post/postxconfig-NT-gefs-anl.txt
+parm/post/postxconfig-NT-gefs-f00.txt
+parm/post/postxconfig-NT-gefs-f00-aerosol.txt
+parm/post/postxconfig-NT-gefs-wafs.txt
+parm/post/postxconfig-NT-gefs-aerosol.txt
+parm/post/postxconfig-NT-gefs.txt
+parm/post/postxconfig-NT-gfs-anl.txt
+parm/post/postxconfig-NT-gfs-f00-two.txt
+parm/post/postxconfig-NT-gfs-f00.txt
+parm/post/postxconfig-NT-gfs-flux-f00.txt
+parm/post/postxconfig-NT-gfs-flux.txt
+parm/post/postxconfig-NT-gfs-goes.txt
+parm/post/postxconfig-NT-gfs-two.txt
+parm/post/postxconfig-NT-gfs-wafs-anl.txt
+parm/post/postxconfig-NT-gfs-wafs.txt
+parm/post/postxconfig-NT-gfs.txt
 parm/post/postxconfig-NT-gefs-aerosol.txt
 parm/post/postxconfig-NT-gefs-chem.txt
 parm/post/ocean.csv
diff --git a/jobs/rocoto/upp.sh b/jobs/rocoto/upp.sh
index c3f128ab02..9daf128ca2 100755
--- a/jobs/rocoto/upp.sh
+++ b/jobs/rocoto/upp.sh
@@ -36,7 +36,10 @@ if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
   export PYTHONPATH
 
 else
-  . "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+  # For spack-stack 1.6.0, the UPP has its own environment (upp-addon-dev)
+  module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles"
+  module load "${MACHINE_ID}"
+  #. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
   status=$?
   if (( status != 0 )); then exit "${status}"; fi
 fi
diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
index 189b7ba446..6b6cafa04a 100644
--- a/parm/config/gefs/config.base
+++ b/parm/config/gefs/config.base
@@ -269,8 +269,8 @@ export WRITE_DOPOST=".true." # WRITE_DOPOST=true, use inline POST
 export WRITE_NSFLIP=".true."
 
 # Override normal post flat files for GEFS
-export FLTFILEGFS="${PARMgfs}/post/postxconfig-NT-GEFS.txt"
-export FLTFILEGFSF00="${PARMgfs}/post/postxconfig-NT-GEFS-F00.txt"
+export FLTFILEGFS="${PARMgfs}/post/postxconfig-NT-gefs.txt"
+export FLTFILEGFSF00="${PARMgfs}/post/postxconfig-NT-gefs-f00.txt"
 
 # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL
 export imp_physics=8
diff --git a/parm/post/upp.yaml b/parm/post/upp.yaml
index a39e2be877..aa6768f8f6 100644
--- a/parm/post/upp.yaml
+++ b/parm/post/upp.yaml
@@ -18,7 +18,7 @@ analysis:
     rdaod: True
   data_in:
     copy:
-      - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-ANL.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/postxconfig-NT-gfs-anl.txt", "{{ DATA }}/postxconfig-NT.txt"]
       - ["{{ COM_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmanl.nc", "{{ DATA }}/{{ atmos_filename }}"]
       - ["{{ COM_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcanl.nc", "{{ DATA }}/{{ flux_filename }}"]
   data_out:
@@ -32,9 +32,9 @@ forecast:
   data_in:
     copy:
       {% if forecast_hour == 0 %}
-      - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-F00-TWO.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/postxconfig-NT-gfs-f00-two.txt", "{{ DATA }}/postxconfig-NT.txt"]
       {% else %}
-      - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-TWO.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/postxconfig-NT-gfs-two.txt", "{{ DATA }}/postxconfig-NT.txt"]
       {% endif %}
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"]
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"]
@@ -81,7 +81,7 @@ goes:
       {% endfor %}
       - ["{{ 'CRTM_FIX' | getenv }}/AerosolCoeff.bin", "{{ DATA }}/"]
       - ["{{ 'CRTM_FIX' | getenv }}/CloudCoeff.bin", "{{ DATA }}/"]
-      - ["{{ PARMgfs }}/post/postxconfig-NT-GFS-GOES.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/postxconfig-NT-gfs-goes.txt", "{{ DATA }}/postxconfig-NT.txt"]
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"]
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"]
   data_out:
diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index b6c4e6cc1c..79ae3c937f 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -131,16 +131,16 @@ build_jobs["ufs"]=8
 big_jobs=$((big_jobs+1))
 build_opts["ufs"]="${_wave_opt} ${_verbose_opt} ${_build_ufs_opt} ${_build_debug}"
 
-build_jobs["upp"]=2
+build_jobs["upp"]=1
 build_opts["upp"]="${_build_debug}"
 
-build_jobs["ufs_utils"]=2
+build_jobs["ufs_utils"]=1
 build_opts["ufs_utils"]="${_verbose_opt} ${_build_debug}"
 
 build_jobs["gfs_utils"]=1
 build_opts["gfs_utils"]="${_verbose_opt} ${_build_debug}"
 
-build_jobs["ww3prepost"]=2
+build_jobs["ww3prepost"]=1
 build_opts["ww3prepost"]="${_wave_opt} ${_verbose_opt} ${_build_ufs_opt} ${_build_debug}"
 
 # Optional DA builds
@@ -154,7 +154,7 @@ if [[ "${_build_ufsda}" == "YES" ]]; then
    fi
 fi
 if [[ "${_build_gsi}" == "YES" ]]; then
-   build_jobs["gsi_enkf"]=8
+   build_jobs["gsi_enkf"]=2
    build_opts["gsi_enkf"]="${_verbose_opt} ${_build_debug}"
 fi
 if [[ "${_build_gsi}" == "YES" || "${_build_ufsda}" == "YES" ]] ; then
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 92cc1d50b1..528c60e5da 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -162,15 +162,22 @@ cd "${HOMEgfs}/parm/ufs" || exit 1
 ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/noahmptable.tbl" .
 
 cd "${HOMEgfs}/parm/post" || exit 1
-for file in postxconfig-NT-GEFS-F00.txt postxconfig-NT-GEFS.txt postxconfig-NT-GEFS-WAFS.txt \
-    postxconfig-NT-GEFS-F00-aerosol.txt postxconfig-NT-GEFS-aerosol.txt \
-    postxconfig-NT-GFS-ANL.txt postxconfig-NT-GFS-F00.txt postxconfig-NT-GFS-FLUX-F00.txt \
-    postxconfig-NT-GFS.txt postxconfig-NT-GFS-FLUX.txt postxconfig-NT-GFS-GOES.txt \
-    postxconfig-NT-GFS-F00-TWO.txt postxconfig-NT-GFS-TWO.txt \
-    params_grib2_tbl_new post_tag_gfs128 post_tag_gfs65 nam_micro_lookup.dat
+for file in params_grib2_tbl_new nam_micro_lookup.dat
 do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${file}" .
 done
+for file in postxconfig-NT-gfs-anl.txt postxconfig-NT-gfs-f00.txt postxconfig-NT-gfs-flux-f00.txt \
+    postxconfig-NT-gfs.txt postxconfig-NT-gfs-flux.txt postxconfig-NT-gfs-goes.txt \
+    postxconfig-NT-gfs-f00-two.txt postxconfig-NT-gfs-two.txt \
+    post_tag_gfs128 post_tag_gfs65
+do
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/gfs/${file}" .
+done
+for file in postxconfig-NT-gefs-f00.txt postxconfig-NT-gefs.txt postxconfig-NT-gefs-wafs.txt \
+    postxconfig-NT-gefs-f00-aerosol.txt postxconfig-NT-gefs-aerosol.txt
+do
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/gefs/${file}" .
+done
 for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \
     optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \
     optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat
@@ -202,7 +209,7 @@ done
 
 # Link these templates from ufs-weather-model
 cd "${HOMEgfs}/parm/ufs" || exit 1
-declare -a ufs_templates=("model_configure.IN" "model_configure_nest.IN"\
+declare -a ufs_templates=("model_configure.IN" "input_global_nest.nml.IN"\
                           "MOM_input_025.IN" "MOM_input_050.IN" "MOM_input_100.IN" "MOM_input_500.IN" \
                           "MOM6_data_table.IN" \
                           "ice_in.IN" \
diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd
index ee4f19a0a6..fcc9f8461d 160000
--- a/sorc/ufs_model.fd
+++ b/sorc/ufs_model.fd
@@ -1 +1 @@
-Subproject commit ee4f19a0a630fc2245a313bfe20302b5a6b555aa
+Subproject commit fcc9f8461db5eafbfd1f080da61ea79156ca0145
diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh
index 6255b95175..426c6ca91b 100755
--- a/ush/forecast_predet.sh
+++ b/ush/forecast_predet.sh
@@ -538,8 +538,8 @@ FV3_predet(){
   # Inline UPP fix files
   if [[ "${WRITE_DOPOST:-}" == ".true." ]]; then
     ${NCP} "${PARMgfs}/post/post_tag_gfs${LEVS}"                              "${DATA}/itag"
-    ${NCP} "${FLTFILEGFS:-${PARMgfs}/post/postxconfig-NT-GFS-TWO.txt}"        "${DATA}/postxconfig-NT.txt"
-    ${NCP} "${FLTFILEGFSF00:-${PARMgfs}/post/postxconfig-NT-GFS-F00-TWO.txt}" "${DATA}/postxconfig-NT_FH00.txt"
+    ${NCP} "${FLTFILEGFS:-${PARMgfs}/post/postxconfig-NT-gfs-two.txt}"        "${DATA}/postxconfig-NT.txt"
+    ${NCP} "${FLTFILEGFSF00:-${PARMgfs}/post/postxconfig-NT-gfs-f00-two.txt}" "${DATA}/postxconfig-NT_FH00.txt"
     ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}"             "${DATA}/params_grib2_tbl_new"
   fi
 
diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh
index 7e8e065d26..63e6fdd436 100755
--- a/ush/parsing_model_configure_FV3.sh
+++ b/ush/parsing_model_configure_FV3.sh
@@ -55,7 +55,7 @@ local IAU_OFFSET=${IAU_OFFSET:-0}
 if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
   local NEST_IMO=${npx_nest}
   local NEST_JMO=${npy_nest}
-  template="${PARMgfs}/ufs/model_configure_nest.IN"
+  template="${PARMgfs}/ufs/input_global_nest.nml.IN"
 else
   template="${PARMgfs}/ufs/model_configure.IN"
 fi

From 9b270f9f7d576b6a8b2b5a6841c18dcf083b8301 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 29 Aug 2024 10:41:39 -0500
Subject: [PATCH 02/21] Adjust walltime for eobs, rename walltime variables

---
 parm/config/gfs/config.resources.ORION | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/parm/config/gfs/config.resources.ORION b/parm/config/gfs/config.resources.ORION
index e3e81b0182..6b42d780d4 100644
--- a/parm/config/gfs/config.resources.ORION
+++ b/parm/config/gfs/config.resources.ORION
@@ -9,8 +9,13 @@ case ${step} in
     # Remove this block once GSI issue is resolved
     # https://github.com/NOAA-EMC/GSI/pull/764
     # https://github.com/JCSDA/spack-stack/issues/1166
-    export wtime_anal_gdas="02:40:00"
-    export wtime_anal_gfs="02:00:00"
+    export walltime_gdas="02:40:00"
+    export walltime_gfs="02:00:00"
+  ;;
+  "eobs")
+    # TODO:
+    # Remove this block once the GSI issue is resolved.
+    export walltime="00:45:00"
   ;;
   *)
   ;;

From 878dc4a6fec5d50285f8c2dd919a3af050c6740b Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 29 Aug 2024 13:28:14 -0500
Subject: [PATCH 03/21] Link gfs and gefs upp parm directories instead of files

---
 .gitignore             | 41 +++--------------------------------------
 sorc/link_workflow.sh  | 12 ++----------
 ush/forecast_predet.sh |  8 ++++----
 3 files changed, 9 insertions(+), 52 deletions(-)

diff --git a/.gitignore b/.gitignore
index f732eeb19e..4868d796c0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,11 +53,8 @@ parm/gdas/soca
 parm/gdas/jcb-gdas
 parm/gdas/jcb-algorithms
 parm/monitor
-parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
 parm/post/optics_luts_DUST.dat
-parm/post/gtg.config.gfs
-parm/post/gtg_imprintings.txt
 parm/post/optics_luts_DUST_nasa.dat
 parm/post/optics_luts_NITR_nasa.dat
 parm/post/optics_luts_SALT.dat
@@ -71,46 +68,14 @@ parm/post/optics_luts_WASO_nasa.dat
 parm/post/params_grib2_tbl_new
 parm/post/post_tag_gfs128
 parm/post/post_tag_gfs65
-parm/post/postcntrl_gefs.xml
-parm/post/postcntrl_gefs_aerosol.xml
-parm/post/postcntrl_gefs_anl.xml
-parm/post/postcntrl_gefs_chem.xml
-parm/post/postcntrl_gefs_f00.xml
-parm/post/postcntrl_gfs.xml
-parm/post/postcntrl_gfs_anl.xml
-parm/post/postcntrl_gfs_f00.xml
-parm/post/postcntrl_gfs_f00_two.xml
-parm/post/postcntrl_gfs_flux.xml
-parm/post/postcntrl_gfs_flux_f00.xml
-parm/post/postcntrl_gfs_goes.xml
-parm/post/postcntrl_gfs_goes.xml-new
-parm/post/postcntrl_gfs_two.xml
-parm/post/postcntrl_gfs_wafs.xml
-parm/post/postcntrl_gfs_wafs_anl.xml
-parm/post/postxconfig-NT-gefs-anl.txt
-parm/post/postxconfig-NT-gefs-f00.txt
-parm/post/postxconfig-NT-gefs-f00-aerosol.txt
-parm/post/postxconfig-NT-gefs-wafs.txt
-parm/post/postxconfig-NT-gefs-aerosol.txt
-parm/post/postxconfig-NT-gefs.txt
-parm/post/postxconfig-NT-gfs-anl.txt
-parm/post/postxconfig-NT-gfs-f00-two.txt
-parm/post/postxconfig-NT-gfs-f00.txt
-parm/post/postxconfig-NT-gfs-flux-f00.txt
-parm/post/postxconfig-NT-gfs-flux.txt
-parm/post/postxconfig-NT-gfs-goes.txt
-parm/post/postxconfig-NT-gfs-two.txt
-parm/post/postxconfig-NT-gfs-wafs-anl.txt
-parm/post/postxconfig-NT-gfs-wafs.txt
-parm/post/postxconfig-NT-gfs.txt
-parm/post/postxconfig-NT-gefs-aerosol.txt
-parm/post/postxconfig-NT-gefs-chem.txt
+parm/post/gfs
+parm/post/gefs
 parm/post/ocean.csv
 parm/post/ice.csv
 parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
 parm/ufs/model_configure.IN
-parm/ufs/model_configure_nest.IN
+parm/ufs/input_global_nest.nml.IN
 parm/ufs/MOM_input_*.IN
 parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 528c60e5da..7521e23f16 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -166,17 +166,9 @@ for file in params_grib2_tbl_new nam_micro_lookup.dat
 do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${file}" .
 done
-for file in postxconfig-NT-gfs-anl.txt postxconfig-NT-gfs-f00.txt postxconfig-NT-gfs-flux-f00.txt \
-    postxconfig-NT-gfs.txt postxconfig-NT-gfs-flux.txt postxconfig-NT-gfs-goes.txt \
-    postxconfig-NT-gfs-f00-two.txt postxconfig-NT-gfs-two.txt \
-    post_tag_gfs128 post_tag_gfs65
+for dir in gfs gefs
 do
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/gfs/${file}" .
-done
-for file in postxconfig-NT-gefs-f00.txt postxconfig-NT-gefs.txt postxconfig-NT-gefs-wafs.txt \
-    postxconfig-NT-gefs-f00-aerosol.txt postxconfig-NT-gefs-aerosol.txt
-do
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/gefs/${file}" .
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${dir}" .
 done
 for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \
     optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \
diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh
index 426c6ca91b..10581fc84a 100755
--- a/ush/forecast_predet.sh
+++ b/ush/forecast_predet.sh
@@ -537,10 +537,10 @@ FV3_predet(){
 
   # Inline UPP fix files
   if [[ "${WRITE_DOPOST:-}" == ".true." ]]; then
-    ${NCP} "${PARMgfs}/post/post_tag_gfs${LEVS}"                              "${DATA}/itag"
-    ${NCP} "${FLTFILEGFS:-${PARMgfs}/post/postxconfig-NT-gfs-two.txt}"        "${DATA}/postxconfig-NT.txt"
-    ${NCP} "${FLTFILEGFSF00:-${PARMgfs}/post/postxconfig-NT-gfs-f00-two.txt}" "${DATA}/postxconfig-NT_FH00.txt"
-    ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}"             "${DATA}/params_grib2_tbl_new"
+    ${NCP} "${PARMgfs}/post/post_tag_gfs${LEVS}"                                  "${DATA}/itag"
+    ${NCP} "${FLTFILEGFS:-${PARMgfs}/post/gfs/postxconfig-NT-gfs-two.txt}"        "${DATA}/postxconfig-NT.txt"
+    ${NCP} "${FLTFILEGFSF00:-${PARMgfs}/post/gfs/postxconfig-NT-gfs-f00-two.txt}" "${DATA}/postxconfig-NT_FH00.txt"
+    ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}"                 "${DATA}/params_grib2_tbl_new"
   fi
 
 }

From 210acb9012a63c04a60f0a5e829949c2c4b064c8 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 29 Aug 2024 13:28:57 -0500
Subject: [PATCH 04/21] Load more modules for the UPP

---
 jobs/rocoto/upp.sh           | 20 ++++++++++++++++++++
 parm/config/gefs/config.base |  4 ++--
 ush/python/pygfs/__init__.py | 15 ++++++++++++---
 versions/run.spack.ver       |  1 +
 4 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/jobs/rocoto/upp.sh b/jobs/rocoto/upp.sh
index 9daf128ca2..e2604db02a 100755
--- a/jobs/rocoto/upp.sh
+++ b/jobs/rocoto/upp.sh
@@ -37,9 +37,29 @@ if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
 
 else
   # For spack-stack 1.6.0, the UPP has its own environment (upp-addon-dev)
+  source "${HOMEgfs}/ush/module-setup.sh"
   module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles"
   module load "${MACHINE_ID}"
+  source "${HOMEgfs}/versions/run.spack.ver"
+  module load "prod_util/${prod_util_ver}"
+  module load "grib-util/${grib_util_ver}"
+  # The numpy version in the UPP environment differs from the gsi-addon
+  module load "py-numpy/1.22.3"
+  module load "py-pyyaml/${py_pyyaml_ver}"
+  module load "py-netcdf4/${py_netcdf4_ver}"
+  module load "py-f90nml/${py_f90nml_ver}"
+
+  # The jinja2 version in the UPP environment differs from the gsi-addon
+  module load "py-jinja2/3.0.3"
+
+  # xarray is not in this environment...
+  # module load "py-xarray/${py_xarray_ver}"
+
+  wxflowPATH="${HOMEgfs}/ush/python"
+  PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush:${wxflowPATH}"
+  export PYTHONPATH
   #. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+
   status=$?
   if (( status != 0 )); then exit "${status}"; fi
 fi
diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
index 6b6cafa04a..700504c2bf 100644
--- a/parm/config/gefs/config.base
+++ b/parm/config/gefs/config.base
@@ -269,8 +269,8 @@ export WRITE_DOPOST=".true." # WRITE_DOPOST=true, use inline POST
 export WRITE_NSFLIP=".true."
 
 # Override normal post flat files for GEFS
-export FLTFILEGFS="${PARMgfs}/post/postxconfig-NT-gefs.txt"
-export FLTFILEGFSF00="${PARMgfs}/post/postxconfig-NT-gefs-f00.txt"
+export FLTFILEGFS="${PARMgfs}/post/gefs/postxconfig-NT-gefs.txt"
+export FLTFILEGFSF00="${PARMgfs}/post/gefs/postxconfig-NT-gefs-f00.txt"
 
 # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL
 export imp_physics=8
diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py
index b87bba0c2c..ecdb738e4b 100644
--- a/ush/python/pygfs/__init__.py
+++ b/ush/python/pygfs/__init__.py
@@ -6,13 +6,22 @@
 from .task.aero_analysis import AerosolAnalysis
 from .task.atm_analysis import AtmAnalysis
 from .task.atmens_analysis import AtmEnsAnalysis
-from .task.marine_bmat import MarineBMat
 from .task.snow_analysis import SnowAnalysis
 from .task.snowens_analysis import SnowEnsAnalysis
 from .task.upp import UPP
-from .task.oceanice_products import OceanIceProducts
 from .task.gfs_forecast import GFSForecast
-from .utils import marine_da_utils
+
+# TODO Move this out of a try block once moving to spack-stack 1.8.0
+try:
+    # For spack-stack 1.6.0, the UPP requires its own environment
+    # This environment does not include xarray, which is required
+    # by marine tasks
+    from .task.marine_bmat import MarineBMat
+    from .task.oceanice_products import OceanIceProducts
+    from .utils import marine_da_utils
+
+except ModuleNotFoundError:
+    print("WARNING: not all required python modules were found!")
 
 __docformat__ = "restructuredtext"
 __version__ = "0.1.0"
diff --git a/versions/run.spack.ver b/versions/run.spack.ver
index 9aa5460c80..4a17d0ef27 100644
--- a/versions/run.spack.ver
+++ b/versions/run.spack.ver
@@ -23,6 +23,7 @@ export py_jinja2_ver=3.1.2
 export py_pandas_ver=1.5.3
 export py_python_dateutil_ver=2.8.2
 export py_f90nml_ver=1.4.3
+export py_numpy_ver=1.23.4
 
 export met_ver=9.1.3
 export metplus_ver=3.1.1

From 42e8d3a01f9e8710cfd0fc3794592d0cbb5acea3 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 29 Aug 2024 14:10:21 -0500
Subject: [PATCH 05/21] Update path to post_tag_gfs*

---
 ush/forecast_predet.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh
index 10581fc84a..ce8aa1c416 100755
--- a/ush/forecast_predet.sh
+++ b/ush/forecast_predet.sh
@@ -537,7 +537,7 @@ FV3_predet(){
 
   # Inline UPP fix files
   if [[ "${WRITE_DOPOST:-}" == ".true." ]]; then
-    ${NCP} "${PARMgfs}/post/post_tag_gfs${LEVS}"                                  "${DATA}/itag"
+    ${NCP} "${PARMgfs}/post/gfs/post_tag_gfs${LEVS}"                              "${DATA}/itag"
     ${NCP} "${FLTFILEGFS:-${PARMgfs}/post/gfs/postxconfig-NT-gfs-two.txt}"        "${DATA}/postxconfig-NT.txt"
     ${NCP} "${FLTFILEGFSF00:-${PARMgfs}/post/gfs/postxconfig-NT-gfs-f00-two.txt}" "${DATA}/postxconfig-NT_FH00.txt"
     ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}"                 "${DATA}/params_grib2_tbl_new"

From 4c21543a1c54d95b0c2e98f013e0d8740748e826 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 29 Aug 2024 14:24:03 -0500
Subject: [PATCH 06/21] Update UPP parm paths

---
 parm/post/upp.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/parm/post/upp.yaml b/parm/post/upp.yaml
index aa6768f8f6..efc8ba2994 100644
--- a/parm/post/upp.yaml
+++ b/parm/post/upp.yaml
@@ -18,7 +18,7 @@ analysis:
     rdaod: True
   data_in:
     copy:
-      - ["{{ PARMgfs }}/post/postxconfig-NT-gfs-anl.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-anl.txt", "{{ DATA }}/postxconfig-NT.txt"]
       - ["{{ COM_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmanl.nc", "{{ DATA }}/{{ atmos_filename }}"]
       - ["{{ COM_ATMOS_ANALYSIS }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcanl.nc", "{{ DATA }}/{{ flux_filename }}"]
   data_out:
@@ -32,9 +32,9 @@ forecast:
   data_in:
     copy:
       {% if forecast_hour == 0 %}
-      - ["{{ PARMgfs }}/post/postxconfig-NT-gfs-f00-two.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-f00-two.txt", "{{ DATA }}/postxconfig-NT.txt"]
       {% else %}
-      - ["{{ PARMgfs }}/post/postxconfig-NT-gfs-two.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-two.txt", "{{ DATA }}/postxconfig-NT.txt"]
       {% endif %}
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"]
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"]
@@ -81,7 +81,7 @@ goes:
       {% endfor %}
       - ["{{ 'CRTM_FIX' | getenv }}/AerosolCoeff.bin", "{{ DATA }}/"]
       - ["{{ 'CRTM_FIX' | getenv }}/CloudCoeff.bin", "{{ DATA }}/"]
-      - ["{{ PARMgfs }}/post/postxconfig-NT-gfs-goes.txt", "{{ DATA }}/postxconfig-NT.txt"]
+      - ["{{ PARMgfs }}/post/gfs/postxconfig-NT-gfs-goes.txt", "{{ DATA }}/postxconfig-NT.txt"]
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.atmf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ atmos_filename }}"]
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"]
   data_out:

From 6a802caf4db0c041ce6c60360d6212e3ee1438ac Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 29 Aug 2024 14:59:01 -0500
Subject: [PATCH 07/21] Revert UPP modules

---
 jobs/rocoto/upp.sh           | 24 +-----------------------
 ush/python/pygfs/__init__.py | 15 +++------------
 2 files changed, 4 insertions(+), 35 deletions(-)

diff --git a/jobs/rocoto/upp.sh b/jobs/rocoto/upp.sh
index e2604db02a..df4a80b377 100755
--- a/jobs/rocoto/upp.sh
+++ b/jobs/rocoto/upp.sh
@@ -36,29 +36,7 @@ if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
   export PYTHONPATH
 
 else
-  # For spack-stack 1.6.0, the UPP has its own environment (upp-addon-dev)
-  source "${HOMEgfs}/ush/module-setup.sh"
-  module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles"
-  module load "${MACHINE_ID}"
-  source "${HOMEgfs}/versions/run.spack.ver"
-  module load "prod_util/${prod_util_ver}"
-  module load "grib-util/${grib_util_ver}"
-  # The numpy version in the UPP environment differs from the gsi-addon
-  module load "py-numpy/1.22.3"
-  module load "py-pyyaml/${py_pyyaml_ver}"
-  module load "py-netcdf4/${py_netcdf4_ver}"
-  module load "py-f90nml/${py_f90nml_ver}"
-
-  # The jinja2 version in the UPP environment differs from the gsi-addon
-  module load "py-jinja2/3.0.3"
-
-  # xarray is not in this environment...
-  # module load "py-xarray/${py_xarray_ver}"
-
-  wxflowPATH="${HOMEgfs}/ush/python"
-  PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush:${wxflowPATH}"
-  export PYTHONPATH
-  #. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+  . "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
 
   status=$?
   if (( status != 0 )); then exit "${status}"; fi
diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py
index ecdb738e4b..b87bba0c2c 100644
--- a/ush/python/pygfs/__init__.py
+++ b/ush/python/pygfs/__init__.py
@@ -6,22 +6,13 @@
 from .task.aero_analysis import AerosolAnalysis
 from .task.atm_analysis import AtmAnalysis
 from .task.atmens_analysis import AtmEnsAnalysis
+from .task.marine_bmat import MarineBMat
 from .task.snow_analysis import SnowAnalysis
 from .task.snowens_analysis import SnowEnsAnalysis
 from .task.upp import UPP
+from .task.oceanice_products import OceanIceProducts
 from .task.gfs_forecast import GFSForecast
-
-# TODO Move this out of a try block once moving to spack-stack 1.8.0
-try:
-    # For spack-stack 1.6.0, the UPP requires its own environment
-    # This environment does not include xarray, which is required
-    # by marine tasks
-    from .task.marine_bmat import MarineBMat
-    from .task.oceanice_products import OceanIceProducts
-    from .utils import marine_da_utils
-
-except ModuleNotFoundError:
-    print("WARNING: not all required python modules were found!")
+from .utils import marine_da_utils
 
 __docformat__ = "restructuredtext"
 __version__ = "0.1.0"

From 7243745fadcb98fe853bbef09dd20a2c6b302dd8 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 29 Aug 2024 15:03:42 -0500
Subject: [PATCH 08/21] Remove extra whitespace

---
 jobs/rocoto/upp.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/jobs/rocoto/upp.sh b/jobs/rocoto/upp.sh
index df4a80b377..c3f128ab02 100755
--- a/jobs/rocoto/upp.sh
+++ b/jobs/rocoto/upp.sh
@@ -37,7 +37,6 @@ if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
 
 else
   . "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
-
   status=$?
   if (( status != 0 )); then exit "${status}"; fi
 fi

From 04c4835c5cc631a5904944e302e3ce5dc8076445 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Fri, 30 Aug 2024 08:06:58 -0500
Subject: [PATCH 09/21] Link post_tag files to parm/post

---
 sorc/link_workflow.sh  | 5 +++++
 ush/forecast_predet.sh | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 7521e23f16..a5aa671eaa 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -170,6 +170,11 @@ for dir in gfs gefs
 do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${dir}" .
 done
+# Link the post_tag files into parm/post from upp.fd/parm/gfs
+for file in post_tag_gfs128 post_tag_gfs65
+do
+  ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/gfs/${file}" .
+done
 for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \
     optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \
     optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat
diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh
index ce8aa1c416..10581fc84a 100755
--- a/ush/forecast_predet.sh
+++ b/ush/forecast_predet.sh
@@ -537,7 +537,7 @@ FV3_predet(){
 
   # Inline UPP fix files
   if [[ "${WRITE_DOPOST:-}" == ".true." ]]; then
-    ${NCP} "${PARMgfs}/post/gfs/post_tag_gfs${LEVS}"                              "${DATA}/itag"
+    ${NCP} "${PARMgfs}/post/post_tag_gfs${LEVS}"                                  "${DATA}/itag"
     ${NCP} "${FLTFILEGFS:-${PARMgfs}/post/gfs/postxconfig-NT-gfs-two.txt}"        "${DATA}/postxconfig-NT.txt"
     ${NCP} "${FLTFILEGFSF00:-${PARMgfs}/post/gfs/postxconfig-NT-gfs-f00-two.txt}" "${DATA}/postxconfig-NT_FH00.txt"
     ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}"                 "${DATA}/params_grib2_tbl_new"

From ee39fc9040f211ef16a0e165a23f4bbaf7083c6d Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Wed, 4 Sep 2024 13:29:28 +0000
Subject: [PATCH 10/21] Update gfsgoes file name convention

---
 parm/post/upp.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/parm/post/upp.yaml b/parm/post/upp.yaml
index efc8ba2994..41dbb7defb 100644
--- a/parm/post/upp.yaml
+++ b/parm/post/upp.yaml
@@ -86,5 +86,5 @@ goes:
       - ["{{ COM_ATMOS_HISTORY }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.sfcf{{ '%03d' % forecast_hour }}.nc", "{{ DATA }}/{{ flux_filename }}"]
   data_out:
     copy:
-      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2f{{ '%03d' % forecast_hour }}"]
-      - ["{{ DATA }}/GFSPRS.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2if{{ '%03d' % forecast_hour }}"]
+      - ["{{ DATA }}/GFSGOES.GrbF{{ '%02d' % forecast_hour }}", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2f{{ '%03d' % forecast_hour }}"]
+      - ["{{ DATA }}/GFSGOES.GrbF{{ '%02d' % forecast_hour }}.idx", "{{ COM_ATMOS_MASTER }}/{{ RUN }}.t{{ current_cycle | strftime('%H') }}z.special.grb2if{{ '%03d' % forecast_hour }}"]

From 10961a825965c7072e2b9587b8012acc1bf56236 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Wed, 4 Sep 2024 13:58:48 +0000
Subject: [PATCH 11/21] Remove 65-level upp itag files.

---
 .gitignore            | 1 -
 sorc/link_workflow.sh | 7 ++-----
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 4868d796c0..83706de085 100644
--- a/.gitignore
+++ b/.gitignore
@@ -67,7 +67,6 @@ parm/post/optics_luts_WASO.dat
 parm/post/optics_luts_WASO_nasa.dat
 parm/post/params_grib2_tbl_new
 parm/post/post_tag_gfs128
-parm/post/post_tag_gfs65
 parm/post/gfs
 parm/post/gefs
 parm/post/ocean.csv
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index fd5b5603dd..df73cf647e 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -170,11 +170,8 @@ for dir in gfs gefs
 do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${dir}" .
 done
-# Link the post_tag files into parm/post from upp.fd/parm/gfs
-for file in post_tag_gfs128 post_tag_gfs65
-do
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/gfs/${file}" .
-done
+# Link the 128-level post_tag file into parm/post from upp.fd/parm/gfs
+${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/gfs/post_tag_gfs128" .
 for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \
     optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \
     optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat

From 061a854bc28782d7d247299464ecfdfd2320ac4f Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 4 Sep 2024 11:07:24 -0400
Subject: [PATCH 12/21] Update sorc/link_workflow.sh

---
 sorc/link_workflow.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index df73cf647e..d774c888b8 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -170,8 +170,6 @@ for dir in gfs gefs
 do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${dir}" .
 done
-# Link the 128-level post_tag file into parm/post from upp.fd/parm/gfs
-${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/gfs/post_tag_gfs128" .
 for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \
     optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \
     optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat

From 9e3cf00ffe56236ba4b94eaba429c47abedd5647 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Wed, 4 Sep 2024 11:08:04 -0400
Subject: [PATCH 13/21] Update .gitignore

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 83706de085..9f43f1be04 100644
--- a/.gitignore
+++ b/.gitignore
@@ -66,7 +66,6 @@ parm/post/optics_luts_SUSO_nasa.dat
 parm/post/optics_luts_WASO.dat
 parm/post/optics_luts_WASO_nasa.dat
 parm/post/params_grib2_tbl_new
-parm/post/post_tag_gfs128
 parm/post/gfs
 parm/post/gefs
 parm/post/ocean.csv

From cfee7148aae8ff423921ee72fe9cc5e054aed4e3 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Wed, 4 Sep 2024 10:26:07 -0500
Subject: [PATCH 14/21] Wait to run wave post until after forecast completes

The new forecast segments capability introduced an issue where the
wave post job could time out waiting for the output of later segments
before those segments were complete. For now, the dependency is changed
to wait until the forecast is complete until wave post can be refactored.
---
 workflow/rocoto/gefs_tasks.py | 9 +++------
 workflow/rocoto/gfs_tasks.py  | 9 +++------
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/workflow/rocoto/gefs_tasks.py b/workflow/rocoto/gefs_tasks.py
index 5d706071b6..d1ece0f096 100644
--- a/workflow/rocoto/gefs_tasks.py
+++ b/workflow/rocoto/gefs_tasks.py
@@ -324,12 +324,9 @@ def atmos_ensstat(self):
 
     def wavepostsbs(self):
         deps = []
-        for wave_grid in self._configs['wavepostsbs']['waveGRD'].split():
-            wave_hist_path = self._template_to_rocoto_cycstring(self._base["COM_WAVE_HISTORY_TMPL"], {'MEMDIR': 'mem#member#'})
-            data = f'{wave_hist_path}/gefswave.out_grd.{wave_grid}.@Y@m@d.@H0000'
-            dep_dict = {'type': 'data', 'data': data}
-            deps.append(rocoto.add_dependency(dep_dict))
-        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+        dep_dict = {'type': 'metatask', 'name': f'fcst_mem#member#'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
 
         wave_post_envars = self.envars.copy()
         postenvar_dict = {'ENSMEM': '#member#',
diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py
index 76db3db88e..89da933d00 100644
--- a/workflow/rocoto/gfs_tasks.py
+++ b/workflow/rocoto/gfs_tasks.py
@@ -1188,12 +1188,9 @@ def _atmosoceaniceprod(self, component: str):
 
     def wavepostsbs(self):
         deps = []
-        for wave_grid in self._configs['wavepostsbs']['waveGRD'].split():
-            wave_hist_path = self._template_to_rocoto_cycstring(self._base["COM_WAVE_HISTORY_TMPL"])
-            data = f'{wave_hist_path}/{self.run}wave.out_grd.{wave_grid}.@Y@m@d.@H0000'
-            dep_dict = {'type': 'data', 'data': data}
-            deps.append(rocoto.add_dependency(dep_dict))
-        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+        dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst'}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
 
         resources = self.get_resource('wavepostsbs')
         task_name = f'{self.run}wavepostsbs'

From 7096a050530dedbdd45bda18d59ee1a3f3dbc89b Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 5 Sep 2024 17:33:32 +0000
Subject: [PATCH 15/21] Link in the 128-level itag file

---
 .gitignore            | 1 +
 sorc/link_workflow.sh | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index 9f43f1be04..83706de085 100644
--- a/.gitignore
+++ b/.gitignore
@@ -66,6 +66,7 @@ parm/post/optics_luts_SUSO_nasa.dat
 parm/post/optics_luts_WASO.dat
 parm/post/optics_luts_WASO_nasa.dat
 parm/post/params_grib2_tbl_new
+parm/post/post_tag_gfs128
 parm/post/gfs
 parm/post/gefs
 parm/post/ocean.csv
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index d774c888b8..df73cf647e 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -170,6 +170,8 @@ for dir in gfs gefs
 do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${dir}" .
 done
+# Link the 128-level post_tag file into parm/post from upp.fd/parm/gfs
+${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/gfs/post_tag_gfs128" .
 for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \
     optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \
     optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat

From 386ce380dd5c875155407d1c2f8841716c6af049 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 5 Sep 2024 14:24:02 -0400
Subject: [PATCH 16/21] fixes for UPP and compression in ufswm

---
 parm/config/gefs/config.base       |  4 ----
 parm/config/gefs/config.ufs        | 21 ++++++++++++++++++++-
 parm/config/gfs/config.ufs         | 19 +++++++++++++++++++
 sorc/link_workflow.sh              | 13 ++++++-------
 ush/forecast_predet.sh             | 14 ++++++++++----
 ush/parsing_model_configure_FV3.sh |  6 +++---
 6 files changed, 58 insertions(+), 19 deletions(-)

diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
index da8433c16d..47474fb108 100644
--- a/parm/config/gefs/config.base
+++ b/parm/config/gefs/config.base
@@ -268,10 +268,6 @@ export OUTPUT_GRID="gaussian_grid"
 export WRITE_DOPOST=".true." # WRITE_DOPOST=true, use inline POST
 export WRITE_NSFLIP=".true."
 
-# Override normal post flat files for GEFS
-export FLTFILEGFS="${PARMgfs}/post/gefs/postxconfig-NT-gefs.txt"
-export FLTFILEGFSF00="${PARMgfs}/post/gefs/postxconfig-NT-gefs-f00.txt"
-
 # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL
 export imp_physics=8
 
diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs
index 584e4769a8..bfc11e3c5a 100644
--- a/parm/config/gefs/config.ufs
+++ b/parm/config/gefs/config.ufs
@@ -254,6 +254,25 @@ export ntasks_fv3_gfs
 export ntasks_quilt
 export ntasks_quilt_gfs
 
+# Determine whether to use compression in the write grid component based on resolution
+case ${fv3_res} in
+  "C48" | "C96" | "C192" | "C384")
+    zstandard_level=0
+    ideflate=0
+    quantize_nsd=0
+    ;;
+  "C768" | "C1152" | "C3072")
+    zstandard_level=0
+    ideflate=1
+    quantize_nsd=5
+    ;;
+  *)
+    echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}"
+    exit 15
+    ;;
+esac
+export zstandard_level ideflate quantize_nsd
+
 # Determine whether to use parallel NetCDF based on resolution
 case ${fv3_res} in
   "C48" | "C96" | "C192" | "C384")
@@ -353,7 +372,7 @@ if [[ "${skip_mom6}" == "false" ]]; then
       if [[ ${RUN} == "gfs" || "${RUN}" == "gefs" ]]; then
         MOM6_DIAG_COORD_DEF_Z_FILE="interpolate_zgrid_40L.nc"
         MOM6_DIAG_MISVAL="-1e34"
-      else            
+      else
         MOM6_DIAG_COORD_DEF_Z_FILE="oceanda_zgrid_75L.nc"
         MOM6_DIAG_MISVAL="0.0"
       fi
diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs
index 148793927f..b27845aec7 100644
--- a/parm/config/gfs/config.ufs
+++ b/parm/config/gfs/config.ufs
@@ -356,6 +356,25 @@ export ntasks_fv3_gfs
 export ntasks_quilt_gdas
 export ntasks_quilt_gfs
 
+# Determine whether to use compression in the write grid component based on resolution
+case ${fv3_res} in
+  "C48" | "C96" | "C192" | "C384")
+    zstandard_level=0
+    ideflate=0
+    quantize_nsd=0
+    ;;
+  "C768" | "C1152" | "C3072")
+    zstandard_level=0
+    ideflate=1
+    quantize_nsd=5
+    ;;
+  *)
+    echo "FATAL ERROR: Unrecognized FV3 resolution ${fv3_res}"
+    exit 15
+    ;;
+esac
+export zstandard_level ideflate quantize_nsd
+
 # Determine whether to use parallel NetCDF based on resolution
 case ${fv3_res} in
   "C48" | "C96" | "C192" | "C384")
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index d774c888b8..1b88bdd18c 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -170,12 +170,6 @@ for dir in gfs gefs
 do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${dir}" .
 done
-for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \
-    optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \
-    optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat
-do
-  ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/fix/chem/${file}" .
-done
 for file in ice.csv ocean.csv ocnicepost.nml.jinja2
 do
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/gfs_utils.fd/parm/ocnicepost/${file}" .
@@ -218,7 +212,12 @@ declare -a ufs_templates=("model_configure.IN" "input_global_nest.nml.IN"\
                           "ufs.configure.s2swa.IN" \
                           "ufs.configure.s2swa_esmf.IN" \
                           "ufs.configure.leapfrog_atm_wav.IN" \
-                          "ufs.configure.leapfrog_atm_wav_esmf.IN" )
+                          "ufs.configure.leapfrog_atm_wav_esmf.IN" \
+                          "post_itag_gfs" \  # TODO: Need a GEFS version of this in the UFS-weather-model
+                          "postxconfig-NT-gfs.txt" \  # TODO: Need a GEFS version of this in the UFS-weather-model
+                          "postxconfig-NT-gfs_FH00.txt")  # TODO: Need a GEFS version of this in the UFS-weather-model
+                          # TODO: The above postxconfig files in the UFSWM are not the same as the ones in UPP
+                          # See forecast_predet.sh where the UPP versions are used.  They will need to be replaced with these.
 for file in "${ufs_templates[@]}"; do
   [[ -s "${file}" ]] && rm -f "${file}"
   ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/${file}" .
diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh
index 10581fc84a..dda0551164 100755
--- a/ush/forecast_predet.sh
+++ b/ush/forecast_predet.sh
@@ -537,10 +537,16 @@ FV3_predet(){
 
   # Inline UPP fix files
   if [[ "${WRITE_DOPOST:-}" == ".true." ]]; then
-    ${NCP} "${PARMgfs}/post/post_tag_gfs${LEVS}"                                  "${DATA}/itag"
-    ${NCP} "${FLTFILEGFS:-${PARMgfs}/post/gfs/postxconfig-NT-gfs-two.txt}"        "${DATA}/postxconfig-NT.txt"
-    ${NCP} "${FLTFILEGFSF00:-${PARMgfs}/post/gfs/postxconfig-NT-gfs-f00-two.txt}" "${DATA}/postxconfig-NT_FH00.txt"
-    ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}"                 "${DATA}/params_grib2_tbl_new"
+    ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}" "${DATA}/params_grib2_tbl_new"
+    ${NCP} "${PARMgfs}/ufs/post_tag_gfs"                          "${DATA}/itag"  # TODO: Need a GEFS version when available in the UFS-weather-model
+    # TODO: These should be replaced with ones from the ufs-weather-model when available there
+    if [[ "${RUN}" =~ "gdas" || "${RUN}" =~ "gfs" ]]; then  # RUN = gdas | enkfgdas | gfs | enkfgfs
+      ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-two.txt"     "${DATA}/postxconfig-NT.txt"
+-     ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-f00-two.txt" "${DATA}/postxconfig-NT_FH00.txt"
+    elif [[ "${RUN}" == "gefs" ]]; then  # RUN = gefs
+      ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs.txt"       "${DATA}/postxconfig-NT.txt"
+-     ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs-f00.txt"   "${DATA}/postxconfig-NT_FH00.txt"
+    fi
   fi
 
 }
diff --git a/ush/parsing_model_configure_FV3.sh b/ush/parsing_model_configure_FV3.sh
index 63e6fdd436..8f102fe298 100755
--- a/ush/parsing_model_configure_FV3.sh
+++ b/ush/parsing_model_configure_FV3.sh
@@ -38,9 +38,9 @@ local NUM_FILES=${NUM_FILES:-2}
 local FILENAME_BASE="'atm' 'sfc'"
 # OUTPUT_GRID
 local OUTPUT_FILE="'${OUTPUT_FILETYPE_ATM}' '${OUTPUT_FILETYPE_SFC}'"
-local ZSTANDARD_LEVEL=0
-local IDEFLATE=0  # netCDF zlib lossless compression (0-9); 0: no compression
-local QUANTIZE_NSD=${QUANTIZE_NSD:-0}  # netCDF compression
+local ZSTANDARD_LEVEL=${zstandard_level:-0}
+local IDEFLATE=${ideflate:-0}  # netCDF zlib lossless compression (0-9); 0: no compression
+local QUANTIZE_NSD=${quantize_nsd:-0}  # netCDF compression
 local ICHUNK2D=$((4*restile))
 local JCHUNK2D=$((2*restile))
 local ICHUNK3D=$((4*restile))

From 78825c81c4ec1674f51d8c7ecbbb1c8b4734b357 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 5 Sep 2024 14:29:05 -0400
Subject: [PATCH 17/21] post tag name is incorrect

---
 ush/forecast_predet.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh
index dda0551164..c089aeda58 100755
--- a/ush/forecast_predet.sh
+++ b/ush/forecast_predet.sh
@@ -538,7 +538,7 @@ FV3_predet(){
   # Inline UPP fix files
   if [[ "${WRITE_DOPOST:-}" == ".true." ]]; then
     ${NCP} "${POSTGRB2TBL:-${PARMgfs}/post/params_grib2_tbl_new}" "${DATA}/params_grib2_tbl_new"
-    ${NCP} "${PARMgfs}/ufs/post_tag_gfs"                          "${DATA}/itag"  # TODO: Need a GEFS version when available in the UFS-weather-model
+    ${NCP} "${PARMgfs}/ufs/post_itag_gfs"                         "${DATA}/itag"  # TODO: Need a GEFS version when available in the UFS-weather-model
     # TODO: These should be replaced with ones from the ufs-weather-model when available there
     if [[ "${RUN}" =~ "gdas" || "${RUN}" =~ "gfs" ]]; then  # RUN = gdas | enkfgdas | gfs | enkfgfs
       ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-two.txt"     "${DATA}/postxconfig-NT.txt"

From 22b1e2fdf0bfa43bc5f21e70fc0e5bed10a23579 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 5 Sep 2024 14:35:16 -0400
Subject: [PATCH 18/21] fix shellnorm errors

---
 ush/forecast_predet.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ush/forecast_predet.sh b/ush/forecast_predet.sh
index c089aeda58..365bebb973 100755
--- a/ush/forecast_predet.sh
+++ b/ush/forecast_predet.sh
@@ -542,10 +542,10 @@ FV3_predet(){
     # TODO: These should be replaced with ones from the ufs-weather-model when available there
     if [[ "${RUN}" =~ "gdas" || "${RUN}" =~ "gfs" ]]; then  # RUN = gdas | enkfgdas | gfs | enkfgfs
       ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-two.txt"     "${DATA}/postxconfig-NT.txt"
--     ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-f00-two.txt" "${DATA}/postxconfig-NT_FH00.txt"
+      ${NCP} "${PARMgfs}/post/gfs/postxconfig-NT-gfs-f00-two.txt" "${DATA}/postxconfig-NT_FH00.txt"
     elif [[ "${RUN}" == "gefs" ]]; then  # RUN = gefs
       ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs.txt"       "${DATA}/postxconfig-NT.txt"
--     ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs-f00.txt"   "${DATA}/postxconfig-NT_FH00.txt"
+      ${NCP} "${PARMgfs}/post/gefs/postxconfig-NT-gefs-f00.txt"   "${DATA}/postxconfig-NT_FH00.txt"
     fi
   fi
 

From 5accc45ee65aea6a27023f1c7b2eb58968d52c13 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 5 Sep 2024 14:36:33 -0400
Subject: [PATCH 19/21] Update ush/forecast_predet.sh

Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>

From 14c449c443f082bb61ec0d388200b7fe16125da4 Mon Sep 17 00:00:00 2001
From: Rahul Mahajan <aerorahul@users.noreply.github.com>
Date: Thu, 5 Sep 2024 14:39:30 -0400
Subject: [PATCH 20/21] comment cleanup

---
 sorc/link_workflow.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 1b88bdd18c..92404afc01 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -213,10 +213,11 @@ declare -a ufs_templates=("model_configure.IN" "input_global_nest.nml.IN"\
                           "ufs.configure.s2swa_esmf.IN" \
                           "ufs.configure.leapfrog_atm_wav.IN" \
                           "ufs.configure.leapfrog_atm_wav_esmf.IN" \
-                          "post_itag_gfs" \  # TODO: Need a GEFS version of this in the UFS-weather-model
-                          "postxconfig-NT-gfs.txt" \  # TODO: Need a GEFS version of this in the UFS-weather-model
-                          "postxconfig-NT-gfs_FH00.txt")  # TODO: Need a GEFS version of this in the UFS-weather-model
+                          "post_itag_gfs" \
+                          "postxconfig-NT-gfs.txt" \
+                          "postxconfig-NT-gfs_FH00.txt")
                           # TODO: The above postxconfig files in the UFSWM are not the same as the ones in UPP
+                          # TODO: GEFS postxconfig files also need to be received from UFSWM
                           # See forecast_predet.sh where the UPP versions are used.  They will need to be replaced with these.
 for file in "${ufs_templates[@]}"; do
   [[ -s "${file}" ]] && rm -f "${file}"

From 8f7bbc446c39bf92c9fb5f94a1a80243bbd37477 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Fri, 6 Sep 2024 07:25:07 -0500
Subject: [PATCH 21/21] Add GOES to the list of indexable file types.

---
 ush/python/pygfs/task/upp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/python/pygfs/task/upp.py b/ush/python/pygfs/task/upp.py
index 7e42e07c64..1b37b845f6 100644
--- a/ush/python/pygfs/task/upp.py
+++ b/ush/python/pygfs/task/upp.py
@@ -202,7 +202,7 @@ def index(cls, workdir: Union[str, os.PathLike], forecast_hour: int) -> None:
 
         template = f"GFS{{file_type}}.GrbF{forecast_hour:02d}"
 
-        for ftype in ['PRS', 'FLX']:
+        for ftype in ['PRS', 'FLX', 'GOES']:
             grbfile = template.format(file_type=ftype)
             grbfidx = f"{grbfile}.idx"