Add a yaml for snow DA testing. (NOAA-EMC#2199)

- adds a new test yaml C96_atmsnowDA.yaml for 3DVar atmosphere with GSI and Land (Snow) DA with JEDI
- moves a few yamls from platforms/ to yamls/
- adds ability to overwrite a previously created experiment as an addition to user input.
---------

Co-authored-by: Cory Martin <[email protected]>

ci/cases/pr/C48_ATM.yaml

@@ -10,4 +10,4 @@ arguments:
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312
   edate: 2021032312
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/pr/C48_S2SW.yaml

@@ -11,4 +11,4 @@ arguments:
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312
   edate: 2021032312
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -15,4 +15,4 @@ arguments:
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312
   edate: 2021032312
-  yaml: {{ HOMEgfs }}/ci/platforms/gefs_ci_defaults.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml

ci/cases/pr/C96C48_hybatmDA.yaml

@@ -16,4 +16,4 @@ arguments:
   nens: 2
   gfs_cyc: 1
   start: cold
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/pr/C96_atm3DVar.yaml

@@ -14,4 +14,4 @@ arguments:
   nens: 0
   gfs_cyc: 1
   start: cold
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/pr/C96_atmsnowDA.yaml

@@ -0,0 +1,21 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: ATM
+  resdetatmos: 96
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  idate: 2021122012
+  edate: 2021122100
+  nens: 0
+  gfs_cyc: 1
+  start: cold
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/atmsnowDA_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - orion
+  - hercules

ci/cases/weekly/C384C192_hybatmda.yaml

@@ -16,4 +16,4 @@ arguments:
   nens: 2
   gfs_cyc: 1
   start: cold
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/weekly/C384_S2SWA.yaml

@@ -11,4 +11,4 @@ arguments:
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2016070100
   edate: 2016070100
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/weekly/C384_atm3DVar.yaml

@@ -16,4 +16,4 @@ arguments:
   nens: 0
   gfs_cyc: 1
   start: cold
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/yamls/atmsnowDA_defaults_ci.yaml

@@ -0,0 +1,6 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+base:
+  DOIAU: "NO"
+  DO_JEDILANDDA: "YES"
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}

jobs/JGLOBAL_LAND_ANALYSIS

@@ -1,6 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
+export DATA=${DATA:-${DATAROOT}/${RUN}landanl_${cyc}}
 source "${HOMEgfs}/ush/jjob_header.sh" -e "landanl" -c "base landanl"
 
 ##############################################

jobs/JGLOBAL_PREP_LAND_OBS

@@ -1,6 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
+export DATA=${DATA:-${DATAROOT}/${RUN}landanl_${cyc}}
 source "${HOMEgfs}/ush/jjob_header.sh" -e "preplandobs" -c "base preplandobs"
 
 ##############################################

ush/python/pygfs/task/land_analysis.py

@@ -26,7 +26,7 @@ class LandAnalysis(Analysis):
     Class for global land analysis tasks
     """
 
-    NMEM_LANDENS = 2  # The size of the land ensemble is fixed at 2.  Does this need to be a variable?
+    NMEM_LANDENS = 2
 
     @logit(logger, name="LandAnalysis")
     def __init__(self, config):
@@ -74,7 +74,7 @@ def prepare_GTS(self) -> None:
         # create a temporary dict of all keys needed in this method
         localconf = AttrDict()
         keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV',
-                'OPREFIX', 'CASE', 'ntiles']
+                'OPREFIX', 'CASE', 'OCNRES', 'ntiles']
         for key in keys:
             localconf[key] = self.task_config[key]
 
@@ -198,7 +198,7 @@ def prepare_IMS(self) -> None:
             raise WorkflowException(f"An error occured during execution of {exe}")
 
         # Ensure the snow depth IMS file is produced by the above executable
-        input_file = f"IMSscf.{to_YMD(localconf.current_cycle)}.{localconf.CASE}.mx{localconf.OCNRES}_oro_data.nc"
+        input_file = f"IMSscf.{to_YMD(localconf.current_cycle)}.{localconf.CASE}_oro_data.nc"
         if not os.path.isfile(f"{os.path.join(localconf.DATA, input_file)}"):
             logger.exception(f"{self.task_config.CALCFIMSEXE} failed to produce {input_file}")
             raise FileNotFoundError(f"{os.path.join(localconf.DATA, input_file)}")
@@ -249,7 +249,7 @@ def initialize(self) -> None:
         # create a temporary dict of all keys needed in this method
         localconf = AttrDict()
         keys = ['DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV',
-                'OPREFIX', 'CASE', 'ntiles']
+                'OPREFIX', 'CASE', 'OCNRES', 'ntiles']
         for key in keys:
             localconf[key] = self.task_config[key]
 
@@ -301,7 +301,7 @@ def execute(self) -> None:
         localconf = AttrDict()
         keys = ['HOMEgfs', 'DATA', 'current_cycle',
                 'COM_ATMOS_RESTART_PREV', 'COM_LAND_ANALYSIS', 'APREFIX',
-                'SNOWDEPTHVAR', 'BESTDDEV', 'CASE', 'ntiles',
+                'SNOWDEPTHVAR', 'BESTDDEV', 'CASE', 'OCNRES', 'ntiles',
                 'APRUN_LANDANL', 'JEDIEXE', 'jedi_yaml',
                 'APPLY_INCR_NML_TMPL', 'APPLY_INCR_EXE', 'APRUN_APPLY_INCR']
         for key in keys:
@@ -530,6 +530,7 @@ def add_increments(config: Dict) -> None:
              DATA
              current_cycle
              CASE
+             OCNRES
              ntiles
              APPLY_INCR_NML_TMPL
              APPLY_INCR_EXE

workflow/create_experiment.py

@@ -63,7 +63,9 @@ def input_args():
                             formatter_class=ArgumentDefaultsHelpFormatter)
 
     parser.add_argument(
-        '--yaml', help='full path to yaml file describing the experiment configuration', type=Path, required=True)
+        '-y', '--yaml', help='full path to yaml file describing the experiment configuration', type=Path, required=True)
+    parser.add_argument(
+        '-o', '--overwrite', help='overwrite previously created experiment', action="store_true", required=False)
 
     return parser.parse_args()
 
@@ -89,6 +91,9 @@ def input_args():
         setup_expt_args.append(f"--{kk}")
         setup_expt_args.append(str(vv))
 
+    if user_inputs.overwrite:
+        setup_expt_args.append("--overwrite")
+
     logger.info(f"Call: setup_expt.main()")
     logger.debug(f"setup_expt.py {' '.join(setup_expt_args)}")
     setup_expt.main(setup_expt_args)

workflow/setup_expt.py

@@ -399,6 +399,8 @@ def _common_args(parser):
         parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!',
                             required=True, type=lambda dd: to_datetime(dd))
         parser.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: to_datetime(dd))
+        parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)',
+                            action='store_true', required=False)
         return parser
 
     def _gfs_args(parser):
@@ -493,17 +495,19 @@ def _gefs_args(parser):
     return parser.parse_args(list(*argv) if len(argv) else None)
 
 
-def query_and_clean(dirname):
+def query_and_clean(dirname, force_clean=False):
     """
     Method to query if a directory exists and gather user input for further action
     """
 
     create_dir = True
     if os.path.exists(dirname):
-        print()
-        print(f'directory already exists in {dirname}')
-        print()
-        overwrite = input('Do you wish to over-write [y/N]: ')
+        print(f'\ndirectory already exists in {dirname}')
+        if force_clean:
+            overwrite = True
+            print(f'removing directory ........ {dirname}\n')
+        else:
+            overwrite = input('Do you wish to over-write [y/N]: ')
         create_dir = True if overwrite in [
             'y', 'yes', 'Y', 'YES'] else False
         if create_dir:
@@ -553,8 +557,8 @@ def main(*argv):
     rotdir = os.path.join(user_inputs.comroot, user_inputs.pslot)
     expdir = os.path.join(user_inputs.expdir, user_inputs.pslot)
 
-    create_rotdir = query_and_clean(rotdir)
-    create_expdir = query_and_clean(expdir)
+    create_rotdir = query_and_clean(rotdir, force_clean=user_inputs.overwrite)
+    create_expdir = query_and_clean(expdir, force_clean=user_inputs.overwrite)
 
     if create_rotdir:
         makedirs_if_missing(rotdir)
@@ -565,6 +569,11 @@ def main(*argv):
         fill_EXPDIR(user_inputs)
         update_configs(host, user_inputs)
 
+    print(f"*" * 100)
+    print(f'EXPDIR: {expdir}')
+    print(f'ROTDIR: {rotdir}')
+    print(f"*" * 100)
+
 
 if __name__ == '__main__':