Change structure of output and log directories (#75)

* change structrue of output dir * move slurm log file to LOGDIR * additional change for com structure * remove lmod-setup * Update Jenkinsfile accounts * Update Jenkinsfile: hera build option * Update Jenkinsfile: orion build option --------- Co-authored-by: zach1221 <[email protected]> Co-authored-by: JONG KIM <[email protected]>
ufs-community · Mar 13, 2024 · 241a71e · 241a71e
1 parent 2e382e1
commit 241a71e
Show file tree

Hide file tree

Showing 9 changed files with 68 additions and 69 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@ parm/*.db
 parm/*.out
 parm/*.err
 parm/*.log
+parm/land_analysis.yaml
 
 __pycache__
 *.swp

diff --git a/parm/land_analysis_era5_hera.yaml b/parm/land_analysis_era5_hera.yaml
@@ -10,7 +10,9 @@ workflow:
     MACHINE: "hera"
     ACCOUNT: "nems"
     EXP_NAME: "LETKF"
-    EXP_BASEDIR: "/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/landda_version"
+    NET: "landda"
+    model_ver: "v1.2.1"
+    EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test"
     JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi"
     LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs"
     FORCING: "era5"
@@ -23,16 +25,16 @@ workflow:
     DAtype: "letkfoi_snow"
     SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
-    WORKDIR: "&EXP_BASEDIR;/workdir"
+    WORKDIR: "&EXP_BASEDIR;/workdir/run_&FORCING;"
     CYCLEDIR: "&EXP_BASEDIR;/land-DA_workflow"
     EXECdir: "&CYCLEDIR;/exec"
-    OUTDIR: "&EXP_BASEDIR;/landda_expts/DA_era5_test"
-    LOG: "&EXP_BASEDIR;/tests"
+    OUTDIR: "&EXP_BASEDIR;/com/&NET;/&model_ver;/run_&FORCING;"
+    LOGDIR: "&EXP_BASEDIR;/com/output/logs/run_&FORCING;"
     PATHRT: "&EXP_BASEDIR;"
     CTIME:  "<cyclestr>@Y@m@d@H</cyclestr>"
     PTIME:  "<cyclestr offset='-1:00:00'>@Y@m@d@H</cyclestr>"
     NTIME:  "<cyclestr offset='01:00:00'>@Y@m@d@H</cyclestr>"
-  log: "&LOG;/workflow.log"
+  log: "&LOGDIR;/workflow.log"
   tasks:
     task_prepexp:
       envars:
@@ -55,6 +57,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_exp.log"
     task_prepobs:
       envars:
         OBS_TYPES: "&OBS_TYPES;"
@@ -75,6 +78,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_obs.log"
       dependency:
         taskdep:
           attrs:
@@ -101,6 +105,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_bmat.log"
       dependency:
         taskdep:
           attrs:
@@ -133,6 +138,7 @@ workflow:
       nodes: "1:ppn=&NPROCS_ANA;"
       walltime: 00:15:00
       queue: batch
+      join: "&LOGDIR;/run_ana.log"
       dependency:
         taskdep:
           attrs:
@@ -165,6 +171,7 @@ workflow:
       nodes: "1:ppn=&NPROCS_FCST;"
       walltime: 00:30:00
       queue: batch
+      join: "&LOGDIR;/run_fcst.log"
       dependency:
         taskdep:
           attrs:

diff --git a/parm/land_analysis_era5_orion.yaml b/parm/land_analysis_era5_orion.yaml
@@ -10,7 +10,9 @@ workflow:
     MACHINE: "orion"
     ACCOUNT: "epic"
     EXP_NAME: "LETKF"
-    EXP_BASEDIR: "/work/noaa/epic/chjeon/landda_version"
+    NET: "landda"
+    model_ver: "v1.2.1"
+    EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test"
     JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi"
     LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs"
     FORCING: "era5"
@@ -23,16 +25,16 @@ workflow:
     DAtype: "letkfoi_snow"
     SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
-    WORKDIR: "&EXP_BASEDIR;/workdir"
+    WORKDIR: "&EXP_BASEDIR;/workdir/run_&FORCING;"
     CYCLEDIR: "&EXP_BASEDIR;/land-DA_workflow"
     EXECdir: "&CYCLEDIR;/exec"
-    OUTDIR: "&EXP_BASEDIR;/landda_expts/DA_era5_test"
-    LOG: "&EXP_BASEDIR;/tests"
+    OUTDIR: "&EXP_BASEDIR;/com/&NET;/&model_ver;/run_&FORCING;"
+    LOGDIR: "&EXP_BASEDIR;/com/output/logs/run_&FORCING;"
     PATHRT: "&EXP_BASEDIR;"
     CTIME:  "<cyclestr>@Y@m@d@H</cyclestr>"
     PTIME:  "<cyclestr offset='-1:00:00'>@Y@m@d@H</cyclestr>"
     NTIME:  "<cyclestr offset='01:00:00'>@Y@m@d@H</cyclestr>"
-  log: "&LOG;/workflow.log"
+  log: "&LOGDIR;/workflow.log"
   tasks:
     task_prepexp:
       envars:
@@ -55,6 +57,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_exp.log"
     task_prepobs:
       envars:
         OBS_TYPES: "&OBS_TYPES;"
@@ -75,6 +78,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_obs.log"
       dependency:
         taskdep:
           attrs:
@@ -101,6 +105,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_bmat.log"
       dependency:
         taskdep:
           attrs:
@@ -133,6 +138,7 @@ workflow:
       nodes: "1:ppn=&NPROCS_ANA;"
       walltime: 00:15:00
       queue: batch
+      join: "&LOGDIR;/run_ana.log"
       dependency:
         taskdep:
           attrs:
@@ -165,6 +171,7 @@ workflow:
       nodes: "1:ppn=&NPROCS_FCST;"
       walltime: 00:30:00
       queue: batch
+      join: "&LOGDIR;/run_fcst.log"
       dependency:
         taskdep:
           attrs:

diff --git a/parm/land_analysis_gswp3_hera.yaml b/parm/land_analysis_gswp3_hera.yaml
@@ -10,7 +10,9 @@ workflow:
     MACHINE: "hera"
     ACCOUNT: "epic"
     EXP_NAME: "LETKF"
-    EXP_BASEDIR: "/scratch2/NAGAPE/epic/Chan-hoo.Jeon/landda_gswp3"
+    NET: "landda"
+    model_ver: "v1.2.1"
+    EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test"
     JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi"
     LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs"
     FORCING: "gswp3"
@@ -23,16 +25,16 @@ workflow:
     DAtype: "letkfoi_snow"
     SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
-    WORKDIR: "&EXP_BASEDIR;/workdir"
+    WORKDIR: "&EXP_BASEDIR;/workdir/run_&FORCING;"
     CYCLEDIR: "&EXP_BASEDIR;/land-DA_workflow"
     EXECdir: "&CYCLEDIR;/exec"
-    OUTDIR: "&EXP_BASEDIR;/landda_expts/DA_gswp3_test"
-    LOG: "&EXP_BASEDIR;/tests"
+    OUTDIR: "&EXP_BASEDIR;/com/&NET;/&model_ver;/run_&FORCING;"
+    LOGDIR: "&EXP_BASEDIR;/com/output/logs/run_&FORCING;"
     PATHRT: "&EXP_BASEDIR;"
     CTIME:  "<cyclestr>@Y@m@d@H</cyclestr>"
     PTIME:  "<cyclestr offset='-24:00:00'>@Y@m@d@H</cyclestr>"
     NTIME:  "<cyclestr offset='24:00:00'>@Y@m@d@H</cyclestr>"
-  log: "&LOG;/workflow.log"
+  log: "&LOGDIR;/workflow.log"
   tasks:
     task_prepexp:
       envars:
@@ -55,6 +57,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_exp.log"
     task_prepobs:
       envars:
         OBS_TYPES: "&OBS_TYPES;"
@@ -75,6 +78,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_obs.log"
       dependency:
         taskdep:
           attrs:
@@ -101,6 +105,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_bmat.log"
       dependency:
         taskdep:
           attrs:
@@ -133,6 +138,7 @@ workflow:
       nodes: "1:ppn=&NPROCS_ANA;"
       walltime: 00:15:00
       queue: batch
+      join: "&LOGDIR;/run_ana.log"
       dependency:
         taskdep:
           attrs:
@@ -165,6 +171,7 @@ workflow:
       nodes: "1:ppn=&NPROCS_FCST;"
       walltime: 00:30:00
       queue: batch
+      join: "&LOGDIR;/run_fcst.log"
       dependency:
         taskdep:
           attrs:

diff --git a/parm/land_analysis_gswp3_orion.yaml b/parm/land_analysis_gswp3_orion.yaml
@@ -10,7 +10,9 @@ workflow:
     MACHINE: "orion"
     ACCOUNT: "epic"
     EXP_NAME: "LETKF"
-    EXP_BASEDIR: "/work/noaa/epic/chjeon/landda_fixjjob"
+    NET: "landda"
+    model_ver: "v1.2.1"
+    EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test"
     JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi"
     LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs"
     FORCING: "gswp3"
@@ -23,16 +25,16 @@ workflow:
     DAtype: "letkfoi_snow"
     SNOWDEPTHVAR: "snwdph"
     TSTUB: "oro_C96.mx100"
-    WORKDIR: "&EXP_BASEDIR;/workdir"
+    WORKDIR: "&EXP_BASEDIR;/workdir/run_&FORCING;"
     CYCLEDIR: "&EXP_BASEDIR;/land-DA_workflow"
     EXECdir: "&CYCLEDIR;/exec"
-    OUTDIR: "&EXP_BASEDIR;/landda_expts/DA_gswp3_test"
-    LOG: "&EXP_BASEDIR;/tests"
+    OUTDIR: "&EXP_BASEDIR;/com/&NET;/&model_ver;/run_&FORCING;"
+    LOGDIR: "&EXP_BASEDIR;/com/output/logs/run_&FORCING;"
     PATHRT: "&EXP_BASEDIR;"
     CTIME:  "<cyclestr>@Y@m@d@H</cyclestr>"
     PTIME:  "<cyclestr offset='-24:00:00'>@Y@m@d@H</cyclestr>"
     NTIME:  "<cyclestr offset='24:00:00'>@Y@m@d@H</cyclestr>"
-  log: "&LOG;/workflow.log"
+  log: "&LOGDIR;/workflow.log"
   tasks:
     task_prepexp:
       envars:
@@ -55,6 +57,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_exp.log"
     task_prepobs:
       envars:
         OBS_TYPES: "&OBS_TYPES;"
@@ -75,6 +78,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_obs.log"
       dependency:
         taskdep:
           attrs:
@@ -101,6 +105,7 @@ workflow:
       cores: 1
       walltime: 00:02:00
       queue: batch
+      join: "&LOGDIR;/prep_bmat.log"
       dependency:
         taskdep:
           attrs:
@@ -133,6 +138,7 @@ workflow:
       nodes: "1:ppn=&NPROCS_ANA;"
       walltime: 00:15:00
       queue: batch
+      join: "&LOGDIR;/run_ana.log"
       dependency:
         taskdep:
           attrs:
@@ -165,6 +171,7 @@ workflow:
       nodes: "1:ppn=&NPROCS_FCST;"
       walltime: 00:45:00
       queue: batch
+      join: "&LOGDIR;/run_fcst.log"
       dependency:
         taskdep:
           attrs:

diff --git a/parm/lmod-setup.sh b/parm/lmod-setup.sh
diff --git a/parm/run_without_rocoto.sh b/parm/run_without_rocoto.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #SBATCH --job-name=land_da_wflow
-#SBATCH --account=nems
+#SBATCH --account=epic
 #SBATCH --qos=debug
 #SBATCH --nodes=1
 #SBATCH --tasks-per-node=6
@@ -10,16 +10,18 @@
 #SBATCH -e err_landda_wflow.%j.err
 
 
-export MACHINE="hera"
-export ACCOUNT="nems"
-export FORCING="gswp3"
+export MACHINE="orion"
+export ACCOUNT="epic"
+export FORCING="era5"
+export NET="landda"
+export model_ver="v1.2.1"
 
 if [ "${MACHINE}" = "hera" ]; then
-  export EXP_BASEDIR="/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/landda_nonrocoto"
+  export EXP_BASEDIR="/scratch2/NAGAPE/epic/{USER}/landda_test"
   export JEDI_INSTALL="/scratch2/NAGAPE/epic/UFS_Land-DA/jedi"
   export LANDDA_INPUTS="/scratch2/NAGAPE/epic/UFS_Land-DA/inputs"
 elif [ "${MACHINE}" = "orion" ]; then
-  export EXP_BASEDIR="/work/noaa/epic/chjeon/landda_nonrocoto"
+  export EXP_BASEDIR="/work/noaa/epic/{USER}/landda_test"
   export JEDI_INSTALL="/work/noaa/epic/UFS_Land-DA/jedi"
   export LANDDA_INPUTS="/work/noaa/epic/UFS_Land-DA/inputs"
 fi
@@ -33,11 +35,11 @@ export fv3bundle_vn="psl_develop"
 export DAtype="letkfoi_snow"
 export SNOWDEPTHVAR="snwdph"
 export TSTUB="oro_C96.mx100"
-export WORKDIR="${EXP_BASEDIR}/workdir"
+export WORKDIR="${EXP_BASEDIR}/workdir/run_&FORCING;"
 export CYCLEDIR="${EXP_BASEDIR}/land-DA_workflow"
 export EXECdir="${CYCLEDIR}/exec"
-export OUTDIR="${EXP_BASEDIR}/landda_expts/DA_${FORCING}_test"
-export LOG="${EXP_BASEDIR}/tests"
+export OUTDIR="${EXP_BASEDIR}/com/${NET}/${model_ver}/run_${FORCING}"
+export LOGDIR="${EXP_BASEDIR}/com/output/logs"
 export PATHRT="${EXP_BASEDIR}"
 
 export ATMOS_FORC="${FORCING}"

diff --git a/sorc/app_build.sh b/sorc/app_build.sh
@@ -343,8 +343,7 @@ if [ "${VERBOSE}" = true ]; then
   MAKE_SETTINGS="${MAKE_SETTINGS} VERBOSE=1"
 fi
 
-# Before we go on load modules, we first need to activate Lmod for some systems
-source ${HOME_DIR}/parm/lmod-setup.sh ${PLATFORM}
+module purge
 
 # source version file for build
 BUILD_VERSION_FILE="${HOME_DIR}/versions/build.ver_${PLATFORM}"