From 241a71eb2417c005a01cba37ac5383a45625be8b Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 13 Mar 2024 14:49:11 -0400 Subject: [PATCH] Change structure of output and log directories (#75) * change structrue of output dir * move slurm log file to LOGDIR * additional change for com structure * remove lmod-setup * Update Jenkinsfile accounts * Update Jenkinsfile: hera build option * Update Jenkinsfile: orion build option --------- Co-authored-by: zach1221 <99902696+zach1221@users.noreply.github.com> Co-authored-by: JONG KIM --- .gitignore | 1 + parm/land_analysis_era5_hera.yaml | 17 +++++++++++----- parm/land_analysis_era5_orion.yaml | 17 +++++++++++----- parm/land_analysis_gswp3_hera.yaml | 17 +++++++++++----- parm/land_analysis_gswp3_orion.yaml | 17 +++++++++++----- parm/lmod-setup.sh | 31 ----------------------------- parm/run_without_rocoto.sh | 20 ++++++++++--------- sorc/app_build.sh | 3 +-- sorc/test/ci/Jenkinsfile | 14 ++++++------- 9 files changed, 68 insertions(+), 69 deletions(-) delete mode 100755 parm/lmod-setup.sh diff --git a/.gitignore b/.gitignore index e11ea82b..718c857b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ parm/*.db parm/*.out parm/*.err parm/*.log +parm/land_analysis.yaml __pycache__ *.swp diff --git a/parm/land_analysis_era5_hera.yaml b/parm/land_analysis_era5_hera.yaml index d23c7e7e..a5664795 100644 --- a/parm/land_analysis_era5_hera.yaml +++ b/parm/land_analysis_era5_hera.yaml @@ -10,7 +10,9 @@ workflow: MACHINE: "hera" ACCOUNT: "nems" EXP_NAME: "LETKF" - EXP_BASEDIR: "/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/landda_version" + NET: "landda" + model_ver: "v1.2.1" + EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test" JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi" LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" FORCING: "era5" @@ -23,16 +25,16 @@ workflow: DAtype: "letkfoi_snow" SNOWDEPTHVAR: "snwdph" TSTUB: "oro_C96.mx100" - WORKDIR: "&EXP_BASEDIR;/workdir" + WORKDIR: "&EXP_BASEDIR;/workdir/run_&FORCING;" CYCLEDIR: "&EXP_BASEDIR;/land-DA_workflow" EXECdir: "&CYCLEDIR;/exec" - OUTDIR: "&EXP_BASEDIR;/landda_expts/DA_era5_test" - LOG: "&EXP_BASEDIR;/tests" + OUTDIR: "&EXP_BASEDIR;/com/&NET;/&model_ver;/run_&FORCING;" + LOGDIR: "&EXP_BASEDIR;/com/output/logs/run_&FORCING;" PATHRT: "&EXP_BASEDIR;" CTIME: "@Y@m@d@H" PTIME: "@Y@m@d@H" NTIME: "@Y@m@d@H" - log: "&LOG;/workflow.log" + log: "&LOGDIR;/workflow.log" tasks: task_prepexp: envars: @@ -55,6 +57,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_exp.log" task_prepobs: envars: OBS_TYPES: "&OBS_TYPES;" @@ -75,6 +78,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_obs.log" dependency: taskdep: attrs: @@ -101,6 +105,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_bmat.log" dependency: taskdep: attrs: @@ -133,6 +138,7 @@ workflow: nodes: "1:ppn=&NPROCS_ANA;" walltime: 00:15:00 queue: batch + join: "&LOGDIR;/run_ana.log" dependency: taskdep: attrs: @@ -165,6 +171,7 @@ workflow: nodes: "1:ppn=&NPROCS_FCST;" walltime: 00:30:00 queue: batch + join: "&LOGDIR;/run_fcst.log" dependency: taskdep: attrs: diff --git a/parm/land_analysis_era5_orion.yaml b/parm/land_analysis_era5_orion.yaml index ce93b273..a676ed84 100644 --- a/parm/land_analysis_era5_orion.yaml +++ b/parm/land_analysis_era5_orion.yaml @@ -10,7 +10,9 @@ workflow: MACHINE: "orion" ACCOUNT: "epic" EXP_NAME: "LETKF" - EXP_BASEDIR: "/work/noaa/epic/chjeon/landda_version" + NET: "landda" + model_ver: "v1.2.1" + EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test" JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi" LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs" FORCING: "era5" @@ -23,16 +25,16 @@ workflow: DAtype: "letkfoi_snow" SNOWDEPTHVAR: "snwdph" TSTUB: "oro_C96.mx100" - WORKDIR: "&EXP_BASEDIR;/workdir" + WORKDIR: "&EXP_BASEDIR;/workdir/run_&FORCING;" CYCLEDIR: "&EXP_BASEDIR;/land-DA_workflow" EXECdir: "&CYCLEDIR;/exec" - OUTDIR: "&EXP_BASEDIR;/landda_expts/DA_era5_test" - LOG: "&EXP_BASEDIR;/tests" + OUTDIR: "&EXP_BASEDIR;/com/&NET;/&model_ver;/run_&FORCING;" + LOGDIR: "&EXP_BASEDIR;/com/output/logs/run_&FORCING;" PATHRT: "&EXP_BASEDIR;" CTIME: "@Y@m@d@H" PTIME: "@Y@m@d@H" NTIME: "@Y@m@d@H" - log: "&LOG;/workflow.log" + log: "&LOGDIR;/workflow.log" tasks: task_prepexp: envars: @@ -55,6 +57,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_exp.log" task_prepobs: envars: OBS_TYPES: "&OBS_TYPES;" @@ -75,6 +78,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_obs.log" dependency: taskdep: attrs: @@ -101,6 +105,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_bmat.log" dependency: taskdep: attrs: @@ -133,6 +138,7 @@ workflow: nodes: "1:ppn=&NPROCS_ANA;" walltime: 00:15:00 queue: batch + join: "&LOGDIR;/run_ana.log" dependency: taskdep: attrs: @@ -165,6 +171,7 @@ workflow: nodes: "1:ppn=&NPROCS_FCST;" walltime: 00:30:00 queue: batch + join: "&LOGDIR;/run_fcst.log" dependency: taskdep: attrs: diff --git a/parm/land_analysis_gswp3_hera.yaml b/parm/land_analysis_gswp3_hera.yaml index 4817d667..d061a01f 100644 --- a/parm/land_analysis_gswp3_hera.yaml +++ b/parm/land_analysis_gswp3_hera.yaml @@ -10,7 +10,9 @@ workflow: MACHINE: "hera" ACCOUNT: "epic" EXP_NAME: "LETKF" - EXP_BASEDIR: "/scratch2/NAGAPE/epic/Chan-hoo.Jeon/landda_gswp3" + NET: "landda" + model_ver: "v1.2.1" + EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test" JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi" LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" FORCING: "gswp3" @@ -23,16 +25,16 @@ workflow: DAtype: "letkfoi_snow" SNOWDEPTHVAR: "snwdph" TSTUB: "oro_C96.mx100" - WORKDIR: "&EXP_BASEDIR;/workdir" + WORKDIR: "&EXP_BASEDIR;/workdir/run_&FORCING;" CYCLEDIR: "&EXP_BASEDIR;/land-DA_workflow" EXECdir: "&CYCLEDIR;/exec" - OUTDIR: "&EXP_BASEDIR;/landda_expts/DA_gswp3_test" - LOG: "&EXP_BASEDIR;/tests" + OUTDIR: "&EXP_BASEDIR;/com/&NET;/&model_ver;/run_&FORCING;" + LOGDIR: "&EXP_BASEDIR;/com/output/logs/run_&FORCING;" PATHRT: "&EXP_BASEDIR;" CTIME: "@Y@m@d@H" PTIME: "@Y@m@d@H" NTIME: "@Y@m@d@H" - log: "&LOG;/workflow.log" + log: "&LOGDIR;/workflow.log" tasks: task_prepexp: envars: @@ -55,6 +57,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_exp.log" task_prepobs: envars: OBS_TYPES: "&OBS_TYPES;" @@ -75,6 +78,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_obs.log" dependency: taskdep: attrs: @@ -101,6 +105,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_bmat.log" dependency: taskdep: attrs: @@ -133,6 +138,7 @@ workflow: nodes: "1:ppn=&NPROCS_ANA;" walltime: 00:15:00 queue: batch + join: "&LOGDIR;/run_ana.log" dependency: taskdep: attrs: @@ -165,6 +171,7 @@ workflow: nodes: "1:ppn=&NPROCS_FCST;" walltime: 00:30:00 queue: batch + join: "&LOGDIR;/run_fcst.log" dependency: taskdep: attrs: diff --git a/parm/land_analysis_gswp3_orion.yaml b/parm/land_analysis_gswp3_orion.yaml index 7e32ccf4..c06e091e 100644 --- a/parm/land_analysis_gswp3_orion.yaml +++ b/parm/land_analysis_gswp3_orion.yaml @@ -10,7 +10,9 @@ workflow: MACHINE: "orion" ACCOUNT: "epic" EXP_NAME: "LETKF" - EXP_BASEDIR: "/work/noaa/epic/chjeon/landda_fixjjob" + NET: "landda" + model_ver: "v1.2.1" + EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test" JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi" LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs" FORCING: "gswp3" @@ -23,16 +25,16 @@ workflow: DAtype: "letkfoi_snow" SNOWDEPTHVAR: "snwdph" TSTUB: "oro_C96.mx100" - WORKDIR: "&EXP_BASEDIR;/workdir" + WORKDIR: "&EXP_BASEDIR;/workdir/run_&FORCING;" CYCLEDIR: "&EXP_BASEDIR;/land-DA_workflow" EXECdir: "&CYCLEDIR;/exec" - OUTDIR: "&EXP_BASEDIR;/landda_expts/DA_gswp3_test" - LOG: "&EXP_BASEDIR;/tests" + OUTDIR: "&EXP_BASEDIR;/com/&NET;/&model_ver;/run_&FORCING;" + LOGDIR: "&EXP_BASEDIR;/com/output/logs/run_&FORCING;" PATHRT: "&EXP_BASEDIR;" CTIME: "@Y@m@d@H" PTIME: "@Y@m@d@H" NTIME: "@Y@m@d@H" - log: "&LOG;/workflow.log" + log: "&LOGDIR;/workflow.log" tasks: task_prepexp: envars: @@ -55,6 +57,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_exp.log" task_prepobs: envars: OBS_TYPES: "&OBS_TYPES;" @@ -75,6 +78,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_obs.log" dependency: taskdep: attrs: @@ -101,6 +105,7 @@ workflow: cores: 1 walltime: 00:02:00 queue: batch + join: "&LOGDIR;/prep_bmat.log" dependency: taskdep: attrs: @@ -133,6 +138,7 @@ workflow: nodes: "1:ppn=&NPROCS_ANA;" walltime: 00:15:00 queue: batch + join: "&LOGDIR;/run_ana.log" dependency: taskdep: attrs: @@ -165,6 +171,7 @@ workflow: nodes: "1:ppn=&NPROCS_FCST;" walltime: 00:45:00 queue: batch + join: "&LOGDIR;/run_fcst.log" dependency: taskdep: attrs: diff --git a/parm/lmod-setup.sh b/parm/lmod-setup.sh deleted file mode 100755 index e04c4bbd..00000000 --- a/parm/lmod-setup.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -if [ $# = 0 ]; then - L_MACHINE=${MACHINE} - cat << EOF_USAGE -Usage: source lmod-setup.sh PLATFORM - -OPTIONS: - PLATFORM - name of machine you are building on - (e.g. hera | jet | orion | hercules | wcoss2 ) -EOF_USAGE - exit 1 -else - L_MACHINE=$1 -fi - -if [ "$L_MACHINE" != wcoss2 ]; then - [[ ${SHELLOPTS} =~ nounset ]] && has_mu=true || has_mu=false - [[ ${SHELLOPTS} =~ errexit ]] && has_me=true || has_me=false - $has_mu && set +u - $has_me && set +e - source /etc/profile - $has_mu && set -u - $has_me && set -e -fi - -if [ "$L_MACHINE" = wcoss2 ]; then - module reset -else - module purge -fi diff --git a/parm/run_without_rocoto.sh b/parm/run_without_rocoto.sh index b589660e..4e8a3bb6 100644 --- a/parm/run_without_rocoto.sh +++ b/parm/run_without_rocoto.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=land_da_wflow -#SBATCH --account=nems +#SBATCH --account=epic #SBATCH --qos=debug #SBATCH --nodes=1 #SBATCH --tasks-per-node=6 @@ -10,16 +10,18 @@ #SBATCH -e err_landda_wflow.%j.err -export MACHINE="hera" -export ACCOUNT="nems" -export FORCING="gswp3" +export MACHINE="orion" +export ACCOUNT="epic" +export FORCING="era5" +export NET="landda" +export model_ver="v1.2.1" if [ "${MACHINE}" = "hera" ]; then - export EXP_BASEDIR="/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/landda_nonrocoto" + export EXP_BASEDIR="/scratch2/NAGAPE/epic/{USER}/landda_test" export JEDI_INSTALL="/scratch2/NAGAPE/epic/UFS_Land-DA/jedi" export LANDDA_INPUTS="/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" elif [ "${MACHINE}" = "orion" ]; then - export EXP_BASEDIR="/work/noaa/epic/chjeon/landda_nonrocoto" + export EXP_BASEDIR="/work/noaa/epic/{USER}/landda_test" export JEDI_INSTALL="/work/noaa/epic/UFS_Land-DA/jedi" export LANDDA_INPUTS="/work/noaa/epic/UFS_Land-DA/inputs" fi @@ -33,11 +35,11 @@ export fv3bundle_vn="psl_develop" export DAtype="letkfoi_snow" export SNOWDEPTHVAR="snwdph" export TSTUB="oro_C96.mx100" -export WORKDIR="${EXP_BASEDIR}/workdir" +export WORKDIR="${EXP_BASEDIR}/workdir/run_&FORCING;" export CYCLEDIR="${EXP_BASEDIR}/land-DA_workflow" export EXECdir="${CYCLEDIR}/exec" -export OUTDIR="${EXP_BASEDIR}/landda_expts/DA_${FORCING}_test" -export LOG="${EXP_BASEDIR}/tests" +export OUTDIR="${EXP_BASEDIR}/com/${NET}/${model_ver}/run_${FORCING}" +export LOGDIR="${EXP_BASEDIR}/com/output/logs" export PATHRT="${EXP_BASEDIR}" export ATMOS_FORC="${FORCING}" diff --git a/sorc/app_build.sh b/sorc/app_build.sh index e726bd66..95666241 100755 --- a/sorc/app_build.sh +++ b/sorc/app_build.sh @@ -343,8 +343,7 @@ if [ "${VERBOSE}" = true ]; then MAKE_SETTINGS="${MAKE_SETTINGS} VERBOSE=1" fi -# Before we go on load modules, we first need to activate Lmod for some systems -source ${HOME_DIR}/parm/lmod-setup.sh ${PLATFORM} +module purge # source version file for build BUILD_VERSION_FILE="${HOME_DIR}/versions/build.ver_${PLATFORM}" diff --git a/sorc/test/ci/Jenkinsfile b/sorc/test/ci/Jenkinsfile index a9370e8b..43d57a4b 100644 --- a/sorc/test/ci/Jenkinsfile +++ b/sorc/test/ci/Jenkinsfile @@ -60,7 +60,7 @@ pipeline { label "hera" } environment { - ACCNR = 'nems' + ACCNR = 'epic' NODE_PATH = '/scratch2/NAGAPE/epic/role.epic/' } steps { @@ -70,7 +70,7 @@ pipeline { sh ''' git submodule update --init --recursive ln -fs /scratch2/NAGAPE/epic/UFS_Land-DA/inputs /scratch2/NAGAPE/epic/role.epic/jenkins/workspace/ - sh sorc/app_build.sh + sh sorc/app_build.sh -c=intel --conda=off --build cp sorc/test/hera_ctest.sh sorc/build/ cp sorc/test/check_ctest.sh sorc/build/ cp sorc/test/run_hera_ctest.sh sorc/build/ @@ -88,7 +88,7 @@ pipeline { label "orion" } environment { - ACCNR = 'nems' + ACCNR = 'epic' NODE_PATH = '/work/noaa/epic/role-epic/' } steps { @@ -98,13 +98,13 @@ pipeline { sh ''' git submodule update --init --recursive ln -fs /work/noaa/epic/UFS_Land-DA/inputs /work/noaa/epic/role-epic/jenkins/workspace/ - sh sorc/app_build.sh + sh sorc/app_build.sh -c=intel --conda=off --build cp sorc/test/orion_ctest.sh sorc/build/ cp sorc/test/check_ctest.sh sorc/build/ cp sorc/test/run_orion_ctest.sh sorc/build/ - cd sorc/test/ - sed -i 's|MACHINE_ID=${MACHINE_ID:-hera}|MACHINE_ID=${MACHINE_ID:-orion}|g' run_ufs_datm_lnd.sh - cd ../../ + cd sorc/test/ + sed -i 's|MACHINE_ID=${MACHINE_ID:-hera}|MACHINE_ID=${MACHINE_ID:-orion}|g' run_ufs_datm_lnd.sh + cd ../../ cd sorc/build/ echo $(pwd) ctest