From fc3146fb10eb6f252fff86ba5dcdcb0deec2327a Mon Sep 17 00:00:00 2001 From: jkbk2004 Date: Mon, 12 Feb 2024 09:13:36 -0600 Subject: [PATCH 1/5] add initial jobs and parm --- jobs/JGLOBAL_LAND_PREP | 174 +++++++++++++++++++++++++++++++++ jobs/JGLOBAL_PREP_LAND_OBS | 66 +++++++++++++ parm/land_da_orion.xml | 193 +++++++++++++++++++++++++++++++++++++ parm/land_da_orion.yaml | 93 ++++++++++++++++++ 4 files changed, 526 insertions(+) create mode 100755 jobs/JGLOBAL_LAND_PREP create mode 100755 jobs/JGLOBAL_PREP_LAND_OBS create mode 100644 parm/land_da_orion.xml create mode 100644 parm/land_da_orion.yaml diff --git a/jobs/JGLOBAL_LAND_PREP b/jobs/JGLOBAL_LAND_PREP new file mode 100755 index 00000000..1dac5800 --- /dev/null +++ b/jobs/JGLOBAL_LAND_PREP @@ -0,0 +1,174 @@ +#!/bin/sh + +set -ex + +############################ +# copy restarts to workdir, convert to UFS tile for DA (all members) + +if [[ ${EXP_NAME} == "openloop" ]]; then + do_jedi="NO" +else + do_jedi="YES" + SAVE_TILE="YES" + LANDDADIR=${CYCLEDIR}/DA_update/ +fi + +echo ${LANDDA_INPUTS}, ${ATMOS_FORC} +TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ +YYYY=${CYYYY} +MM=${CMM} +DD=${CDD} +HH=${CHH} +YYYP=${PYYYY} +MP=${PMM} +DP=${PDD} +HP=${PHH} +mem_ens="mem000" + +MEM_WORKDIR=${WORKDIR}/${mem_ens} +MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} +RSTRDIR=${MEM_WORKDIR} +JEDIWORKDIR=${WORKDIR}/mem000/jedi +FILEDATE=${YYYY}${MM}${DD}.${HH}0000 + +if [[ ! -e ${MEM_WORKDIR} ]]; then + mkdir -p ${MEM_WORKDIR} +fi +if [[ ! -e ${MEM_MODL_OUTDIR} ]]; then + mkdir -p ${MEM_MODL_OUTDIR} +fi + +mkdir -p $MEM_WORKDIR/modulefiles; cp ${CYCLEDIR}/modulefiles/landda_${MACHINE}.intel.lua $MEM_WORKDIR/modulefiles/modules.landda.lua +cd $MEM_WORKDIR + +# load modulefiles +module use modulefiles; module load modules.landda + +if [[ $do_jedi == "YES" && $ATMOS_FORC == "era5" ]]; then + + # copy restarts into work directory + rst_in=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + if [[ ! -e ${rst_in} ]]; then + rst_in=${LANDDA_INPUTS}/restarts/${ATMOS_FORC}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + fi + rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + cp ${rst_in} ${rst_out} + + echo '************************************************' + echo 'calling vector2tile' + + export MEM_WORKDIR + + # update vec2tile and tile2vec namelists + cp ${CYCLEDIR}/template.vector2tile vector2tile.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" vector2tile.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" vector2tile.namelist + sed -i -e "s/XXMM/${MM}/g" vector2tile.namelist + sed -i -e "s/XXDD/${DD}/g" vector2tile.namelist + sed -i -e "s/XXHH/${HH}/g" vector2tile.namelist + sed -i -e "s/XXHH/${HH}/g" vector2tile.namelist + sed -i -e "s/MODEL_FORCING/${ATMOS_FORC}/g" vector2tile.namelist + sed -i -e "s/XXRES/${RES}/g" vector2tile.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" vector2tile.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" vector2tile.namelist + + # submit vec2tile + echo '************************************************' + echo 'calling vector2tile' + + ${BUILDDIR}/tile2tile_converter.exe vector2tile.namelist + if [[ $? != 0 ]]; then + echo "vec2tile failed" + exit + fi +fi # vector2tile for DA + +if [[ $do_jedi == "YES" && $ATMOS_FORC == "gswp3" ]]; then + + echo '************************************************' + echo 'calling tile2tile' + + export MEM_WORKDIR + + # copy restarts into work directory + for tile in 1 2 3 4 5 6 + do + rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + if [[ ! -e ${rst_in} ]]; then + rst_in=${LANDDA_INPUTS}/restarts/${ATMOS_FORC}/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc + fi + rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + cp ${rst_in} ${rst_out} + done + + # update tile2tile namelist + cp ${CYCLEDIR}/template.ufs2jedi ufs2jedi.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs2jedi.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" ufs2jedi.namelist + sed -i -e "s/XXMM/${MM}/g" ufs2jedi.namelist + sed -i -e "s/XXDD/${DD}/g" ufs2jedi.namelist + sed -i -e "s/XXHH/${HH}/g" ufs2jedi.namelist + sed -i -e "s/XXHH/${HH}/g" ufs2jedi.namelist + sed -i -e "s/MODEL_FORCING/${ATMOS_FORC}/g" ufs2jedi.namelist + sed -i -e "s/XXRES/${RES}/g" ufs2jedi.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" ufs2jedi.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" ufs2jedi.namelist + + # submit tile2tile + ${BUILDDIR}/tile2tile_converter.exe ufs2jedi.namelist + if [[ $? != 0 ]]; then + echo "tile2tile failed" + exit + fi +fi # tile2tile for DA + +if [[ $do_jedi == "YES" ]]; then + if [[ ! -e ${OUTDIR}/DA ]]; then + mkdir -p ${OUTDIR}/DA/jedi_incr + mkdir -p ${OUTDIR}/DA/logs + mkdir -p ${OUTDIR}/DA/hofx + fi + if [[ ! -e $JEDIWORKDIR ]]; then + mkdir -p $JEDIWORKDIR + fi + cd $JEDIWORKDIR + + if [[ ! -e ${JEDIWORKDIR}/output ]]; then + ln -s ${OUTDIR} ${JEDIWORKDIR}/output + fi + + if [[ $SAVE_TILE == "YES" ]]; then + for tile in 1 2 3 4 5 6 + do + cp ${RSTRDIR}/${FILEDATE}.sfc_data.tile${tile}.nc ${RSTRDIR}/${FILEDATE}.sfc_data_back.tile${tile}.nc + done + fi + + #stage restarts for applying JEDI update (files will get directly updated) + for tile in 1 2 3 4 5 6 + do + ln -fs ${RSTRDIR}/${FILEDATE}.sfc_data.tile${tile}.nc ${JEDIWORKDIR}/${FILEDATE}.sfc_data.tile${tile}.nc + done + + cres_file=${JEDIWORKDIR}/${FILEDATE}.coupler.res + + if [[ -e ${RSTRDIR}/${FILEDATE}.coupler.res ]]; then + ln -sf ${RSTRDIR}/${FILEDATE}.coupler.res $cres_file + else # if not present, need to create coupler.res for JEDI + cp ${LANDDADIR}/template.coupler.res $cres_file + + sed -i -e "s/XXYYYY/${YYYY}/g" $cres_file + sed -i -e "s/XXMM/${MM}/g" $cres_file + sed -i -e "s/XXDD/${DD}/g" $cres_file + sed -i -e "s/XXHH/${HH}/g" $cres_file + + sed -i -e "s/XXYYYP/${YYYP}/g" $cres_file + sed -i -e "s/XXMP/${MP}/g" $cres_file + sed -i -e "s/XXDP/${DP}/g" $cres_file + sed -i -e "s/XXHP/${HP}/g" $cres_file + + fi +fi # do_jedi setup + diff --git a/jobs/JGLOBAL_PREP_LAND_OBS b/jobs/JGLOBAL_PREP_LAND_OBS new file mode 100755 index 00000000..3a6618e3 --- /dev/null +++ b/jobs/JGLOBAL_PREP_LAND_OBS @@ -0,0 +1,66 @@ +#!/bin/sh + +set -ex + +############################ +# copy restarts to workdir, convert to UFS tile for DA (all members) + +if [[ ${EXP_NAME} == "openloop" ]]; then + do_jedi="NO" +else + do_jedi="YES" + SAVE_TILE="YES" + LANDDADIR=${CYCLEDIR}/DA_update/ +fi + +TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ +YYYY=${CYYYY} +MM=${CMM} +DD=${CDD} +HH=${CHH} +YYYP=${PYYYY} +MP=${PMM} +DP=${PDD} +HP=${PHH} +mem_ens="mem000" + +MEM_WORKDIR=${WORKDIR}/${mem_ens} +MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} +RSTRDIR=${MEM_WORKDIR} +JEDIWORKDIR=${WORKDIR}/mem000/jedi +FILEDATE=${YYYY}${MM}${DD}.${HH}0000 + +cd $JEDIWORKDIR + +# load modulefiles +#module use modulefiles; module load modules.landda + +OBSDIR=${LANDDA_INPUTS}/DA +################################################ +# 2. PREPARE OBS FILES +################################################ + +# get the obs file name +if [ ${OBS_TYPES} == "GTS" ]; then + obsfile=$OBSDIR/snow_depth/GTS/data_proc/${YYYY}${MM}/adpsfc_snow_${YYYY}${MM}${DD}${HH}.nc4 + # GHCN are time-stamped at 18. If assimilating at 00, need to use previous day's obs, so that + # obs are within DA window. +elif [ $ATMOS_FORC == "era5" ] && [ ${OBS_TYPES} == "GHCN" ]; then + obsfile=$OBSDIR/snow_depth/GHCN/data_proc/v3/${YYYY}/ghcn_snwd_ioda_${YYYP}${MP}${DP}.nc +elif [ $ATMOS_FORC == "gswp3" ] && [ ${OBS_TYPES} == "GHCN" ]; then + obsfile=$OBSDIR/snow_depth/GHCN/data_proc/v3/${YYYY}/fake_ghcn_snwd_ioda_${YYYP}${MP}${DP}.nc +elif [ ${OBS_TYPES} == "SYNTH" ]; then + obsfile=$OBSDIR/synthetic_noahmp/IODA.synthetic_gswp_obs.${YYYY}${MM}${DD}${HH}.nc +else + echo "do_landDA: Unknown obs type requested ${OBS_TYPES[$ii]}, exiting" + exit 1 +fi + +# check obs are available +if [[ -e $obsfile ]]; then + echo "do_landDA: ${OBS_TYPES} observations found: $obsfile" + ln -fs $obsfile ${OBS_TYPES}_${YYYY}${MM}${DD}${HH}.nc +else + echo "${OBS_TYPES} observations not found: $obsfile" +# JEDI_TYPES[$ii]="SKIP" +fi diff --git a/parm/land_da_orion.xml b/parm/land_da_orion.xml new file mode 100644 index 00000000..8adfa2ed --- /dev/null +++ b/parm/land_da_orion.xml @@ -0,0 +1,193 @@ + + + + + + + + + + + + + + @Y"> + @m"> + @d"> + @H"> + @Y"> + @m"> + @d"> + @H"> + @Y"> + @m"> + @d"> + @H"> +]> + + 200001030000 200001030000 24:00:00 + /work/noaa/epic/jongkim/land-DA/tests/workflow.log; + + epic + 1 + batch + 00:02:00 + &CYCLEDIR;/jobs/JGLOBAL_LAND_PREP + prepexp + + MACHINE + &MACHINE; + + + EXP_NAME + &EXP_NAME; + + + LANDDA_INPUTS + &LANDDA_INPUTS; + + + ATMOS_FORC + &FORCING; + + + RES + &RES; + + + TSTUB + &TSTUB; + + + WORKDIR + &WORKDIR; + + + CYCLEDIR + &CYCLEDIR; + + + BUILDDIR + &BUILDDIR; + + + OUTDIR + &OUTDIR; + + + CYYYY + &CYEAR; + + + CMM + &CMONTH; + + + CDD + &CDATE; + + + CHH + &CHOUR; + + + PYYYY + &PYEAR; + + + PMM + &PMONTH; + + + PDD + &PDATE; + + + PHH + &PHOUR; + + + + GHCN + + epic + 1 + batch + 00:02:00 + &CYCLEDIR;/jobs/JGLOBAL_PREP_LAND_OBS + prepobs_#obstypes# + + OBS_TYPES + #obstypes# + + + MACHINE + &MACHINE; + + + EXP_NAME + &EXP_NAME; + + + LANDDA_INPUTS + &LANDDA_INPUTS; + + + ATMOS_FORC + &FORCING; + + + WORKDIR + &WORKDIR; + + + CYCLEDIR + &CYCLEDIR; + + + BUILDDIR + &BUILDDIR; + + + OUTDIR + &OUTDIR; + + + CYYYY + &CYEAR; + + + CMM + &CMONTH; + + + CDD + &CDATE; + + + CHH + &CHOUR; + + + PYYYY + &PYEAR; + + + PMM + &PMONTH; + + + PDD + &PDATE; + + + PHH + &PHOUR; + + + + + + + diff --git a/parm/land_da_orion.yaml b/parm/land_da_orion.yaml new file mode 100644 index 00000000..16aafc28 --- /dev/null +++ b/parm/land_da_orion.yaml @@ -0,0 +1,93 @@ +workflow: + attrs: + realtime: false + scheduler: slurm + cycledef: + - attrs: + group: epic + spec: 200001030000 200001030000 24:00:00 + entities: + EXP_NAME: "LETKF" + LANDDA_INPUTS: "/work/noaa/epic/jongkim/land-DA/inputs" + FORCING: "gswp3" + RES: "96" + MACHINE: orion + TSTUB: "oro_C96.mx100" + WORKDIR: /work/noaa/epic/jongkim/land-DA/workdir + CYCLEDIR: /work/noaa/epic/jongkim/land-DA/land-DA_workflow + BUILDDIR: /work/noaa/epic/jongkim/land-DA/land-DA_workflow/build/bin + OUTDIR: /work/noaa/epic/jongkim/land-DA/landda_expts/DA_GSWP3_test + LOG: /work/noaa/epic/jongkim/land-DA/tests + PATHRT: /work/noaa/epic/jongkim/land-DA + CYEAR: "@Y" + CMONTH: "@m" + CDATE: "@d" + CHOUR: "@H" + PYEAR: "@Y" + PMONTH: "@m" + PDATE: "@d" + PHOUR: "@H" + NYEAR: "@Y" + NMONTH: "@m" + NDATE: "@d" + NHOUR: "@H" + log: /work/noaa/epic/jongkim/land-DA/tests/workflow.log; + tasks: + task_prepexp: + envars: + MACHINE: "&MACHINE;" + EXP_NAME: "&EXP_NAME;" + LANDDA_INPUTS: "&LANDDA_INPUTS;" + ATMOS_FORC: "&FORCING;" + RES: "&RES;" + TSTUB: "&TSTUB;" + WORKDIR: "&WORKDIR;" + CYCLEDIR: "&CYCLEDIR;" + BUILDDIR: "&BUILDDIR;" + OUTDIR: "&OUTDIR;" + CYYYY: "&CYEAR;" + CMM: "&CMONTH;" + CDD: "&CDATE;" + CHH: "&CHOUR;" + PYYYY: "&PYEAR;" + PMM: "&PMONTH;" + PDD: "&PDATE;" + PHH: "&PHOUR;" + account: epic + command: "&CYCLEDIR;/jobs/JGLOBAL_LAND_PREP" + jobname: prepexp + cores: 1 + walltime: 00:02:00 + queue: batch + metatask_prepobs: + var: + obstypes: GHCN + task_prepobs_#obstypes#: + account: epic + envars: + OBS_TYPES: "#obstypes#" + MACHINE: "&MACHINE;" + EXP_NAME: "&EXP_NAME;" + LANDDA_INPUTS: "&LANDDA_INPUTS;" + ATMOS_FORC: "&FORCING;" + WORKDIR: "&WORKDIR;" + CYCLEDIR: "&CYCLEDIR;" + BUILDDIR: "&BUILDDIR;" + OUTDIR: "&OUTDIR;" + CYYYY: "&CYEAR;" + CMM: "&CMONTH;" + CDD: "&CDATE;" + CHH: "&CHOUR;" + PYYYY: "&PYEAR;" + PMM: "&PMONTH;" + PDD: "&PDATE;" + PHH: "&PHOUR;" + command: "&CYCLEDIR;/jobs/JGLOBAL_PREP_LAND_OBS" + jobname: prepobs_#obstypes# + cores: 1 + walltime: 00:02:00 + queue: batch + dependency: + taskdep: + attrs: + task: prepexp \ No newline at end of file From f75bf3ec764faca69563d49f97b4a62ead409e67 Mon Sep 17 00:00:00 2001 From: jkbk2004 Date: Thu, 15 Feb 2024 08:13:06 -0600 Subject: [PATCH 2/5] update jobs yaml xml --- jobs/JGLOBAL_PREP_LAND_OBS | 66 ---- jobs/JLAND_PREP_BMAT | 85 +++++ jobs/{JGLOBAL_LAND_PREP => JLAND_PREP_EXP} | 0 jobs/JLAND_PREP_OBS | 67 ++++ jobs/JLAND_RUN_ANA | 210 ++++++++++++ parm/land_da_orion.xml | 363 ++++++++++++++++----- parm/land_da_orion.yaml | 136 ++++++-- 7 files changed, 745 insertions(+), 182 deletions(-) delete mode 100755 jobs/JGLOBAL_PREP_LAND_OBS create mode 100755 jobs/JLAND_PREP_BMAT rename jobs/{JGLOBAL_LAND_PREP => JLAND_PREP_EXP} (100%) create mode 100755 jobs/JLAND_PREP_OBS create mode 100755 jobs/JLAND_RUN_ANA diff --git a/jobs/JGLOBAL_PREP_LAND_OBS b/jobs/JGLOBAL_PREP_LAND_OBS deleted file mode 100755 index 3a6618e3..00000000 --- a/jobs/JGLOBAL_PREP_LAND_OBS +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh - -set -ex - -############################ -# copy restarts to workdir, convert to UFS tile for DA (all members) - -if [[ ${EXP_NAME} == "openloop" ]]; then - do_jedi="NO" -else - do_jedi="YES" - SAVE_TILE="YES" - LANDDADIR=${CYCLEDIR}/DA_update/ -fi - -TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ -YYYY=${CYYYY} -MM=${CMM} -DD=${CDD} -HH=${CHH} -YYYP=${PYYYY} -MP=${PMM} -DP=${PDD} -HP=${PHH} -mem_ens="mem000" - -MEM_WORKDIR=${WORKDIR}/${mem_ens} -MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} -RSTRDIR=${MEM_WORKDIR} -JEDIWORKDIR=${WORKDIR}/mem000/jedi -FILEDATE=${YYYY}${MM}${DD}.${HH}0000 - -cd $JEDIWORKDIR - -# load modulefiles -#module use modulefiles; module load modules.landda - -OBSDIR=${LANDDA_INPUTS}/DA -################################################ -# 2. PREPARE OBS FILES -################################################ - -# get the obs file name -if [ ${OBS_TYPES} == "GTS" ]; then - obsfile=$OBSDIR/snow_depth/GTS/data_proc/${YYYY}${MM}/adpsfc_snow_${YYYY}${MM}${DD}${HH}.nc4 - # GHCN are time-stamped at 18. If assimilating at 00, need to use previous day's obs, so that - # obs are within DA window. -elif [ $ATMOS_FORC == "era5" ] && [ ${OBS_TYPES} == "GHCN" ]; then - obsfile=$OBSDIR/snow_depth/GHCN/data_proc/v3/${YYYY}/ghcn_snwd_ioda_${YYYP}${MP}${DP}.nc -elif [ $ATMOS_FORC == "gswp3" ] && [ ${OBS_TYPES} == "GHCN" ]; then - obsfile=$OBSDIR/snow_depth/GHCN/data_proc/v3/${YYYY}/fake_ghcn_snwd_ioda_${YYYP}${MP}${DP}.nc -elif [ ${OBS_TYPES} == "SYNTH" ]; then - obsfile=$OBSDIR/synthetic_noahmp/IODA.synthetic_gswp_obs.${YYYY}${MM}${DD}${HH}.nc -else - echo "do_landDA: Unknown obs type requested ${OBS_TYPES[$ii]}, exiting" - exit 1 -fi - -# check obs are available -if [[ -e $obsfile ]]; then - echo "do_landDA: ${OBS_TYPES} observations found: $obsfile" - ln -fs $obsfile ${OBS_TYPES}_${YYYY}${MM}${DD}${HH}.nc -else - echo "${OBS_TYPES} observations not found: $obsfile" -# JEDI_TYPES[$ii]="SKIP" -fi diff --git a/jobs/JLAND_PREP_BMAT b/jobs/JLAND_PREP_BMAT new file mode 100755 index 00000000..3ea6b3d2 --- /dev/null +++ b/jobs/JLAND_PREP_BMAT @@ -0,0 +1,85 @@ +#!/bin/sh + +set -ex + +############################ +# copy restarts to workdir, convert to UFS tile for DA (all members) + +if [[ ${EXP_NAME} == "openloop" ]]; then + do_jedi="NO" +else + do_jedi="YES" + SAVE_TILE="YES" + LANDDADIR=${CYCLEDIR}/DA_update/ +fi + +TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ +YYYY=${CYYYY} +MM=${CMM} +DD=${CDD} +HH=${CHH} +YYYP=${PYYYY} +MP=${PMM} +DP=${PDD} +HP=${PHH} +mem_ens="mem000" + +MEM_WORKDIR=${WORKDIR}/${mem_ens} +MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} +RSTRDIR=${MEM_WORKDIR} +JEDIWORKDIR=${WORKDIR}/mem000/jedi +FILEDATE=${YYYY}${MM}${DD}.${HH}0000 + +cd $MEM_WORKDIR + +# load modulefiles +module use modulefiles; module load modules.landda +PYTHON=$(/usr/bin/which python) + +#fv3bundle_vn=psl_develop +#DAtype=letkfoi_snow +#SNOWDEPTHVAR=snwdph +YAML_DA=construct +GFSv17="NO" +B=30 # back ground error std for LETKFOI +cd $JEDIWORKDIR + +################################################ +# 4. CREATE BACKGROUND ENSEMBLE (LETKFOI) +################################################ + +if [[ ${DAtype} == 'letkfoi_snow' ]]; then + + JEDI_EXEC="fv3jedi_letkf.x" + + if [ $GFSv17 == "YES" ]; then + SNOWDEPTHVAR="snodl" + # field overwrite file with GFSv17 variables. + cp ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/gfs-land-v17.yaml ${JEDIWORKDIR}/gfs-land-v17.yaml + else + SNOWDEPTHVAR="snwdph" + fi + # FOR LETKFOI, CREATE THE PSEUDO-ENSEMBLE + for ens in pos neg + do + if [ -e $JEDIWORKDIR/mem_${ens} ]; then + rm -r $JEDIWORKDIR/mem_${ens} + fi + mkdir -p $JEDIWORKDIR/mem_${ens} + for tile in 1 2 3 4 5 6 + do + cp ${JEDIWORKDIR}/${FILEDATE}.sfc_data.tile${tile}.nc ${JEDIWORKDIR}/mem_${ens}/${FILEDATE}.sfc_data.tile${tile}.nc + done + cp ${JEDIWORKDIR}/${FILEDATE}.coupler.res ${JEDIWORKDIR}/mem_${ens}/${FILEDATE}.coupler.res + done + + echo 'do_landDA: calling create ensemble' + + # using ioda mods to get a python version with netCDF4 + ${PYTHON} ${LANDDADIR}/letkf_create_ens.py $FILEDATE $SNOWDEPTHVAR $B + if [[ $? != 0 ]]; then + echo "letkf create failed" + exit 10 + fi + +fi diff --git a/jobs/JGLOBAL_LAND_PREP b/jobs/JLAND_PREP_EXP similarity index 100% rename from jobs/JGLOBAL_LAND_PREP rename to jobs/JLAND_PREP_EXP diff --git a/jobs/JLAND_PREP_OBS b/jobs/JLAND_PREP_OBS new file mode 100755 index 00000000..950c48c1 --- /dev/null +++ b/jobs/JLAND_PREP_OBS @@ -0,0 +1,67 @@ +#!/bin/sh + +set -ex + +############################ +# copy restarts to workdir, convert to UFS tile for DA (all members) + +if [[ ${EXP_NAME} == "openloop" ]]; then + do_jedi="NO" +else + do_jedi="YES" + SAVE_TILE="YES" + LANDDADIR=${CYCLEDIR}/DA_update/ +fi + +TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ +YYYY=${CYYYY} +MM=${CMM} +DD=${CDD} +HH=${CHH} +YYYP=${PYYYY} +MP=${PMM} +DP=${PDD} +HP=${PHH} +mem_ens="mem000" + +MEM_WORKDIR=${WORKDIR}/${mem_ens} +MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} +RSTRDIR=${MEM_WORKDIR} +JEDIWORKDIR=${WORKDIR}/mem000/jedi +FILEDATE=${YYYY}${MM}${DD}.${HH}0000 + +cd $JEDIWORKDIR + +# load modulefiles +#module use modulefiles; module load modules.landda + +OBSDIR=${LANDDA_INPUTS}/DA +################################################ +# 2. PREPARE OBS FILES +################################################ +for obs in "${OBS_TYPES[@]}"; do + # get the. obs file name + if [ ${obs} == "GTS" ]; then + obsfile=$OBSDIR/snow_depth/GTS/data_proc/${YYYY}${MM}/adpsfc_snow_${YYYY}${MM}${DD}${HH}.nc4 + # GHCN are time-stamped at 18. If assimilating at 00, need to use previous day's obs, so that + # obs are within DA window. + elif [ $ATMOS_FORC == "era5" ] && [ ${obs} == "GHCN" ]; then + obsfile=$OBSDIR/snow_depth/GHCN/data_proc/v3/${YYYY}/ghcn_snwd_ioda_${YYYP}${MP}${DP}.nc + elif [ $ATMOS_FORC == "gswp3" ] && [ ${obs} == "GHCN" ]; then + obsfile=$OBSDIR/snow_depth/GHCN/data_proc/v3/${YYYY}/fake_ghcn_snwd_ioda_${YYYP}${MP}${DP}.nc + elif [ ${obs} == "SYNTH" ]; then + obsfile=$OBSDIR/synthetic_noahmp/IODA.synthetic_gswp_obs.${YYYY}${MM}${DD}${HH}.nc + else + echo "do_landDA: Unknown obs type requested ${obs}, exiting" + exit 1 + fi + + # check obs are available + if [[ -e $obsfile ]]; then + echo "do_landDA: $i observations found: $obsfile" + ln -fs $obsfile ${obs}_${YYYY}${MM}${DD}${HH}.nc + else + echo "${obs} observations not found: $obsfile" + # JEDI_TYPES[$ii]="SKIP" + fi +done diff --git a/jobs/JLAND_RUN_ANA b/jobs/JLAND_RUN_ANA new file mode 100755 index 00000000..d4fef6cf --- /dev/null +++ b/jobs/JLAND_RUN_ANA @@ -0,0 +1,210 @@ +#!/bin/sh + +set -ex + +############################ +# copy restarts to workdir, convert to UFS tile for DA (all members) + +if [[ ${EXP_NAME} == "openloop" ]]; then + do_jedi="NO" +else + do_jedi="YES" + SAVE_TILE="YES" + LANDDADIR=${CYCLEDIR}/DA_update/ +fi + +TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ +YYYY=${CYYYY} +MM=${CMM} +DD=${CDD} +HH=${CHH} +YYYP=${PYYYY} +MP=${PMM} +DP=${PDD} +HP=${PHH} +mem_ens="mem000" + +MEM_WORKDIR=${WORKDIR}/${mem_ens} +MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} +RSTRDIR=${MEM_WORKDIR} +JEDIWORKDIR=${WORKDIR}/mem000/jedi +FILEDATE=${YYYY}${MM}${DD}.${HH}0000 +JEDI_STATICDIR=${JEDI_INSTALL}/jedi-bundle/fv3-jedi/test/Data +JEDI_EXECDIR=${JEDI_INSTALL}/build/bin +JEDI_EXEC=$JEDI_EXECDIR/fv3jedi_letkf.x +LOGDIR=${OUTDIR}/DA/logs +apply_incr_EXEC=${BUILDDIR}/apply_incr.exe +SAVE_INCR="YES" +KEEPJEDIDIR="YES" + +cd $MEM_WORKDIR + +# load modulefiles +module use modulefiles; module load modules.landda +PYTHON=$(/usr/bin/which python) +MPIEXEC=`which mpiexec` + +#fv3bundle_vn=psl_develop +#DAtype=letkfoi_snow +#SNOWDEPTHVAR=snwdph +YAML_DA=construct +GFSv17="NO" +B=30 # back ground error std for LETKFOI +cd $JEDIWORKDIR + +################################################ +# 3. DETERMINE REQUESTED JEDI TYPE, CONSTRUCT YAMLS +################################################ + +do_DA="YES" +do_HOFX="NO" + +if [[ $do_DA == "NO" && $do_HOFX == "NO" ]]; then + echo "do_landDA:No obs found, not calling JEDI" + exit 0 +fi + +# if yaml is specified by user, use that. Otherwise, build the yaml +if [[ $do_DA == "YES" ]]; then + + if [[ $YAML_DA == "construct" ]];then # construct the yaml + + cp ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${DAtype}.yaml ${JEDIWORKDIR}/letkf_land.yaml + + for obs in "${OBS_TYPES[@]}"; + do + cat ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${obs}.yaml >> letkf_land.yaml + done + + else # use specified yaml + echo "Using user specified YAML: ${YAML_DA}" + cp ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${YAML_DA} ${JEDIWORKDIR}/letkf_land.yaml + fi + + sed -i -e "s/XXYYYY/${YYYY}/g" letkf_land.yaml + sed -i -e "s/XXMM/${MM}/g" letkf_land.yaml + sed -i -e "s/XXDD/${DD}/g" letkf_land.yaml + sed -i -e "s/XXHH/${HH}/g" letkf_land.yaml + + sed -i -e "s/XXYYYP/${YYYP}/g" letkf_land.yaml + sed -i -e "s/XXMP/${MP}/g" letkf_land.yaml + sed -i -e "s/XXDP/${DP}/g" letkf_land.yaml + sed -i -e "s/XXHP/${HP}/g" letkf_land.yaml + + sed -i -e "s/XXTSTUB/${TSTUB}/g" letkf_land.yaml + sed -i -e "s#XXTPATH#${TPATH}#g" letkf_land.yaml + sed -i -e "s/XXRES/${RES}/g" letkf_land.yaml + RESP1=$((RES+1)) + sed -i -e "s/XXREP/${RESP1}/g" letkf_land.yaml + + sed -i -e "s/XXHOFX/false/g" letkf_land.yaml # do DA +fi + +if [[ $do_HOFX == "YES" ]]; then + + if [[ $YAML_HOFX == "construct" ]];then # construct the yaml + + cp ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${DAtype}.yaml ${JEDIWORKDIR}/hofx_land.yaml + + for obs in "${OBS_TYPES[@]}"; + do + cat ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${obs}.yaml >> hofx_land.yaml + done + else # use specified yaml + echo "Using user specified YAML: ${YAML_HOFX}" + cp ${LANDDADIR}/jedi/fv3-jedi/yaml_files/${fv3bundle_vn}/${YAML_HOFX} ${JEDIWORKDIR}/hofx_land.yaml + fi + + sed -i -e "s/XXYYYY/${YYYY}/g" hofx_land.yaml + sed -i -e "s/XXMM/${MM}/g" hofx_land.yaml + sed -i -e "s/XXDD/${DD}/g" hofx_land.yaml + sed -i -e "s/XXHH/${HH}/g" hofx_land.yaml + + sed -i -e "s/XXYYYP/${YYYP}/g" hofx_land.yaml + sed -i -e "s/XXMP/${MP}/g" hofx_land.yaml + sed -i -e "s/XXDP/${DP}/g" hofx_land.yaml + sed -i -e "s/XXHP/${HP}/g" hofx_land.yaml + + sed -i -e "s#XXTPATH#${TPATH}#g" hofx_land.yaml + sed -i -e "s/XXTSTUB/${TSTUB}/g" hofx_land.yaml + sed -i -e "s/XXRES/${RES}/g" hofx_land.yaml + RESP1=$((RES+1)) + sed -i -e "s/XXREP/${RESP1}/g" hofx_land.yaml + + sed -i -e "s/XXHOFX/true/g" hofx_land.yaml # do HOFX + +fi + +################################################ +# 4. CREATE BACKGROUND ENSEMBLE (LETKFOI) +################################################ +################################################ +# 5. RUN JEDI +################################################ + +if [[ ! -e Data ]]; then + ln -s $JEDI_STATICDIR Data +fi + +echo 'do_landDA: calling fv3-jedi' + +if [[ $do_DA == "YES" ]]; then + ${MPIEXEC} -n $NPROC_JEDI ${JEDI_EXEC} letkf_land.yaml ${LOGDIR}/jedi_letkf.log + if [[ $? != 0 ]]; then + echo "JEDI DA failed" + exit 10 + fi +fi +if [[ $do_HOFX == "YES" ]]; then + ${MPIEXEC} -n $NPROC_JEDI ${JEDI_EXEC} hofx_land.yaml ${LOGDIR}/jedi_hofx.log + if [[ $? != 0 ]]; then + echo "JEDI hofx failed" + exit 10 + fi +fi + +################################################ +# 6. APPLY INCREMENT TO UFS RESTARTS +################################################ + +if [[ $do_DA == "YES" ]]; then + + if [[ $DAtype == "letkfoi_snow" ]]; then + +cat << EOF > apply_incr_nml +&noahmp_snow + date_str=${YYYY}${MM}${DD} + hour_str=$HH + res=$RES + frac_grid=$GFSv17 + orog_path="$TPATH" + otype="$TSTUB" +/ +EOF + + echo 'do_landDA: calling apply snow increment' + + # (n=6) -> this is fixed, at one task per tile (with minor code change, could run on a single proc). + ${MPIEXEC} -n 6 ${apply_incr_EXEC} ${LOGDIR}/apply_incr.log + if [[ $? != 0 ]]; then + echo "apply snow increment failed" + exit 10 + fi + + fi + +fi + +################################################ +# 7. CLEAN UP +################################################ + +# keep increments +if [ $SAVE_INCR == "YES" ] && [ $do_DA == "YES" ]; then + cp ${JEDIWORKDIR}/${FILEDATE}.xainc.sfc_data.tile*.nc ${OUTDIR}/DA/jedi_incr/ +fi + +# clean up +if [[ $KEEPJEDIDIR == "NO" ]]; then + rm -rf ${JEDIWORKDIR} +fi diff --git a/parm/land_da_orion.xml b/parm/land_da_orion.xml index 8adfa2ed..df0f5f03 100644 --- a/parm/land_da_orion.xml +++ b/parm/land_da_orion.xml @@ -4,11 +4,17 @@ + + + + + + @@ -33,7 +39,7 @@ 1 batch 00:02:00 - &CYCLEDIR;/jobs/JGLOBAL_LAND_PREP + &CYCLEDIR;/jobs/JLAND_PREP_EXP prepexp MACHINE @@ -108,86 +114,277 @@ &PHOUR; - - GHCN - - epic - 1 - batch - 00:02:00 - &CYCLEDIR;/jobs/JGLOBAL_PREP_LAND_OBS - prepobs_#obstypes# - - OBS_TYPES - #obstypes# - - - MACHINE - &MACHINE; - - - EXP_NAME - &EXP_NAME; - - - LANDDA_INPUTS - &LANDDA_INPUTS; - - - ATMOS_FORC - &FORCING; - - - WORKDIR - &WORKDIR; - - - CYCLEDIR - &CYCLEDIR; - - - BUILDDIR - &BUILDDIR; - - - OUTDIR - &OUTDIR; - - - CYYYY - &CYEAR; - - - CMM - &CMONTH; - - - CDD - &CDATE; - - - CHH - &CHOUR; - - - PYYYY - &PYEAR; - - - PMM - &PMONTH; - - - PDD - &PDATE; - - - PHH - &PHOUR; - - - - - - + + epic + 1 + batch + 00:02:00 + &CYCLEDIR;/jobs/JLAND_PREP_OBS + prepobs + + OBS_TYPES + &OBS_TYPES; + + + MACHINE + &MACHINE; + + + EXP_NAME + &EXP_NAME; + + + LANDDA_INPUTS + &LANDDA_INPUTS; + + + ATMOS_FORC + &FORCING; + + + WORKDIR + &WORKDIR; + + + CYCLEDIR + &CYCLEDIR; + + + BUILDDIR + &BUILDDIR; + + + OUTDIR + &OUTDIR; + + + CYYYY + &CYEAR; + + + CMM + &CMONTH; + + + CDD + &CDATE; + + + CHH + &CHOUR; + + + PYYYY + &PYEAR; + + + PMM + &PMONTH; + + + PDD + &PDATE; + + + PHH + &PHOUR; + + + + + + + epic + 1 + batch + 00:02:00 + &CYCLEDIR;/jobs/JLAND_PREP_BMAT + prepbmat + + MACHINE + &MACHINE; + + + EXP_NAME + &EXP_NAME; + + + LANDDA_INPUTS + &LANDDA_INPUTS; + + + ATMOS_FORC + &FORCING; + + + WORKDIR + &WORKDIR; + + + CYCLEDIR + &CYCLEDIR; + + + BUILDDIR + &BUILDDIR; + + + OUTDIR + &OUTDIR; + + + CYYYY + &CYEAR; + + + CMM + &CMONTH; + + + CDD + &CDATE; + + + CHH + &CHOUR; + + + PYYYY + &PYEAR; + + + PMM + &PMONTH; + + + PDD + &PDATE; + + + PHH + &PHOUR; + + + fv3bundle_vn + &fv3bundle_vn; + + + DAtype + &DAtype; + + + SNOWDEPTHVAR + &SNOWDEPTHVAR; + + + + + + + epic + 1:ppn=&NPROCS; + batch + 00:15:00 + /work/noaa/epic/jongkim/land-DA/ana.sh + runana + + OBS_TYPES + &OBS_TYPES; + + + MACHINE + &MACHINE; + + + EXP_NAME + &EXP_NAME; + + + LANDDA_INPUTS + &LANDDA_INPUTS; + + + ATMOS_FORC + &FORCING; + + + RES + &RES; + + + TSTUB + &TSTUB; + + + WORKDIR + &WORKDIR; + + + CYCLEDIR + &CYCLEDIR; + + + BUILDDIR + &BUILDDIR; + + + OUTDIR + &OUTDIR; + + + CYYYY + &CYEAR; + + + CMM + &CMONTH; + + + CDD + &CDATE; + + + CHH + &CHOUR; + + + PYYYY + &PYEAR; + + + PMM + &PMONTH; + + + PDD + &PDATE; + + + PHH + &PHOUR; + + + fv3bundle_vn + &fv3bundle_vn; + + + DAtype + &DAtype; + + + SNOWDEPTHVAR + &SNOWDEPTHVAR; + + + NPROC_JEDI + &NPROCS; + + + JEDI_INSTALL + &JEDI_INSTALL; + + + + + diff --git a/parm/land_da_orion.yaml b/parm/land_da_orion.yaml index 16aafc28..df55cda1 100644 --- a/parm/land_da_orion.yaml +++ b/parm/land_da_orion.yaml @@ -11,11 +11,17 @@ workflow: LANDDA_INPUTS: "/work/noaa/epic/jongkim/land-DA/inputs" FORCING: "gswp3" RES: "96" + NPROCS: "6" + OBS_TYPES: "GHCN" + fv3bundle_vn: "psl_develop" + DAtype: "letkfoi_snow" + SNOWDEPTHVAR: "snwdph" MACHINE: orion TSTUB: "oro_C96.mx100" WORKDIR: /work/noaa/epic/jongkim/land-DA/workdir CYCLEDIR: /work/noaa/epic/jongkim/land-DA/land-DA_workflow BUILDDIR: /work/noaa/epic/jongkim/land-DA/land-DA_workflow/build/bin + JEDI_INSTALL: /work/noaa/epic/UFS_Land-DA/jedi OUTDIR: /work/noaa/epic/jongkim/land-DA/landda_expts/DA_GSWP3_test LOG: /work/noaa/epic/jongkim/land-DA/tests PATHRT: /work/noaa/epic/jongkim/land-DA @@ -54,40 +60,104 @@ workflow: PDD: "&PDATE;" PHH: "&PHOUR;" account: epic - command: "&CYCLEDIR;/jobs/JGLOBAL_LAND_PREP" + command: "&CYCLEDIR;/jobs/JLAND_PREP_EXP" jobname: prepexp cores: 1 walltime: 00:02:00 queue: batch - metatask_prepobs: - var: - obstypes: GHCN - task_prepobs_#obstypes#: - account: epic - envars: - OBS_TYPES: "#obstypes#" - MACHINE: "&MACHINE;" - EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" - ATMOS_FORC: "&FORCING;" - WORKDIR: "&WORKDIR;" - CYCLEDIR: "&CYCLEDIR;" - BUILDDIR: "&BUILDDIR;" - OUTDIR: "&OUTDIR;" - CYYYY: "&CYEAR;" - CMM: "&CMONTH;" - CDD: "&CDATE;" - CHH: "&CHOUR;" - PYYYY: "&PYEAR;" - PMM: "&PMONTH;" - PDD: "&PDATE;" - PHH: "&PHOUR;" - command: "&CYCLEDIR;/jobs/JGLOBAL_PREP_LAND_OBS" - jobname: prepobs_#obstypes# - cores: 1 - walltime: 00:02:00 - queue: batch - dependency: - taskdep: - attrs: - task: prepexp \ No newline at end of file + task_prepobs: + envars: + OBS_TYPES: "&OBS_TYPES;" + MACHINE: "&MACHINE;" + EXP_NAME: "&EXP_NAME;" + LANDDA_INPUTS: "&LANDDA_INPUTS;" + ATMOS_FORC: "&FORCING;" + WORKDIR: "&WORKDIR;" + CYCLEDIR: "&CYCLEDIR;" + BUILDDIR: "&BUILDDIR;" + OUTDIR: "&OUTDIR;" + CYYYY: "&CYEAR;" + CMM: "&CMONTH;" + CDD: "&CDATE;" + CHH: "&CHOUR;" + PYYYY: "&PYEAR;" + PMM: "&PMONTH;" + PDD: "&PDATE;" + PHH: "&PHOUR;" + account: epic + command: "&CYCLEDIR;/jobs/JLAND_PREP_OBS" + jobname: prepobs + cores: 1 + walltime: 00:02:00 + queue: batch + dependency: + taskdep: + attrs: + task: prepexp + task_prepbmat: + envars: + MACHINE: "&MACHINE;" + EXP_NAME: "&EXP_NAME;" + LANDDA_INPUTS: "&LANDDA_INPUTS;" + ATMOS_FORC: "&FORCING;" + WORKDIR: "&WORKDIR;" + CYCLEDIR: "&CYCLEDIR;" + BUILDDIR: "&BUILDDIR;" + OUTDIR: "&OUTDIR;" + CYYYY: "&CYEAR;" + CMM: "&CMONTH;" + CDD: "&CDATE;" + CHH: "&CHOUR;" + PYYYY: "&PYEAR;" + PMM: "&PMONTH;" + PDD: "&PDATE;" + PHH: "&PHOUR;" + fv3bundle_vn: "&fv3bundle_vn;" + DAtype: "&DAtype;" + SNOWDEPTHVAR: "&SNOWDEPTHVAR;" + account: epic + command: "&CYCLEDIR;/jobs/JLAND_PREP_BMAT" + jobname: prepbmat + cores: 1 + walltime: 00:02:00 + queue: batch + dependency: + taskdep: + attrs: + task: prepobs + task_runana: + envars: + OBS_TYPES: "&OBS_TYPES;" + MACHINE: "&MACHINE;" + EXP_NAME: "&EXP_NAME;" + LANDDA_INPUTS: "&LANDDA_INPUTS;" + ATMOS_FORC: "&FORCING;" + RES: "&RES;" + TSTUB: "&TSTUB;" + WORKDIR: "&WORKDIR;" + CYCLEDIR: "&CYCLEDIR;" + BUILDDIR: "&BUILDDIR;" + OUTDIR: "&OUTDIR;" + CYYYY: "&CYEAR;" + CMM: "&CMONTH;" + CDD: "&CDATE;" + CHH: "&CHOUR;" + PYYYY: "&PYEAR;" + PMM: "&PMONTH;" + PDD: "&PDATE;" + PHH: "&PHOUR;" + fv3bundle_vn: "&fv3bundle_vn;" + DAtype: "&DAtype;" + SNOWDEPTHVAR: "&SNOWDEPTHVAR;" + NPROC_JEDI: "&NPROCS;" + JEDI_INSTALL: "&JEDI_INSTALL;" + account: epic + command: "/work/noaa/epic/jongkim/land-DA/ana.sh" + jobname: runana + nodes: "1:ppn=&NPROCS;" + walltime: 00:15:00 + queue: batch + dependency: + taskdep: + attrs: + task: prepbmat From d24bb0ef22263efcb3a595eb565de95f44d129d4 Mon Sep 17 00:00:00 2001 From: JONG KIM Date: Thu, 15 Feb 2024 09:25:30 -0500 Subject: [PATCH 3/5] Update land_da_orion.yaml --- parm/land_da_orion.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/land_da_orion.yaml b/parm/land_da_orion.yaml index df55cda1..4cde56ae 100644 --- a/parm/land_da_orion.yaml +++ b/parm/land_da_orion.yaml @@ -152,7 +152,7 @@ workflow: NPROC_JEDI: "&NPROCS;" JEDI_INSTALL: "&JEDI_INSTALL;" account: epic - command: "/work/noaa/epic/jongkim/land-DA/ana.sh" + command: "&CYCLEDIR;/jobs/JLAND_RUN_ANA" jobname: runana nodes: "1:ppn=&NPROCS;" walltime: 00:15:00 From 1dd2353acdc06fea420c65e971dcaa398d083b1c Mon Sep 17 00:00:00 2001 From: jkbk2004 Date: Fri, 16 Feb 2024 07:59:06 -0600 Subject: [PATCH 4/5] add run fcst script --- jobs/JLAND_RUN_FCST | 262 ++++++++++++++++++++++++++++++++++++++++ parm/land_da_orion.xml | 146 +++++++++++++++++++++- parm/land_da_orion.yaml | 48 +++++++- 3 files changed, 454 insertions(+), 2 deletions(-) create mode 100755 jobs/JLAND_RUN_FCST diff --git a/jobs/JLAND_RUN_FCST b/jobs/JLAND_RUN_FCST new file mode 100755 index 00000000..1846ea28 --- /dev/null +++ b/jobs/JLAND_RUN_FCST @@ -0,0 +1,262 @@ +#!/bin/sh + +set -ex + +############################ +# copy restarts to workdir, convert to UFS tile for DA (all members) + +if [[ ${EXP_NAME} == "openloop" ]]; then + do_jedi="NO" +else + do_jedi="YES" + SAVE_TILE="YES" + LANDDADIR=${CYCLEDIR}/DA_update/ +fi + +MACHINE_ID=${MACHINE} +TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ +YYYY=${CYYYY} +MM=${CMM} +DD=${CDD} +HH=${CHH} +YYYP=${PYYYY} +MP=${PMM} +DP=${PDD} +HP=${PHH} +nYYYY=${NYYYY} +nMM=${NMM} +nDD=${NDD} +nHH=${NHH} +mem_ens="mem000" + +MEM_WORKDIR=${WORKDIR}/${mem_ens} +MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} +RSTRDIR=${MEM_WORKDIR} +JEDIWORKDIR=${WORKDIR}/mem000/jedi +FILEDATE=${YYYY}${MM}${DD}.${HH}0000 +JEDI_STATICDIR=${JEDI_INSTALL}/jedi-bundle/fv3-jedi/test/Data +JEDI_EXECDIR=${JEDI_INSTALL}/build/bin +JEDI_EXEC=$JEDI_EXECDIR/fv3jedi_letkf.x +LOGDIR=${OUTDIR}/DA/logs +apply_incr_EXEC=${BUILDDIR}/apply_incr.exe +SAVE_INCR="YES" +KEEPJEDIDIR="YES" +FREQ=$((${FCSTHR}*3600)) +RDD=$((${FCSTHR}/24)) +RHH=$((${FCSTHR}%24)) + +cd $MEM_WORKDIR + +# load modulefiles +module use modulefiles; module load modules.landda +PYTHON=$(/usr/bin/which python) +MPIEXEC=`which mpiexec` +MPIRUN=${MPIRUN:-`which mpiexec`} + +#fv3bundle_vn=psl_develop +#DAtype=letkfoi_snow +#SNOWDEPTHVAR=snwdph + +cd $MEM_WORKDIR + +# convert back to vector, run model (all members) convert back to vector, run model (all members) +if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then + echo '************************************************' + echo 'calling tile2vector' + + cp ${CYCLEDIR}/template.tile2vector tile2vector.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" tile2vector.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" tile2vector.namelist + sed -i -e "s/XXMM/${MM}/g" tile2vector.namelist + sed -i -e "s/XXDD/${DD}/g" tile2vector.namelist + sed -i -e "s/XXHH/${HH}/g" tile2vector.namelist + sed -i -e "s/MODEL_FORCING/${ATMOS_FORC}/g" vector2tile.namelist + sed -i -e "s/XXRES/${RES}/g" tile2vector.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" tile2vector.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" tile2vector.namelist + + ${BUILDDIR}/vector2tile_converter.exe tile2vector.namelist + if [[ $? != 0 ]]; then + echo "tile2vector failed" + exit + fi + + # save analysis restart + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.nc +fi + +# convert back to UFS tile, run model (all members) +if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then + echo '************************************************' + echo 'calling tile2tile' + + cp ${CYCLEDIR}/template.jedi2ufs jedi2ufs.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" jedi2ufs.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" jedi2ufs.namelist + sed -i -e "s/XXMM/${MM}/g" jedi2ufs.namelist + sed -i -e "s/XXDD/${DD}/g" jedi2ufs.namelist + sed -i -e "s/XXHH/${HH}/g" jedi2ufs.namelist + sed -i -e "s/MODEL_FORCING/${ATMOS_FORC}/g" jedi2ufs.namelist + sed -i -e "s/XXRES/${RES}/g" jedi2ufs.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" jedi2ufs.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" jedi2ufs.namelist + + ${BUILDDIR}/tile2tile_converter.exe jedi2ufs.namelist + if [[ $? != 0 ]]; then + echo "tile2tile failed" + exit + fi + + # save analysis restart + for tile in 1 2 3 4 5 6 + do + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc + done +fi + + +#jkimmmm +############################ +# run the forecast model + +if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then + echo '************************************************' + echo 'running the forecast model' + + set -x + + # update model namelist + cp ${CYCLEDIR}/template.ufs-noahMP.namelist.${atmos_forc} ufs-land.namelist + + sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs-land.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" ufs-land.namelist + sed -i -e "s/XXMM/${MM}/g" ufs-land.namelist + sed -i -e "s/XXDD/${DD}/g" ufs-land.namelist + sed -i -e "s/XXHH/${HH}/g" ufs-land.namelist + sed -i -e "s/XXFREQ/${FREQ}/g" ufs-land.namelist + sed -i -e "s/XXRDD/${RDD}/g" ufs-land.namelist + sed -i -e "s/XXRHH/${RHH}/g" ufs-land.namelist + + # submit model + echo $MEM_WORKDIR + + nt=$SLURM_NTASKS + + ${MPIEXEC} -n 1 ${CYCLEDIR}/build/bin/ufsLand.exe +fi +# no error codes on exit from model, check for restart below instead + +if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then + set -e + + echo '************************************************' + echo 'running the forecast model' + + TEST_NAME=datm_cdeps_lnd_gswp3 + TEST_NAME_RST=datm_cdeps_lnd_gswp3_rst + PATHRT=${CYCLEDIR}/ufs-weather-model/tests + RT_COMPILER=${RT_COMPILER:-intel} + ATOL="1e-7" + + cp $CYCLEDIR/$TEST_NAME_RST ${PATHRT}/tests/$TEST_NAME_RST + source ${PATHRT}/rt_utils.sh + source ${PATHRT}/default_vars.sh + source ${PATHRT}/tests/$TEST_NAME_RST + source ${PATHRT}/atparse.bash + + BL_DATE=20230816 + RTPWD=${RTPWD:-${LANDDA_INPUTS}/NEMSfv3gfs/develop-${BL_DATE}/INTEL/${TEST_NAME}} + INPUTDATA_ROOT=${INPUTDATA_ROOT:-${LANDDA_INPUTS}/NEMSfv3gfs/input-data-20221101} + + echo "RTPWD= $RTPWD" + echo "INPUTDATA_ROOT= $INPUTDATA_ROOT" + + if [[ ! -d ${INPUTDATA_ROOT} ]] || [[ ! -d ${RTPWD} ]]; then + echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!" + exit 1 + fi + + # create run folder + RUNDIR=${MEM_MODL_OUTDIR}/noahmp/${TEST_NAME_RST} + [[ -d ${RUNDIR} ]] && echo "Warning: remove old run folder!" && rm -rf ${RUNDIR} + mkdir -p ${RUNDIR} + cd ${RUNDIR} + + echo "NoahMP run dir= $RUNDIR" + + # modify some env variables - reduce core usage + export ATM_compute_tasks=0 + export ATM_io_tasks=1 + export LND_tasks=6 + export layout_x=1 + export layout_y=1 + + # FV3 executable: + cp ${CYCLEDIR}/build/ufs-weather-model/src/ufs-weather-model-build/ufs_model ./ufs_model + cp ${CYCLEDIR}/fv3_run ./fv3_run + + if [[ $DATM_CDEPS = 'true' ]] || [[ $FV3 = 'true' ]] || [[ $S2S = 'true' ]]; then + if [[ $HAFS = 'false' ]] || [[ $FV3 = 'true' && $HAFS = 'true' ]]; then + atparse < ${PATHRT}/parm/${INPUT_NML:-input.nml.IN} > input.nml + fi + fi + + atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure + + compute_petbounds_and_tasks + + atparse < ${PATHRT}/parm/${UFS_CONFIGURE:-ufs.configure} > ufs.configure + + # diag table + if [[ "Q${DIAG_TABLE:-}" != Q ]] ; then + atparse < ${PATHRT}/parm/diag_table/${DIAG_TABLE} > diag_table + fi + + # Field table + if [[ "Q${FIELD_TABLE:-}" != Q ]] ; then + cp ${PATHRT}/parm/field_table/${FIELD_TABLE} field_table + fi + + # Field Dictionary + cp ${PATHRT}/parm/fd_ufs.yaml fd_ufs.yaml + + # Set up the run directory + source ./fv3_run + + if [[ $DATM_CDEPS = 'true' ]]; then + atparse < ${PATHRT}/parm/${DATM_IN_CONFIGURE:-datm_in.IN} > datm_in + atparse < ${PATHRT}/parm/${DATM_STREAM_CONFIGURE:-datm.streams.IN} > datm.streams + fi + + # NoahMP table file + cp ${PATHRT}/parm/noahmptable.tbl noahmptable.tbl + + # start runs + echo "Start ufs-cdeps-land model run with TASKS: ${TASKS}" + ${MPIRUN} -n ${TASKS} ./ufs_model +fi + +# no error codes on exit from model, check for restart below instead + +############################ +# check model ouput (all members) + +mem_ens="mem000" + +MEM_WORKDIR=${WORKDIR}/${mem_ens} +MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} + +if [[ ${ATMOS_FORC} == "era5" ]]; then + if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then + cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc +else + for tile in 1 2 3 4 5 6 + do + cp ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc + done +fi + + diff --git a/parm/land_da_orion.xml b/parm/land_da_orion.xml index df0f5f03..d8974d89 100644 --- a/parm/land_da_orion.xml +++ b/parm/land_da_orion.xml @@ -4,7 +4,8 @@ - + + @@ -363,6 +364,22 @@ PHH &PHOUR; + + NYYYY + &NYEAR; + + + NMM + &NMONTH; + + + NDD + &NDATE; + + + NHH + &NHOUR; + fv3bundle_vn &fv3bundle_vn; @@ -387,4 +404,131 @@ + + epic + 1:ppn=&NPROCS; + batch + 00:30:00 + /work/noaa/epic/jongkim/land-DA/fcst.sh + runfcst + + OBS_TYPES + &OBS_TYPES; + + + MACHINE + &MACHINE; + + + EXP_NAME + &EXP_NAME; + + + LANDDA_INPUTS + &LANDDA_INPUTS; + + + ATMOS_FORC + &FORCING; + + + RES + &RES; + + + TSTUB + &TSTUB; + + + WORKDIR + &WORKDIR; + + + CYCLEDIR + &CYCLEDIR; + + + BUILDDIR + &BUILDDIR; + + + OUTDIR + &OUTDIR; + + + CYYYY + &CYEAR; + + + CMM + &CMONTH; + + + CDD + &CDATE; + + + CHH + &CHOUR; + + + PYYYY + &PYEAR; + + + PMM + &PMONTH; + + + PDD + &PDATE; + + + PHH + &PHOUR; + + + NYYYY + &NYEAR; + + + NMM + &NMONTH; + + + NDD + &NDATE; + + + NHH + &NHOUR; + + + fv3bundle_vn + &fv3bundle_vn; + + + DAtype + &DAtype; + + + SNOWDEPTHVAR + &SNOWDEPTHVAR; + + + NPROC_JEDI + &NPROCS; + + + JEDI_INSTALL + &JEDI_INSTALL; + + + FCSTHR + &FCSTHR; + + + + + diff --git a/parm/land_da_orion.yaml b/parm/land_da_orion.yaml index 4cde56ae..efadc604 100644 --- a/parm/land_da_orion.yaml +++ b/parm/land_da_orion.yaml @@ -11,7 +11,8 @@ workflow: LANDDA_INPUTS: "/work/noaa/epic/jongkim/land-DA/inputs" FORCING: "gswp3" RES: "96" - NPROCS: "6" + FCSTHR: "24" + NPROCS: "7" OBS_TYPES: "GHCN" fv3bundle_vn: "psl_develop" DAtype: "letkfoi_snow" @@ -146,6 +147,10 @@ workflow: PMM: "&PMONTH;" PDD: "&PDATE;" PHH: "&PHOUR;" + NYYYY: "&NYEAR;" + NMM: "&NMONTH;" + NDD: "&NDATE;" + NHH: "&NHOUR;" fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" @@ -161,3 +166,44 @@ workflow: taskdep: attrs: task: prepbmat + task_runfcst: + envars: + OBS_TYPES: "&OBS_TYPES;" + MACHINE: "&MACHINE;" + EXP_NAME: "&EXP_NAME;" + LANDDA_INPUTS: "&LANDDA_INPUTS;" + ATMOS_FORC: "&FORCING;" + RES: "&RES;" + TSTUB: "&TSTUB;" + WORKDIR: "&WORKDIR;" + CYCLEDIR: "&CYCLEDIR;" + BUILDDIR: "&BUILDDIR;" + OUTDIR: "&OUTDIR;" + CYYYY: "&CYEAR;" + CMM: "&CMONTH;" + CDD: "&CDATE;" + CHH: "&CHOUR;" + PYYYY: "&PYEAR;" + PMM: "&PMONTH;" + PDD: "&PDATE;" + PHH: "&PHOUR;" + NYYYY: "&NYEAR;" + NMM: "&NMONTH;" + NDD: "&NDATE;" + NHH: "&NHOUR;" + fv3bundle_vn: "&fv3bundle_vn;" + DAtype: "&DAtype;" + SNOWDEPTHVAR: "&SNOWDEPTHVAR;" + NPROC_JEDI: "&NPROCS;" + JEDI_INSTALL: "&JEDI_INSTALL;" + FCSTHR: "&FCSTHR;" + account: epic + command: "/work/noaa/epic/jongkim/land-DA/fcst.sh" + jobname: runfcst + nodes: "1:ppn=&NPROCS;" + walltime: 00:30:00 + queue: batch + dependency: + taskdep: + attrs: + task: runana From 6a4077a0a78c28c2304385aa99ed63820d60dc3b Mon Sep 17 00:00:00 2001 From: jkbk2004 Date: Sat, 17 Feb 2024 12:39:26 -0600 Subject: [PATCH 5/5] update jjobs yaml --- jobs/JLAND_PREP_BMAT | 13 +- jobs/JLAND_PREP_EXP | 18 +- jobs/JLAND_PREP_OBS | 17 +- jobs/JLAND_RUN_ANA | 17 +- jobs/JLAND_RUN_FCST | 49 ++-- parm/{land_da_orion.xml => land_analysis.xml} | 228 +++--------------- ...{land_da_orion.yaml => land_analysis.yaml} | 95 ++------ 7 files changed, 118 insertions(+), 319 deletions(-) rename parm/{land_da_orion.xml => land_analysis.xml} (63%) rename parm/{land_da_orion.yaml => land_analysis.yaml} (64%) diff --git a/jobs/JLAND_PREP_BMAT b/jobs/JLAND_PREP_BMAT index 3ea6b3d2..7126e6ec 100755 --- a/jobs/JLAND_PREP_BMAT +++ b/jobs/JLAND_PREP_BMAT @@ -14,14 +14,11 @@ else fi TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ -YYYY=${CYYYY} -MM=${CMM} -DD=${CDD} -HH=${CHH} -YYYP=${PYYYY} -MP=${PMM} -DP=${PDD} -HP=${PHH} +YYYY=${CTIME:0:4} +MM=${CTIME:4:2} +DD=${CTIME:6:2} +HH=${CTIME:8:2} + mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} diff --git a/jobs/JLAND_PREP_EXP b/jobs/JLAND_PREP_EXP index 1dac5800..3a8d1f84 100755 --- a/jobs/JLAND_PREP_EXP +++ b/jobs/JLAND_PREP_EXP @@ -15,14 +15,14 @@ fi echo ${LANDDA_INPUTS}, ${ATMOS_FORC} TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ -YYYY=${CYYYY} -MM=${CMM} -DD=${CDD} -HH=${CHH} -YYYP=${PYYYY} -MP=${PMM} -DP=${PDD} -HP=${PHH} +YYYY=${CTIME:0:4} +MM=${CTIME:4:2} +DD=${CTIME:6:2} +HH=${CTIME:8:2} +YYYP=${PTIME:0:4} +MP=${PTIME:4:2} +DP=${PTIME:6:2} +HP=${PTIME:8:2} mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} @@ -77,7 +77,7 @@ if [[ $do_jedi == "YES" && $ATMOS_FORC == "era5" ]]; then echo '************************************************' echo 'calling vector2tile' - ${BUILDDIR}/tile2tile_converter.exe vector2tile.namelist + ${BUILDDIR}/vector2tile_converter.exe vector2tile.namelist if [[ $? != 0 ]]; then echo "vec2tile failed" exit diff --git a/jobs/JLAND_PREP_OBS b/jobs/JLAND_PREP_OBS index 950c48c1..6d13b23a 100755 --- a/jobs/JLAND_PREP_OBS +++ b/jobs/JLAND_PREP_OBS @@ -14,14 +14,15 @@ else fi TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ -YYYY=${CYYYY} -MM=${CMM} -DD=${CDD} -HH=${CHH} -YYYP=${PYYYY} -MP=${PMM} -DP=${PDD} -HP=${PHH} +YYYY=${CTIME:0:4} +MM=${CTIME:4:2} +DD=${CTIME:6:2} +HH=${CTIME:8:2} +YYYP=${PTIME:0:4} +MP=${PTIME:4:2} +DP=${PTIME:6:2} +HP=${PTIME:8:2} + mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} diff --git a/jobs/JLAND_RUN_ANA b/jobs/JLAND_RUN_ANA index d4fef6cf..3a5e0dcc 100755 --- a/jobs/JLAND_RUN_ANA +++ b/jobs/JLAND_RUN_ANA @@ -14,14 +14,15 @@ else fi TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ -YYYY=${CYYYY} -MM=${CMM} -DD=${CDD} -HH=${CHH} -YYYP=${PYYYY} -MP=${PMM} -DP=${PDD} -HP=${PHH} +YYYY=${CTIME:0:4} +MM=${CTIME:4:2} +DD=${CTIME:6:2} +HH=${CTIME:8:2} +YYYP=${PTIME:0:4} +MP=${PTIME:4:2} +DP=${PTIME:6:2} +HP=${PTIME:8:2} + mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} diff --git a/jobs/JLAND_RUN_FCST b/jobs/JLAND_RUN_FCST index 1846ea28..5c4276bb 100755 --- a/jobs/JLAND_RUN_FCST +++ b/jobs/JLAND_RUN_FCST @@ -15,18 +15,18 @@ fi MACHINE_ID=${MACHINE} TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ -YYYY=${CYYYY} -MM=${CMM} -DD=${CDD} -HH=${CHH} -YYYP=${PYYYY} -MP=${PMM} -DP=${PDD} -HP=${PHH} -nYYYY=${NYYYY} -nMM=${NMM} -nDD=${NDD} -nHH=${NHH} +YYYY=${CTIME:0:4} +MM=${CTIME:4:2} +DD=${CTIME:6:2} +HH=${CTIME:8:2} +YYYP=${PTIME:0:4} +MP=${PTIME:4:2} +DP=${PTIME:6:2} +HP=${PTIME:8:2} +nYYYY=${NTIME:0:4} +nMM=${NTIME:4:2} +nDD=${NTIME:6:2} +nHH=${NTIME:8:2} mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} @@ -129,7 +129,7 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then set -x # update model namelist - cp ${CYCLEDIR}/template.ufs-noahMP.namelist.${atmos_forc} ufs-land.namelist + cp ${CYCLEDIR}/template.ufs-noahMP.namelist.${ATMOS_FORC} ufs-land.namelist sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs-land.namelist sed -i -e "s/XXYYYY/${YYYY}/g" ufs-land.namelist @@ -244,19 +244,20 @@ fi ############################ # check model ouput (all members) -mem_ens="mem000" +#mem_ens="mem000" -MEM_WORKDIR=${WORKDIR}/${mem_ens} -MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} +#MEM_WORKDIR=${WORKDIR}/${mem_ens} +#MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} if [[ ${ATMOS_FORC} == "era5" ]]; then - if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then - cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc -else - for tile in 1 2 3 4 5 6 - do - cp ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc - done + if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then + cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc + fi fi - +if [[ ${ATMOS_FORC} == "gswp3" ]]; then + for tile in 1 2 3 4 5 6 + do + cp ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc + done +fi diff --git a/parm/land_da_orion.xml b/parm/land_analysis.xml similarity index 63% rename from parm/land_da_orion.xml rename to parm/land_analysis.xml index d8974d89..ff5a37ef 100644 --- a/parm/land_da_orion.xml +++ b/parm/land_analysis.xml @@ -2,10 +2,11 @@ - + - + + @@ -19,21 +20,12 @@ - @Y"> - @m"> - @d"> - @H"> - @Y"> - @m"> - @d"> - @H"> - @Y"> - @m"> - @d"> - @H"> + @Y@m@d@H"> + @Y@m@d@H"> + @Y@m@d@H"> ]> - 200001030000 200001030000 24:00:00 + 201912210000 201912210000 24:00:00 /work/noaa/epic/jongkim/land-DA/tests/workflow.log; epic @@ -83,36 +75,12 @@ &OUTDIR; - CYYYY - &CYEAR; + CTIME + &CTIME; - CMM - &CMONTH; - - - CDD - &CDATE; - - - CHH - &CHOUR; - - - PYYYY - &PYEAR; - - - PMM - &PMONTH; - - - PDD - &PDATE; - - - PHH - &PHOUR; + PTIME + &PTIME; @@ -159,36 +127,12 @@ &OUTDIR; - CYYYY - &CYEAR; - - - CMM - &CMONTH; - - - CDD - &CDATE; - - - CHH - &CHOUR; - - - PYYYY - &PYEAR; - - - PMM - &PMONTH; - - - PDD - &PDATE; + CTIME + &CTIME; - PHH - &PHOUR; + PTIME + &PTIME; @@ -234,36 +178,12 @@ &OUTDIR; - CYYYY - &CYEAR; + CTIME + &CTIME; - CMM - &CMONTH; - - - CDD - &CDATE; - - - CHH - &CHOUR; - - - PYYYY - &PYEAR; - - - PMM - &PMONTH; - - - PDD - &PDATE; - - - PHH - &PHOUR; + PTIME + &PTIME; fv3bundle_vn @@ -283,10 +203,10 @@ epic - 1:ppn=&NPROCS; + 1:ppn=&NPROCS_ANA; batch 00:15:00 - /work/noaa/epic/jongkim/land-DA/ana.sh + &CYCLEDIR;/jobs/JLAND_PREP_OBS runana OBS_TYPES @@ -333,52 +253,16 @@ &OUTDIR; - CYYYY - &CYEAR; - - - CMM - &CMONTH; - - - CDD - &CDATE; - - - CHH - &CHOUR; + CTIME + &CTIME; - PYYYY - &PYEAR; + PTIME + &PTIME; - PMM - &PMONTH; - - - PDD - &PDATE; - - - PHH - &PHOUR; - - - NYYYY - &NYEAR; - - - NMM - &NMONTH; - - - NDD - &NDATE; - - - NHH - &NHOUR; + NTIME + &NTIME; fv3bundle_vn @@ -394,7 +278,7 @@ NPROC_JEDI - &NPROCS; + &NPROCS_ANA; JEDI_INSTALL @@ -406,10 +290,10 @@ epic - 1:ppn=&NPROCS; + 1:ppn=&NPROCS_FCST; batch 00:30:00 - /work/noaa/epic/jongkim/land-DA/fcst.sh + &CYCLEDIR;/jobs/JLAND_RUN_FCST runfcst OBS_TYPES @@ -456,52 +340,16 @@ &OUTDIR; - CYYYY - &CYEAR; - - - CMM - &CMONTH; + CTIME + &CTIME; - CDD - &CDATE; + PTIME + &PTIME; - CHH - &CHOUR; - - - PYYYY - &PYEAR; - - - PMM - &PMONTH; - - - PDD - &PDATE; - - - PHH - &PHOUR; - - - NYYYY - &NYEAR; - - - NMM - &NMONTH; - - - NDD - &NDATE; - - - NHH - &NHOUR; + NTIME + &NTIME; fv3bundle_vn @@ -515,10 +363,6 @@ SNOWDEPTHVAR &SNOWDEPTHVAR; - - NPROC_JEDI - &NPROCS; - JEDI_INSTALL &JEDI_INSTALL; diff --git a/parm/land_da_orion.yaml b/parm/land_analysis.yaml similarity index 64% rename from parm/land_da_orion.yaml rename to parm/land_analysis.yaml index efadc604..5f7daf75 100644 --- a/parm/land_da_orion.yaml +++ b/parm/land_analysis.yaml @@ -5,14 +5,15 @@ workflow: cycledef: - attrs: group: epic - spec: 200001030000 200001030000 24:00:00 + spec: 201912210000 201912210000 24:00:00 entities: EXP_NAME: "LETKF" LANDDA_INPUTS: "/work/noaa/epic/jongkim/land-DA/inputs" - FORCING: "gswp3" + FORCING: "era5" RES: "96" FCSTHR: "24" - NPROCS: "7" + NPROCS_ANA: "6" + NPROCS_FCST: "6" OBS_TYPES: "GHCN" fv3bundle_vn: "psl_develop" DAtype: "letkfoi_snow" @@ -26,18 +27,9 @@ workflow: OUTDIR: /work/noaa/epic/jongkim/land-DA/landda_expts/DA_GSWP3_test LOG: /work/noaa/epic/jongkim/land-DA/tests PATHRT: /work/noaa/epic/jongkim/land-DA - CYEAR: "@Y" - CMONTH: "@m" - CDATE: "@d" - CHOUR: "@H" - PYEAR: "@Y" - PMONTH: "@m" - PDATE: "@d" - PHOUR: "@H" - NYEAR: "@Y" - NMONTH: "@m" - NDATE: "@d" - NHOUR: "@H" + CTIME: "@Y@m@d@H" + PTIME: "@Y@m@d@H" + NTIME: "@Y@m@d@H" log: /work/noaa/epic/jongkim/land-DA/tests/workflow.log; tasks: task_prepexp: @@ -52,14 +44,8 @@ workflow: CYCLEDIR: "&CYCLEDIR;" BUILDDIR: "&BUILDDIR;" OUTDIR: "&OUTDIR;" - CYYYY: "&CYEAR;" - CMM: "&CMONTH;" - CDD: "&CDATE;" - CHH: "&CHOUR;" - PYYYY: "&PYEAR;" - PMM: "&PMONTH;" - PDD: "&PDATE;" - PHH: "&PHOUR;" + CTIME: "&CTIME;" + PTIME: "&PTIME;" account: epic command: "&CYCLEDIR;/jobs/JLAND_PREP_EXP" jobname: prepexp @@ -77,14 +63,8 @@ workflow: CYCLEDIR: "&CYCLEDIR;" BUILDDIR: "&BUILDDIR;" OUTDIR: "&OUTDIR;" - CYYYY: "&CYEAR;" - CMM: "&CMONTH;" - CDD: "&CDATE;" - CHH: "&CHOUR;" - PYYYY: "&PYEAR;" - PMM: "&PMONTH;" - PDD: "&PDATE;" - PHH: "&PHOUR;" + CTIME: "&CTIME;" + PTIME: "&PTIME;" account: epic command: "&CYCLEDIR;/jobs/JLAND_PREP_OBS" jobname: prepobs @@ -105,14 +85,8 @@ workflow: CYCLEDIR: "&CYCLEDIR;" BUILDDIR: "&BUILDDIR;" OUTDIR: "&OUTDIR;" - CYYYY: "&CYEAR;" - CMM: "&CMONTH;" - CDD: "&CDATE;" - CHH: "&CHOUR;" - PYYYY: "&PYEAR;" - PMM: "&PMONTH;" - PDD: "&PDATE;" - PHH: "&PHOUR;" + CTIME: "&CTIME;" + PTIME: "&PTIME;" fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" @@ -139,27 +113,18 @@ workflow: CYCLEDIR: "&CYCLEDIR;" BUILDDIR: "&BUILDDIR;" OUTDIR: "&OUTDIR;" - CYYYY: "&CYEAR;" - CMM: "&CMONTH;" - CDD: "&CDATE;" - CHH: "&CHOUR;" - PYYYY: "&PYEAR;" - PMM: "&PMONTH;" - PDD: "&PDATE;" - PHH: "&PHOUR;" - NYYYY: "&NYEAR;" - NMM: "&NMONTH;" - NDD: "&NDATE;" - NHH: "&NHOUR;" + CTIME: "&CTIME;" + PTIME: "&PTIME;" + NTIME: "&NTIME;" fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" - NPROC_JEDI: "&NPROCS;" + NPROC_JEDI: "&NPROCS_ANA;" JEDI_INSTALL: "&JEDI_INSTALL;" account: epic - command: "&CYCLEDIR;/jobs/JLAND_RUN_ANA" + command: "&CYCLEDIR;/jobs/JLAND_PREP_OBS" jobname: runana - nodes: "1:ppn=&NPROCS;" + nodes: "1:ppn=&NPROCS_ANA;" walltime: 00:15:00 queue: batch dependency: @@ -179,31 +144,21 @@ workflow: CYCLEDIR: "&CYCLEDIR;" BUILDDIR: "&BUILDDIR;" OUTDIR: "&OUTDIR;" - CYYYY: "&CYEAR;" - CMM: "&CMONTH;" - CDD: "&CDATE;" - CHH: "&CHOUR;" - PYYYY: "&PYEAR;" - PMM: "&PMONTH;" - PDD: "&PDATE;" - PHH: "&PHOUR;" - NYYYY: "&NYEAR;" - NMM: "&NMONTH;" - NDD: "&NDATE;" - NHH: "&NHOUR;" + CTIME: "&CTIME;" + PTIME: "&PTIME;" + NTIME: "&NTIME;" fv3bundle_vn: "&fv3bundle_vn;" DAtype: "&DAtype;" SNOWDEPTHVAR: "&SNOWDEPTHVAR;" - NPROC_JEDI: "&NPROCS;" JEDI_INSTALL: "&JEDI_INSTALL;" FCSTHR: "&FCSTHR;" account: epic - command: "/work/noaa/epic/jongkim/land-DA/fcst.sh" + command: "&CYCLEDIR;/jobs/JLAND_RUN_FCST" jobname: runfcst - nodes: "1:ppn=&NPROCS;" + nodes: "1:ppn=&NPROCS_FCST;" walltime: 00:30:00 queue: batch dependency: taskdep: attrs: - task: runana + task: runana \ No newline at end of file