diff --git a/check_gdas_outputs.sh b/check_gdas_outputs.sh deleted file mode 100755 index 6cdab506..00000000 --- a/check_gdas_outputs.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh - -# get OUTDIR -export SETTINGS_FILE=${1:-"settings_DA_cycle_gdas"} -source ./${SETTINGS_FILE} - -export TEST_BASEDIR=${TEST_BASEDIR:-${EPICTESTS}} - -for TEST_DATE in 2016-01-01_18-00-00 2016-01-02_18-00-00 -do - -for state in anal -do - -cmp ${OUTDIR}/mem000/restarts/vector/ufs_land_restart_${state}.${TEST_DATE}.nc ${TEST_BASEDIR}/ufs_land_restart_${state}.${TEST_DATE}.nc - -echo "testing ${state} on ${TEST_DATE}" -if [[ $? != 0 ]]; then - echo TEST FAILED - echo "$TEST_DATE $state are different" - exit 98 -fi - -done -done - -#TEST_DATE=2016-01-03_18-00-00 -for TEST_DATE in 2016-01-02_18-00-00 2016-01-03_18-00-00 -do -state='back' -echo "testing ${state} on ${TEST_DATE}" -cmp ${OUTDIR}/mem000/restarts/vector/ufs_land_restart_${state}.${TEST_DATE}.nc ${TEST_BASEDIR}/ufs_land_restart_${state}.${TEST_DATE}.nc - -if [[ $? != 0 ]]; then - echo TEST FAILED - echo "$TEST_DATE $state are different" - exit 99 -fi -done - -echo "TEST PASSED" - -exit diff --git a/check_test.sh b/check_test.sh deleted file mode 100755 index 69260f12..00000000 --- a/check_test.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -# get OUTDIR -source settings_cycle_test - -TEST_BASEDIR=/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/DA_IMS_test/output/mem000/restarts/vector/ - -for TEST_DATE in 2016-01-01_18-00-00 2016-01-02_18-00-00 -do - -for state in back anal -do - -cmp ${OUTDIR}/mem000/restarts/vector/ufs_land_restart_${state}.${TEST_DATE}.nc ${TEST_BASEDIR}/ufs_land_restart_${state}.${TEST_DATE}.nc - -if [[ $? != 0 ]]; then - echo TEST FAILED - echo "$TEST_DATE $state are different" - exit -fi - -done -done - -TEST_DATE=2016-01-03_18-00-00 -state='back' -cmp ${OUTDIR}/mem000/restarts/vector/ufs_land_restart_${state}.${TEST_DATE}.nc ${TEST_BASEDIR}/ufs_land_restart_${state}.${TEST_DATE}.nc - -if [[ $? != 0 ]]; then - echo TEST FAILED - echo "$TEST_DATE $state are different" - exit -fi - -echo "TEST PASSED" - -exit diff --git a/configures/machine.hera.intel b/configures/machine.hera.intel deleted file mode 100644 index 19f12aeb..00000000 --- a/configures/machine.hera.intel +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -export JEDI_OPT=/scratch1/NCEPDEV/jcsda/jedipara/opt/modules -module use $JEDI_OPT/modulefiles/core -module purge -module load jedi/intel-impi/2020.2 -module load cmake/3.20.1 -module load nco/4.9.1 -source /scratch2/NCEPDEV/marineda/Jong.Kim/hera.anaconda -module load anaconda/3.15.1 -export MPIRUN=$(which srun) -export WORKLOAD_MANAGER=slurm diff --git a/configures/machine.orion.intel b/configures/machine.orion.intel deleted file mode 100644 index 79a224f6..00000000 --- a/configures/machine.orion.intel +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -export JEDI_OPT=/work/noaa/da/jedipara/opt/modules -module use $JEDI_OPT/modulefiles/core -module purge -module load jedi/intel-impi/2020.2 - -. /work/noaa/da/kritib/soca-shared/soca_python-3.9/bin/activate - -# modules to only load during runtime -if [[ "$SOCA_SCIENCE_RUNTIME" == T ]]; then - module load nco/4.9.3 -fi - - -ulimit -s unlimited - -export MPIRUN=$(which srun) -export WORKLOAD_MANAGER=slurm -export JOB_OPTS="--exclusive" diff --git a/do_submit_test.sh b/do_submit_test.sh deleted file mode 100755 index f1f48980..00000000 --- a/do_submit_test.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -source settings_DA_cycle_gdas - -rm -rf ${OUTDIR} - -./do_submit_cycle.sh settings_DA_cycle_gdas - - - diff --git a/README_git b/docs/README_git similarity index 100% rename from README_git rename to docs/README_git diff --git a/jobs/JLANDDA_PREP_BMAT b/jobs/JLANDDA_PREP_BMAT index fdff4df1..52515cfa 100755 --- a/jobs/JLANDDA_PREP_BMAT +++ b/jobs/JLANDDA_PREP_BMAT @@ -30,6 +30,10 @@ FILEDATE=${YYYY}${MM}${DD}.${HH}0000 cd $MEM_WORKDIR # load modulefiles +BUILD_VERSION_FILE="${CYCLEDIR}/versions/build.ver_${MACHINE}" +if [ -e ${BUILD_VERSION_FILE} ]; then + . ${BUILD_VERSION_FILE} +fi module use modulefiles; module load modules.landda PYTHON=$(/usr/bin/which python) diff --git a/jobs/JLANDDA_PREP_EXP b/jobs/JLANDDA_PREP_EXP index 6b98f8e6..4b780e69 100755 --- a/jobs/JLANDDA_PREP_EXP +++ b/jobs/JLANDDA_PREP_EXP @@ -42,6 +42,11 @@ mkdir -p $MEM_WORKDIR/modulefiles; cp ${CYCLEDIR}/modulefiles/build_${MACHINE}_i cd $MEM_WORKDIR # load modulefiles +BUILD_VERSION_FILE="${CYCLEDIR}/versions/build.ver_${MACHINE}" +if [ -e ${BUILD_VERSION_FILE} ]; then + . ${BUILD_VERSION_FILE} +fi + module use modulefiles; module load modules.landda if [[ $do_jedi == "YES" && $ATMOS_FORC == "era5" ]]; then diff --git a/jobs/JLANDDA_RUN_ANA b/jobs/JLANDDA_RUN_ANA index 942827b1..84b4896e 100755 --- a/jobs/JLANDDA_RUN_ANA +++ b/jobs/JLANDDA_RUN_ANA @@ -41,6 +41,10 @@ KEEPJEDIDIR="YES" cd $MEM_WORKDIR # load modulefiles +BUILD_VERSION_FILE="${CYCLEDIR}/versions/build.ver_${MACHINE}" +if [ -e ${BUILD_VERSION_FILE} ]; then + . ${BUILD_VERSION_FILE} +fi module use modulefiles; module load modules.landda PYTHON=$(/usr/bin/which python) MPIEXEC=`which mpiexec` diff --git a/jobs/JLANDDA_RUN_FCST b/jobs/JLANDDA_RUN_FCST index 0e1b8441..bc1d0cce 100755 --- a/jobs/JLANDDA_RUN_FCST +++ b/jobs/JLANDDA_RUN_FCST @@ -48,6 +48,10 @@ RHH=$((${FCSTHR}%24)) cd $MEM_WORKDIR # load modulefiles +BUILD_VERSION_FILE="${CYCLEDIR}/versions/build.ver_${MACHINE}" +if [ -e ${BUILD_VERSION_FILE} ]; then + . ${BUILD_VERSION_FILE} +fi module use modulefiles; module load modules.landda PYTHON=$(/usr/bin/which python) MPIEXEC=`which mpiexec` diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index 0474694d..366735f5 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -1,77 +1,36 @@ help([[ -loads UFS Model prerequisites for Hera/Intel +loads modules necessary for building the land-DA workflow on Hera using Intel ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") +whatis([===[Loads modules necessary for building the land-DA workflow on Hera]===]) -stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -load(pathJoin("stack-intel", stack_intel_ver)) +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) - -stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" load(pathJoin("stack-python", stack_python_ver)) -cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("cmake", cmake_ver)) - -ecbuild_ver=os.getenv("ecbuild_ver") or "3.6.5" load(pathJoin("ecbuild", ecbuild_ver)) -jasper_ver=os.getenv("jasper_ver") or "2.0.32" load(pathJoin("jasper", jasper_ver)) - -zlib_ver=os.getenv("zlib_ver") or "1.2.13" load(pathJoin("zlib", zlib_ver)) - -libpng_ver=os.getenv("libpng_ver") or "1.6.37" load(pathJoin("libpng", libpng_ver)) - -hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" load(pathJoin("hdf5", hdf5_ver)) - -netcdf_c_ver=os.getenv("netcdf_ver") or "4.9.0" load(pathJoin("netcdf-c", netcdf_c_ver)) - -netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) - -pio_ver=os.getenv("pio_ver") or "2.5.9" load(pathJoin("parallelio", pio_ver)) - -esmf_ver=os.getenv("esmf_ver") or "8.3.0b09" load(pathJoin("esmf", esmf_ver)) - -fms_ver=os.getenv("fms_ver") or "2022.04" load(pathJoin("fms",fms_ver)) - -bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) - -crtm_ver=os.getenv("crtm_ver") or "2.4.0" load(pathJoin("crtm", crtm_ver)) - -g2_ver=os.getenv("g2_ver") or "3.4.5" load(pathJoin("g2", g2_ver)) - -g2tmpl_ver=os.getenv("g2tmpl_ver") or "1.10.2" load(pathJoin("g2tmpl", g2tmpl_ver)) - -ip_ver=os.getenv("ip_ver") or "3.3.3" load(pathJoin("ip", ip_ver)) - -sp_ver=os.getenv("sp_ver") or "2.3.3" load(pathJoin("sp", sp_ver)) - -w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" load(pathJoin("w3emc", w3emc_ver)) - -gftl_shared_ver=os.getenv("gftl_shared_ver") or "v1.5.0" load(pathJoin("gftl-shared", gftl_shared_ver)) - -mapl_ver=os.getenv("mapl_ver") or "2.22.0-esmf-8.3.0b09" load(pathJoin("mapl", mapl_ver)) load("ufs-pyenv") @@ -85,4 +44,3 @@ setenv("CMAKE_Platform", "hera.intel") setenv("EPICHOME", "/scratch2/NAGAPE/epic/UFS_Land-DA") setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi")) -whatis("Description: UFS build environment") diff --git a/modulefiles/build_orion_intel.lua b/modulefiles/build_orion_intel.lua index bb515c04..b3db0e2e 100644 --- a/modulefiles/build_orion_intel.lua +++ b/modulefiles/build_orion_intel.lua @@ -1,77 +1,36 @@ help([[ -loads UFS Model prerequisites for Orion/Intel +loads modules necessary for building the land-DA workflow on Orion using Intel ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/work/noaa/da/role-da/spack-stack/modulefiles") +whatis([===[Loads modules necessary for building the land-DA workflow on Orion]===]) -stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" -load(pathJoin("stack-intel", stack_intel_ver)) +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack")) +prepend_path("MODULEPATH", os.getenv("modulepath_spack_stack_jedi")) -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) - -stack_python_ver=os.getenv("stack_python_ver") or "3.9.7" load(pathJoin("stack-python", stack_python_ver)) -cmake_ver=os.getenv("cmake_ver") or "3.22.1" load(pathJoin("cmake", cmake_ver)) - -ecbuild_ver=os.getenv("ecbuild_ver") or "3.6.5" load(pathJoin("ecbuild", ecbuild_ver)) -jasper_ver=os.getenv("jasper_ver") or "2.0.32" load(pathJoin("jasper", jasper_ver)) - -zlib_ver=os.getenv("zlib_ver") or "1.2.13" load(pathJoin("zlib", zlib_ver)) - -libpng_ver=os.getenv("libpng_ver") or "1.6.37" load(pathJoin("libpng", libpng_ver)) - -hdf5_ver=os.getenv("hdf5_ver") or "1.14.0" load(pathJoin("hdf5", hdf5_ver)) - -netcdf_c_ver=os.getenv("netcdf_ver") or "4.9.0" load(pathJoin("netcdf-c", netcdf_c_ver)) - -netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" load(pathJoin("netcdf-fortran", netcdf_fortran_ver)) - -pio_ver=os.getenv("pio_ver") or "2.5.9" load(pathJoin("parallelio", pio_ver)) - -esmf_ver=os.getenv("esmf_ver") or "8.3.0b09" load(pathJoin("esmf", esmf_ver)) - -fms_ver=os.getenv("fms_ver") or "2022.04" load(pathJoin("fms",fms_ver)) - -bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) - -crtm_ver=os.getenv("crtm_ver") or "2.4.0" load(pathJoin("crtm", crtm_ver)) - -g2_ver=os.getenv("g2_ver") or "3.4.5" load(pathJoin("g2", g2_ver)) - -g2tmpl_ver=os.getenv("g2tmpl_ver") or "1.10.2" load(pathJoin("g2tmpl", g2tmpl_ver)) - -ip_ver=os.getenv("ip_ver") or "3.3.3" load(pathJoin("ip", ip_ver)) - -sp_ver=os.getenv("sp_ver") or "2.3.3" load(pathJoin("sp", sp_ver)) - -w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" load(pathJoin("w3emc", w3emc_ver)) - -gftl_shared_ver=os.getenv("gftl_shared_ver") or "v1.5.0" load(pathJoin("gftl-shared", gftl_shared_ver)) - -mapl_ver=os.getenv("mapl_ver") or "2.22.0-esmf-8.3.0b09" load(pathJoin("mapl", mapl_ver)) load("ufs-pyenv") @@ -84,5 +43,3 @@ setenv("CMAKE_Platform", "orion.intel") setenv("EPICHOME", "/work/noaa/epic/UFS_Land-DA") setenv("JEDI_INSTALL", pathJoin(os.getenv("EPICHOME"),"jedi")) -whatis("Description: UFS build environment") - diff --git a/check_release_outputs.sh b/parm/check_release_outputs.sh similarity index 100% rename from check_release_outputs.sh rename to parm/check_release_outputs.sh diff --git a/parm/land_analysis_era5_hera.yaml b/parm/land_analysis_era5_hera.yaml index 5ae85522..d23c7e7e 100644 --- a/parm/land_analysis_era5_hera.yaml +++ b/parm/land_analysis_era5_hera.yaml @@ -10,7 +10,7 @@ workflow: MACHINE: "hera" ACCOUNT: "nems" EXP_NAME: "LETKF" - EXP_BASEDIR: "/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/landda_fixjjob" + EXP_BASEDIR: "/scratch2/NCEPDEV/fv3-cam/Chan-hoo.Jeon/landda_version" JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi" LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" FORCING: "era5" diff --git a/parm/land_analysis_era5_orion.yaml b/parm/land_analysis_era5_orion.yaml index a12e61c4..ce93b273 100644 --- a/parm/land_analysis_era5_orion.yaml +++ b/parm/land_analysis_era5_orion.yaml @@ -10,7 +10,7 @@ workflow: MACHINE: "orion" ACCOUNT: "epic" EXP_NAME: "LETKF" - EXP_BASEDIR: "/work/noaa/epic/chjeon/landda_fixjjob" + EXP_BASEDIR: "/work/noaa/epic/chjeon/landda_version" JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi" LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs" FORCING: "era5" diff --git a/sorc/app_build.sh b/sorc/app_build.sh index 754f8342..e726bd66 100755 --- a/sorc/app_build.sh +++ b/sorc/app_build.sh @@ -347,8 +347,8 @@ fi source ${HOME_DIR}/parm/lmod-setup.sh ${PLATFORM} # source version file for build -BUILD_VERSION_FILE="${HOME_DIR}/versions/build.ver.${PLATFORM}" -if [ -f ${BUILD_VERSION_FILE} ]; then +BUILD_VERSION_FILE="${HOME_DIR}/versions/build.ver_${PLATFORM}" +if [ -e ${BUILD_VERSION_FILE} ]; then . ${BUILD_VERSION_FILE} fi diff --git a/sorc/test/hera_ctest.sh b/sorc/test/hera_ctest.sh index 6387f0c0..cb2b8fbe 100755 --- a/sorc/test/hera_ctest.sh +++ b/sorc/test/hera_ctest.sh @@ -3,6 +3,7 @@ #SBATCH --account=nems set -eux +source ../../versions/build.ver_hera module use ../../modulefiles && module load build_hera_intel ctest diff --git a/sorc/test/orion_ctest.sh b/sorc/test/orion_ctest.sh index 626266ea..44863672 100755 --- a/sorc/test/orion_ctest.sh +++ b/sorc/test/orion_ctest.sh @@ -3,6 +3,7 @@ #SBATCH --account=nems set -eux +source ../../versions/build.ver_orion module use ../../modulefiles && module load build_orion_intel ctest diff --git a/versions/build.ver_hera b/versions/build.ver_hera new file mode 100644 index 00000000..ade8f887 --- /dev/null +++ b/versions/build.ver_hera @@ -0,0 +1,29 @@ +#!/bin/bash + +export modulepath_spack_stack="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core" +export modulepath_spack_stack_jedi="/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles" + +export bacio_ver="2.4.1" +export cmake_ver="3.23.1" +export crtm_ver="2.4.0" +export ecbuild_ver="3.6.5" +export esmf_ver="8.3.0b09" +export fms_ver="2022.04" +export gftl_shared_ver="v1.5.0" +export g2_ver="3.4.5" +export g2tmpl_ver="1.10.2" +export hdf5_ver="1.14.0" +export ip_ver="3.3.3" +export jasper_ver="2.0.32" +export libpng_ver="1.6.37" +export mapl_ver="2.22.0-esmf-8.3.0b09" +export netcdf_c_ver="4.9.0" +export netcdf_fortran_ver="4.6.0" +export pio_ver="2.5.9" +export sp_ver="2.3.3" +export stack_impi_ver="2021.5.1" +export stack_intel_ver="2021.5.0" +export stack_python_ver="3.9.12" +export w3emc_ver="2.9.2" +export zlib_ver="1.2.13" + diff --git a/versions/build.ver_orion b/versions/build.ver_orion new file mode 100644 index 00000000..d78ddc63 --- /dev/null +++ b/versions/build.ver_orion @@ -0,0 +1,29 @@ +#!/bin/bash + +export modulepath_spack_stack="/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.3.0/envs/unified-env/install/modulefiles/Core" +export modulepath_spack_stack_jedi="/work/noaa/da/role-da/spack-stack/modulefiles" + +export bacio_ver="2.4.1" +export cmake_ver="3.22.1" +export crtm_ver="2.4.0" +export ecbuild_ver="3.6.5" +export esmf_ver="8.3.0b09" +export fms_ver="2022.04" +export gftl_shared_ver="v1.5.0" +export g2_ver="3.4.5" +export g2tmpl_ver="1.10.2" +export hdf5_ver="1.14.0" +export ip_ver="3.3.3" +export jasper_ver="2.0.32" +export libpng_ver="1.6.37" +export mapl_ver="2.22.0-esmf-8.3.0b09" +export netcdf_c_ver="4.9.0" +export netcdf_fortran_ver="4.6.0" +export pio_ver="2.5.9" +export sp_ver="2.3.3" +export stack_impi_ver="2021.5.1" +export stack_intel_ver="2022.0.2" +export stack_python_ver="3.9.7" +export w3emc_ver="2.9.2" +export zlib_ver="1.2.13" + diff --git a/versions/run.ver_hera b/versions/run.ver_hera new file mode 120000 index 00000000..443f437e --- /dev/null +++ b/versions/run.ver_hera @@ -0,0 +1 @@ +build.ver_hera \ No newline at end of file diff --git a/versions/run.ver_orion b/versions/run.ver_orion new file mode 120000 index 00000000..ba4830fa --- /dev/null +++ b/versions/run.ver_orion @@ -0,0 +1 @@ +build.ver_orion \ No newline at end of file