Skip to content

Commit

Permalink
updates for derecho
Browse files Browse the repository at this point in the history
  • Loading branch information
@jedwards4b
jedwards4b committed Nov 2, 2023
1 parent 38dfe32 commit 81c91da
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 80 deletions.
130 changes: 66 additions & 64 deletions config/cesm/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ This allows using a different mpirun command to launch unit tests
<mpirun mpilib="default">
<executable>mpirun</executable>
<arguments>
<arg name="ntasks"> -np {{ total_tasks }} </arg>
<arg name="ntasks"> -np {{ total_tasks }} </arg>
</arguments>
</mpirun>

Expand All @@ -152,11 +152,11 @@ This allows using a different mpirun command to launch unit tests
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules compiler="intel">
<command name="unload">openmpi</command>
<command name="load">intel/19.0.4</command>
<command name="load">intelmpi/2019.4.243</command>
<command name="load">netcdf/4.6.3</command>
<command name="load">esmf</command>
<command name="unload">openmpi</command>
<command name="load">intel/19.0.4</command>
<command name="load">intelmpi/2019.4.243</command>
<command name="load">netcdf/4.6.3</command>
<command name="load">esmf</command>
</modules>
</module_system>
<environment_variables>
Expand Down Expand Up @@ -204,18 +204,18 @@ This allows using a different mpirun command to launch unit tests
<cmd_path lang="csh">module</cmd_path>

<modules compiler="gnu">
<command name="load"> PrgEnv-gnu</command>
<command name="load"> cray-hdf5-parallel</command>
<command name="load"> cray-netcdf-hdf5parallel</command>
<command name="load"> cray-parallel-netcdf</command>
<command name="load"> cray-libsci</command>
<command name="load"> PrgEnv-gnu</command>
<command name="load"> cray-hdf5-parallel</command>
<command name="load"> cray-netcdf-hdf5parallel</command>
<command name="load"> cray-parallel-netcdf</command>
<command name="load"> cray-libsci</command>
</modules>
<modules mpilib="mpi-serial">
<command name="rm"> cray-netcdf-hdf5parallel</command>
<command name="rm"> cray-hdf5-parallel</command>
<command name="rm"> cray-parallel-netcdf</command>
<command name="load"> cray-hdf5</command>
<command name="load">cray-netcdf</command>
<command name="rm"> cray-netcdf-hdf5parallel</command>
<command name="rm"> cray-hdf5-parallel</command>
<command name="rm"> cray-parallel-netcdf</command>
<command name="load"> cray-hdf5</command>
<command name="load">cray-netcdf</command>
</modules>
</module_system>

Expand Down Expand Up @@ -339,10 +339,10 @@ This allows using a different mpirun command to launch unit tests
<mpirun mpilib="default">
<executable>srun</executable>
<arguments>
<arg name="num_tasks">-n {{ total_tasks }}</arg>
<arg name="bindinfo"> --cpu-bind=verbose </arg>
<arg name="cpu_bind">--cpu_bind=sockets --cpu_bind=verbose</arg>
<arg name="kill-on-bad-exit">--kill-on-bad-exit</arg>
<arg name="num_tasks">-n {{ total_tasks }}</arg>
<arg name="bindinfo"> --cpu-bind=verbose </arg>
<arg name="cpu_bind">--cpu_bind=sockets --cpu_bind=verbose</arg>
<arg name="kill-on-bad-exit">--kill-on-bad-exit</arg>
</arguments>
</mpirun>
<module_system type="none"/>
Expand Down Expand Up @@ -1111,8 +1111,8 @@ This allows using a different mpirun command to launch unit tests
<mpirun mpilib="default">
<executable>mpiexec</executable>
<arguments>
<arg name="label"> --label</arg>
<arg name="num_tasks" > -n {{ total_tasks }}</arg>
<arg name="label"> --label</arg>
<arg name="num_tasks" > -n {{ total_tasks }}</arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
Expand All @@ -1126,58 +1126,60 @@ This allows using a different mpirun command to launch unit tests
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="load">cesmdev/1.0</command>
<command name="load">ncarenv/23.06</command>
<command name="purge"/>
<command name="load">ncarenv/23.09</command>
<command name="purge"/>
<command name="load">craype</command>
</modules>
<modules compiler="intel">
<command name="load">intel/2023.0.0</command>
<command name="load">intel/2023.2.1</command>
<command name="load">mkl</command>
<command name="load">spherepack/3.2</command>
</modules>
<modules compiler="intel-oneapi">
<command name="load">intel-oneapi/2023.0.0</command>
<command name="load">intel-oneapi/2023.2.1</command>
<command name="load">mkl</command>
</modules>
<modules compiler="intel-classic">
<command name="load">intel-classic/2023.0.0</command>
<command name="load">intel-classic/2023.2.1</command>
<command name="load">mkl</command>
</modules>
<modules compiler="cray">
<command name="load">cce/15.0.1</command>
<command name="load">cray-libsci/23.02.1.1</command>
<command name="load">cce/15.0.1</command>
<command name="load">cray-libsci/23.02.1.1</command>
</modules>
<modules compiler="gnu">
<command name="load">gcc/12.2.0</command>
<command name="load">cray-libsci/23.02.1.1</command>
<command name="load">gcc/12.2.0</command>
<command name="load">cray-libsci/23.02.1.1</command>
</modules>
<modules compiler="nvhpc">
<command name="load">nvhpc/23.1</command>
<command name="load">nvhpc/23.7</command>
</modules>
<modules>
<command name="load">ncarcompilers/1.0.0</command>
<command name="load">cmake</command>
<command name="load">cmake</command>
</modules>
<modules mpilib="mpich">
<command name="load">cray-mpich/8.1.25</command>
<command name="load">cray-mpich/8.1.27</command>
</modules>
<modules mpilib="mpi-serial">
<command name="load">mpi-serial/2.3.0</command>
<command name="load">mpi-serial/2.3.0</command>
</modules>

<modules mpilib="mpi-serial">
<command name="load">netcdf/4.9.2</command>
<command name="load">netcdf/4.9.2</command>
</modules>

<modules mpilib="!mpi-serial">
<command name="load">netcdf-mpi/4.9.2</command>
<command name="load">parallel-netcdf/1.12.3</command>
<command name="load">netcdf-mpi/4.9.2</command>
<command name="load">parallel-netcdf/1.12.3</command>
</modules>

</module_system>

<environment_variables>
<env name="OMP_STACKSIZE">64M</env>
<env name="FI_CXI_RX_MATCH_MODE">hybrid</env>
<env name="SPHEREPACK_LIBDIR">$ENV{NCAR_ROOT_SPHEREPACK}/lib</env>
</environment_variables>
<environment_variables mpilib="mpich">
<env name="MPICH_MPIIO_HINTS">*:romio_cb_read=enable:romio_cb_write=enable:striping_factor=2</env>
Expand Down Expand Up @@ -1504,7 +1506,7 @@ This allows using a different mpirun command to launch unit tests
<mpirun mpilib="impi">
<executable>srun</executable>
<arguments>
<arg name="num_tasks"> -n $TOTALPES</arg>
<arg name="num_tasks"> -n $TOTALPES</arg>
<arg name="hint"> --hint compute_bound </arg>
<arg name="bindinfo"> --cpu-bind=verbose </arg>
<!-- <arg name="bindinfo"> -print-rank-map </arg> -->
Expand All @@ -1520,16 +1522,16 @@ This allows using a different mpirun command to launch unit tests
<cmd_path lang="sh">module</cmd_path>
<cmd_path lang="csh">module</cmd_path>
<modules>
<command name="purge"></command>
<command name="load">intel/2019b</command>
<command name="load">CMake/3.15.3</command>
<command name="load">cURL/7.66.0</command>
<command name="load">Python/2.7.16</command>
<command name="load">XML-LibXML/2.0201</command>
<command name="purge"></command>
<command name="load">intel/2019b</command>
<command name="load">CMake/3.15.3</command>
<command name="load">cURL/7.66.0</command>
<command name="load">Python/2.7.16</command>
<command name="load">XML-LibXML/2.0201</command>
</modules>
<modules mpilib="impi">
<command name="load">impi/2018.5.288</command>
<command name="load">HDF5/1.10.5</command>
<command name="load">impi/2018.5.288</command>
<command name="load">HDF5/1.10.5</command>
</modules>
</module_system>
<environment_variables>
Expand Down Expand Up @@ -1561,8 +1563,8 @@ This allows using a different mpirun command to launch unit tests
<mpirun mpilib="default">
<executable>mpiexec</executable>
<arguments>
<arg name="label"> --label</arg>
<arg name="num_tasks" > -n {{ total_tasks }}</arg>
<arg name="label"> --label</arg>
<arg name="num_tasks" > -n {{ total_tasks }}</arg>
</arguments>
</mpirun>
<module_system type="module" allow_error="true">
Expand All @@ -1577,50 +1579,50 @@ This allows using a different mpirun command to launch unit tests
<modules>
<command name="load">cesmdev/1.0</command>
<command name="load">ncarenv/23.04</command>
<command name="purge"/>
<command name="purge"/>
<command name="load">craype</command>
</modules>
<modules compiler="intel">
<command name="load">intel/2023.0.0</command>
<command name="load">intel/2023.0.0</command>
<command name="load">mkl</command>
</modules>
<modules compiler="intel-oneapi">
<command name="load">intel-oneapi/2023.0.0</command>
<command name="load">intel-oneapi/2023.0.0</command>
<command name="load">mkl</command>
</modules>
<modules compiler="intel-classic">
<command name="load">intel-classic/2023.0.0</command>
<command name="load">intel-classic/2023.0.0</command>
<command name="load">mkl</command>
</modules>
<modules compiler="cray">
<command name="load">cce/15.0.1</command>
<command name="load">cray-libsci/23.02.1.1</command>
<command name="load">cce/15.0.1</command>
<command name="load">cray-libsci/23.02.1.1</command>
</modules>
<modules compiler="gnu">
<command name="load">gcc/12.2.0</command>
<command name="load">cray-libsci/23.02.1.1</command>
<command name="load">gcc/12.2.0</command>
<command name="load">cray-libsci/23.02.1.1</command>
</modules>
<modules compiler="nvhpc">
<command name="load">nvhpc/23.1</command>
<command name="load">nvhpc/23.1</command>
</modules>
<modules>
<command name="load">ncarcompilers/0.8.0</command>
<command name="load">cmake</command>
<command name="load">cmake</command>
</modules>
<modules mpilib="mpich">
<command name="load">cray-mpich/8.1.25</command>
</modules>
<modules mpilib="mpi-serial">
<command name="load">mpi-serial/2.3.0</command>
<command name="load">mpi-serial/2.3.0</command>
</modules>

<modules mpilib="mpi-serial">
<command name="load">netcdf/4.9.1</command>
<command name="load">netcdf/4.9.1</command>
</modules>

<modules mpilib="!mpi-serial">
<command name="load">netcdf-mpi/4.9.1</command>
<command name="load">parallel-netcdf/1.12.3</command>
<command name="load">netcdf-mpi/4.9.1</command>
<command name="load">parallel-netcdf/1.12.3</command>
<command name="load">esmf/8.4.1</command>
</modules>

Expand All @@ -1634,7 +1636,7 @@ This allows using a different mpirun command to launch unit tests
<env name="MPICH_MPIIO_HINTS">*:romio_cb_read=enable:romio_cb_write=enable:striping_factor=2</env>
</environment_variables>
</machine>

<machine MACH="hobart">
<DESC>NCAR CGD Linux Cluster 48 pes/node, batch system is PBS</DESC>
<NODENAME_REGEX>^h.*\.cgd\.ucar\.edu</NODENAME_REGEX>
Expand Down
36 changes: 20 additions & 16 deletions scripts/Tools/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,8 +421,8 @@ else
# GCC needs to be able to link to
# nagfor runtime to get autoconf
# tests to work.
CFLAGS += -Wl,--as-needed,--allow-shlib-undefined
SLIBS += -L$(COMPILER_PATH)/lib/NAG_Fortran -lf62rts
CFLAGS += -Wl,--as-needed,--allow-shlib-undefined
SLIBS += -L$(COMPILER_PATH)/lib/NAG_Fortran -lf62rts
endif
endif
endif
Expand Down Expand Up @@ -461,9 +461,9 @@ CFLAGS+=$(CPPDEFS)
CXXFLAGS := $(CFLAGS)

CONFIG_ARGS += CC="$(CC)" FC="$(FC)" MPICC="$(MPICC)" \
MPIFC="$(MPIFC)" FCFLAGS="$(FFLAGS) $(FREEFLAGS) $(INCLDIR)" \
CPPDEFS="$(CPPDEFS)" CFLAGS="$(CFLAGS) -I.. $(INCLDIR)" \
LDFLAGS="$(LDFLAGS)"
MPIFC="$(MPIFC)" FCFLAGS="$(FFLAGS) $(FREEFLAGS) $(INCLDIR)" \
CPPDEFS="$(CPPDEFS)" CFLAGS="$(CFLAGS) -I.. $(INCLDIR)" \
LDFLAGS="$(LDFLAGS)"

ifeq ($(NETCDF_SEPARATE), false)
CONFIG_ARGS += NETCDF_PATH=$(NETCDF_PATH)
Expand Down Expand Up @@ -534,6 +534,10 @@ endif
ifdef LAPACK_LIBDIR
SLIBS += -L$(LAPACK_LIBDIR) -llapack -lblas
endif
#SPHEREPACK for stoch
ifdef SPHEREPACK_LIBDIR
SLIBS += -L$(SPHEREPACK_LIBDIR) -lspherepack
endif
ifdef LIB_MPI
ifndef MPI_LIB_NAME
SLIBS += -L$(LIB_MPI) -lmpi
Expand Down Expand Up @@ -644,11 +648,11 @@ endif
# doesn't seem to be able to differentiate between free & fixed
# fortran flags)
CMAKE_OPTS += -D CMAKE_Fortran_FLAGS:STRING="$(FFLAGS) $(EXTRA_PIO_FPPDEFS) $(INCLDIR)" \
-D CMAKE_C_FLAGS:STRING="$(CFLAGS) $(EXTRA_PIO_CPPDEFS) $(INCLDIR)" \
-D CMAKE_CXX_FLAGS:STRING="$(CXXFLAGS) $(EXTRA_PIO_CPPDEFS) $(INCLDIR)" \
-D CMAKE_VERBOSE_MAKEFILE:BOOL=ON \
-D GPTL_PATH:STRING=$(INSTALL_SHAREDPATH) \
-D PIO_ENABLE_TESTS:BOOL=OFF \
-D CMAKE_C_FLAGS:STRING="$(CFLAGS) $(EXTRA_PIO_CPPDEFS) $(INCLDIR)" \
-D CMAKE_CXX_FLAGS:STRING="$(CXXFLAGS) $(EXTRA_PIO_CPPDEFS) $(INCLDIR)" \
-D CMAKE_VERBOSE_MAKEFILE:BOOL=ON \
-D GPTL_PATH:STRING=$(INSTALL_SHAREDPATH) \
-D PIO_ENABLE_TESTS:BOOL=OFF \
-D USER_CMAKE_MODULE_PATH:LIST="$(CIMEROOT)/src/CMake;$(CIMEROOT)/src/externals/pio2/cmake" \

# Allow for separate installations of the NetCDF C and Fortran libraries
Expand All @@ -658,13 +662,13 @@ else ifeq ($(NETCDF_SEPARATE), true)
# NETCDF_Fortran_DIR points to the separate
# installation of Fortran NetCDF for PIO
CMAKE_OPTS += -D NetCDF_C_PATH:PATH=$(NETCDF_C_PATH) \
-D NetCDF_Fortran_PATH:PATH=$(NETCDF_FORTRAN_PATH)
-D NetCDF_Fortran_PATH:PATH=$(NETCDF_FORTRAN_PATH)
endif

ifdef PNETCDF_PATH
CMAKE_OPTS += -D PnetCDF_PATH:STRING="$(PNETCDF_PATH)"
else
CMAKE_OPTS += -D WITH_PNETCDF:LOGICAL=FALSE -D PIO_USE_MPIIO:LOGICAL=FALSE
CMAKE_OPTS += -D WITH_PNETCDF:LOGICAL=FALSE -D PIO_USE_MPIIO:LOGICAL=FALSE
endif
ifdef PIO_FILESYSTEM_HINTS
CMAKE_OPTS += -D PIO_FILESYSTEM_HINTS:STRING="$(PIO_FILESYSTEM_HINTS)"
Expand All @@ -680,9 +684,9 @@ ifndef CMAKE_ENV_VARS
CMAKE_ENV_VARS :=
endif
CMAKE_ENV_VARS += CC=$(CC) \
CXX=$(CXX) \
FC=$(FC) \
LDFLAGS="$(LDFLAGS)"
CXX=$(CXX) \
FC=$(FC) \
LDFLAGS="$(LDFLAGS)"


# We declare $(GLC_DIR)/Makefile to be a phony target so that cmake is
Expand Down Expand Up @@ -810,7 +814,7 @@ ifeq ($(ULIBDEP),$(null))
ifeq ($(COMP_GLC), cism)
ULIBDEP += $(CISM_LIBDIR)/libglimmercismfortran.a
ifeq ($(CISM_USE_TRILINOS), TRUE)
ULIBDEP += $(CISM_LIBDIR)/libglimmercismcpp.a
ULIBDEP += $(CISM_LIBDIR)/libglimmercismcpp.a
endif
endif
ifeq ($(OCN_SUBMODEL),moby)
Expand Down

0 comments on commit 81c91da

Please sign in to comment.