Pip Runtime Dependencies & CRAY AMD Support

.gitignore

@@ -89,7 +89,7 @@ src/PBGWraps/SpheralModules_Silo.C
 src/PBGWraps/SpheralModules_Utilities.C
 src/PBGWraps/SpheralModules_WildMagic.C
 
-build-*
+/build-*
 src/*/*cc.2.cc
 src/*/*cc.3.cc
 src/*/*/*cc.2.cc

.gitlab/jobs-mpi.yml

@@ -53,37 +53,37 @@ toss_clang_mvapich2_cleanup:
 
 
 
-blueos_gcc_spectrum_tpls:
-  extends: [.blueos_resource1, .gcc_spectrum, .tpls]
+cray_rocm_mpich_tpls:
+  extends: [.cray_resource1, .rocm_mpich, .tpls]
 
-blueos_gcc_spectrum_build:
-  extends: [.blueos_resource1, .gcc_spectrum, .build_and_test]
-  needs: [blueos_gcc_spectrum_tpls]
+cray_rocm_mpich_build:
+  extends: [.cray_resource1, .rocm_mpich, .build_and_test]
+  needs: [cray_rocm_mpich_tpls]
 
-blueos_gcc_spectrum_test:
-  extends: [.blueos_resource1, .gcc_spectrum, .run_ats]
-  needs: [blueos_gcc_spectrum_build]
+cray_rocm_mpich_test:
+  extends: [.cray_resource1, .rocm_mpich, .run_ats]
+  needs: [cray_rocm_mpich_build]
 
-blueos_gcc_spectrum_cleanup:
-  extends: [.blueos_resource1, .gcc_spectrum, .cleanup_dir]
-  needs: [blueos_gcc_spectrum_test]
+cray_rocm_mpich_cleanup:
+  extends: [.cray_resource1, .rocm_mpich, .cleanup_dir]
+  needs: [cray_rocm_mpich_test]
 
 
 
-blueos_cuda_11_gcc_spectrum_tpls:
-  extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .tpls]
+cray_hip_rocm_mpich_tpls:
+  extends: [.cray_resource2, .hip_rocm_mpich, .tpls]
 
-blueos_cuda_11_gcc_spectrum_build:
-  extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .build_and_test]
-  needs: [blueos_cuda_11_gcc_spectrum_tpls]
+cray_hip_rocm_mpich_build:
+  extends: [.cray_resource2, .hip_rocm_mpich, .build_and_test]
+  needs: [cray_hip_rocm_mpich_tpls]
 
-blueos_cuda_11_gcc_spectrum_test:
-  extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .run_ats]
-  needs: [blueos_cuda_11_gcc_spectrum_build]
+cray_hip_rocm_mpich_test:
+  extends: [.cray_resource2, .hip_rocm_mpich, .run_ats]
+  needs: [cray_hip_rocm_mpich_build]
 
-blueos_cuda_11_gcc_spectrum_cleanup:
-  extends: [.blueos_resource2, .cuda_11_gcc_spectrum, .cleanup_dir]
-  needs: [blueos_cuda_11_gcc_spectrum_test]
+cray_hip_rocm_mpich_cleanup:
+  extends: [.cray_resource2, .hip_rocm_mpich, .cleanup_dir]
+  needs: [cray_hip_rocm_mpich_test]
 
 
 

.gitlab/jobs-prod.yml

@@ -4,8 +4,8 @@
 toss_update_tpls:
   extends: [.toss_resource2, .update_tpls, .merge_pr_rule]
 
-blueos_update_tpls:
-  extends: [.blueos_resource2, .update_tpls, .merge_pr_rule]
+cray_update_tpls:
+  extends: [.cray_resource2, .update_tpls, .merge_pr_rule]
   needs: [toss_update_tpls]
 
 # ------------------------------------------------------------------------------
@@ -37,6 +37,6 @@ toss_release_permissions:
 cleanup_old_dirs_toss:
   extends: [.toss_resource_general, .clean_old_dirs]
 
-cleanup_old_dirs_blueos:
-  extends: [.blueos_resource_general, .clean_old_dirs]
+cleanup_old_dirs_cray:
+  extends: [.cray_resource_general, .clean_old_dirs]
 

.gitlab/jobs-seq.yml

@@ -17,33 +17,33 @@ toss_gcc_~mpi_cleanup:
   needs: [toss_gcc_~mpi_test]
 
 
-blueos_cuda_11_gcc_~mpi_tpls:
-  extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .tpls]
+cray_hip_rocm_~mpi_tpls:
+  extends: [.cray_resource2, .hip_rocm_~mpi, .tpls]
 
-blueos_cuda_11_gcc_~mpi_build:
-  extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .build_and_test]
-  needs: [blueos_cuda_11_gcc_~mpi_tpls]
+cray_hip_rocm_~mpi_build:
+  extends: [.cray_resource2, .hip_rocm_~mpi, .build_and_test]
+  needs: [cray_hip_rocm_~mpi_tpls]
 
-blueos_cuda_11_gcc_~mpi_test:
-  extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .run_ats]
-  needs: [blueos_cuda_11_gcc_~mpi_build]
+cray_hip_rocm_~mpi_test:
+  extends: [.cray_resource2, .hip_rocm_~mpi, .run_ats]
+  needs: [cray_hip_rocm_~mpi_build]
 
-blueos_cuda_11_gcc_~mpi_cleanup:
-  extends: [.blueos_resource2, .cuda_11_gcc_~mpi, .cleanup_dir]
-  needs: [blueos_cuda_11_gcc_~mpi_test]
+cray_hip_rocm_~mpi_cleanup:
+  extends: [.cray_resource2, .hip_rocm_~mpi, .cleanup_dir]
+  needs: [cray_hip_rocm_~mpi_test]
 
 
-blueos_gcc_~mpi_Debug_tpls:
-  extends: [.blueos_resource1, .gcc_~mpi_Debug, .tpls]
+cray_rocm_~mpi_Debug_tpls:
+  extends: [.cray_resource1, .rocm_~mpi_Debug, .tpls]
 
-blueos_gcc_~mpi_Debug_build:
-  extends: [.blueos_resource1, .gcc_~mpi_Debug, .build_and_test]
-  needs: [blueos_gcc_~mpi_Debug_tpls]
+cray_rocm_~mpi_Debug_build:
+  extends: [.cray_resource1, .rocm_~mpi_Debug, .build_and_test]
+  needs: [cray_rocm_~mpi_Debug_tpls]
 
-blueos_gcc_~mpi_Debug_test:
-  extends: [.blueos_resource1, .gcc_~mpi_Debug, .run_ats]
-  needs: [blueos_gcc_~mpi_Debug_build]
+cray_rocm_~mpi_Debug_test:
+  extends: [.cray_resource1, .rocm_~mpi_Debug, .run_ats]
+  needs: [cray_rocm_~mpi_Debug_build]
 
-blueos_gcc_~mpi_Debug_cleanup:
-  extends: [.blueos_resource1, .gcc_~mpi_Debug, .cleanup_dir]
-  needs: [blueos_gcc_~mpi_Debug_test]
+cray_rocm_~mpi_Debug_cleanup:
+  extends: [.cray_resource1, .rocm_~mpi_Debug, .cleanup_dir]
+  needs: [cray_rocm_~mpi_Debug_test]

.gitlab/machines.yml

@@ -1,6 +1,18 @@
 # ------------------------------------------------------------------------------
 # MACHINE TEMPLATES
 
+.on_tioga:
+  tags:
+    - tioga
+    - flux
+  variables:
+    SCHEDULER_ACTION: alloc
+    SCHEDULER_PARAMETERS: "--exclusive -N 2 -t 120"
+    NPROC: 112
+    HOSTNAME: 'tioga'
+  timeout: 120 minutes
+  extends: [.on_toss_4_x86_cray]
+
 .on_ruby:
   tags:
     - ruby
@@ -35,6 +47,9 @@
 .blueos_resource_general:
   extends: [.on_lassen]
 
+.cray_resource_general:
+  extends: [.on_tioga]
+
 # ------------------------------------------------------------------------------
 #
 .toss_resource1:
@@ -53,3 +68,10 @@
   #resource_group: blueos2
   extends: [.blueos_resource_general]
 
+.cray_resource1:
+  #resource_group: cray1
+  extends: [.cray_resource_general]
+
+.cray_resource2:
+  #resource_group: cray2
+  extends: [.cray_resource_general]

.gitlab/os.yml

@@ -13,6 +13,15 @@
     GCC_VERSION: '10.3.1'
     CLANG_VERSION: '14.0.6'
     SPHERAL_BUILDS_DIR: /p/lustre1/sphapp/spheral-ci-builds
+    RUN_CMD: 'srun'
+  extends: [.sys_config]
+
+.on_toss_4_x86_cray:
+  variables:
+    ARCH: 'toss_4_x86_64_ib_cray'
+    ROCMCC_VERSION: '6.2.0'
+    SPHERAL_BUILDS_DIR: /p/lustre1/sphapp/spheral-ci-builds
+    RUN_CMD: 'flux run'
   extends: [.sys_config]
 
 .on_blueos_3_ppc64:

.gitlab/scripts.yml

@@ -76,7 +76,7 @@
 
     - ml load mpifileutils
     - cd $SPHERAL_BUILDS_DIR
-    - drm $CI_BUILD_DIR/..
+    - $RUN_CMD -n 20 drm $CI_BUILD_DIR/..
 
 # ------------------------------------------------------------------------------
 # Shared TPL scripts.
@@ -183,7 +183,7 @@
     - echo $DIR_LIST
 
     - ml load mpifileutils
-    - if [[ $DIR_LIST ]]; then drm $DIR_LIST; else echo "No directories to remove at this time."; fi
+    - if [[ $DIR_LIST ]]; then $RUN_CMD -n 20 drm $DIR_LIST; else echo "No directories to remove at this time."; fi
   when: always
 
 .merge_pr_rule:

.gitlab/specs.yml

@@ -25,11 +25,15 @@
   variables:
     SPEC: 'gcc@$GCC_VERSION^spectrum-mpi'
 
+
+
 .clang_mvapich2:
   variables:
     SPEC: 'clang@$CLANG_VERSION^mvapich2'
     EXTRA_CMAKE_ARGS: '-DENABLE_WARNINGS_AS_ERRORS=On -DENABLE_DEV_BUILD=On'
 
+
+
 .cuda_11_gcc_~mpi:
   variables:
     SPEC: 'gcc@$GCC_VERSION+cuda~mpi cuda_arch=70'
@@ -39,7 +43,25 @@
     SPEC: 'gcc@$GCC_VERSION+cuda cuda_arch=70'
     EXTRA_CMAKE_ARGS: '-DENABLE_TIMER=On'
 
-.oneapi_2022_1_mvapich2:
+
+
+.rocm_mpich:
+  variables:
+    SPEC: 'rocmcc@$ROCMCC_VERSION'
+
+.rocm_~mpi:
+  variables:
+    SPEC: 'rocmcc@$ROCMCC_VERSION~mpi'
+
+.rocm_~mpi_Debug:
   variables:
-    SPEC: '[email protected]^mvapich2'
+    SPEC: 'rocmcc@$ROCMCC_VERSION~mpi'
+    EXTRA_CMAKE_ARGS: '-DCMAKE_BUILD_TYPE=Debug -DENABLE_WARNINGS_AS_ERRORS=On'
 
+.hip_rocm_mpich:
+  variables:
+    SPEC: 'rocmcc@$ROCMCC_VERSION+rocm amdgpu_target=gfx942 ^hip@$ROCMCC_VERSION'
+
+.hip_rocm_~mpi:
+  variables:
+    SPEC: 'rocmcc@$ROCMCC_VERSION~mpi+rocm amdgpu_target=gfx942 ^hip@$ROCMCC_VERSION'

.gitmodules

@@ -16,4 +16,6 @@
 	path = extern/chai
 	url = https://github.com/llnl/chai
 	branch = feature/ManagedSharedPtr
-        ignore = all
+[submodule "extern/ATS"]
+	path = extern/ATS
+	url = https://github.com/LLNL/ATS

Dockerfile

@@ -32,6 +32,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update -y
 RUN apt-get upgrade -y
 RUN apt-get install -y build-essential git gfortran mpich autotools-dev autoconf sqlite pkg-config uuid gettext cmake libncurses-dev libgdbm-dev libffi-dev libssl-dev libexpat-dev libreadline-dev libbz2-dev locales python python3 unzip libtool wget curl tk-dev
+RUN apt-get install -y python3-dev python3-venv python3-pip
+RUN apt-get install -y iputils-ping
 
 # Setup system locale for pip package encoding/decoding 
 RUN locale-gen en_US.UTF-8
@@ -40,9 +42,23 @@ RUN locale-gen en_US.UTF-8
 WORKDIR /home/spheral/workspace/
 COPY scripts scripts
 COPY .uberenv_config.json .
+
 RUN python3 scripts/devtools/tpl-manager.py --spec $SPEC --spheral-spack-dir /home
 
+COPY . .
+
+# Configure Spheral with SPEC TPLs.
+RUN mv *.cmake $HOST_CONFIG.cmake
+RUN python3 scripts/devtools/host-config-build.py --host-config $HOST_CONFIG.cmake
+
+# First time install of Spheral pip dependencies
+WORKDIR build_$HOST_CONFIG/build
+RUN make python_build_env
+RUN make python_runtime_env
+
 # Clean workspace once dependencies are installed
+WORKDIR /home/spheral/workspace/
+
 RUN rm -rf /home/spheral/workspace/*
 # -----------------------------------------------------------------------------
 
@@ -58,23 +74,26 @@ ARG HOST_CONFIG=docker-$SPEC
 ARG JCXX=8
 ARG JPY=1
 
+WORKDIR /home/spheral/workspace/
+
 # Copy Spheral source and generate host config from tpl-manager (all dependencies should already be installed).
 COPY . .
 RUN python3 scripts/devtools/tpl-manager.py --spec $SPEC --upstream-dir /home/spack/opt/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder_ --spack-url /home/spack
 
 # Configure Spheral with SPEC TPLs.
 RUN mv *.cmake $HOST_CONFIG.cmake
-RUN python3 scripts/devtools/host-config-build.py --host-config $HOST_CONFIG.cmake
+RUN python3 scripts/devtools/host-config-build.py --host-config $HOST_CONFIG.cmake -DSPHERAL_NETWORK_CONNECTED=Off
 
 # Build Spheral
 WORKDIR build_$HOST_CONFIG/build
+RUN make python_build_env
+RUN make python_runtime_env
 RUN make -j $JCXX Spheral_CXX
 RUN make -j $JPY
 RUN make install
 
 # Run ATS testing suite.
 WORKDIR ../install
-ENV MPLBACKEND=agg
 
 # ATS currently does not allow us to run in parallel for regular linux machines
 # If it did, we would need some of the following commands

RELEASE_NOTES.md

@@ -57,6 +57,7 @@ Notable changes include:
     * Bugfix for RZ solid CRKSPH with compatible energy.
     * Parsing of None string now always becomes None python type. Tests have been updated accordingly.
     * IO for checkpoints and visuzalization can now be properly turned off through SpheralController input options.
+    * Bugfix for atomicWeight in ANEOS
 
 Version v2024.06.1 -- Release date 2024-07-09
 ==============================================

cmake/Compilers.cmake

@@ -15,6 +15,9 @@ set(CXX_WARNING_FLAGS "")
 if (ENABLE_WARNINGS)
   if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
     list(APPEND CXX_WARNING_FLAGS -fdiagnostics-show-option -Wno-unused-command-line-argument -Wno-c++17-extensions)
+    if(CMAKE_CXX_COMPILER_VERSION GREATER_EQUAL 18.0.0)
+      list(APPEND CXX_WARNING_FLAGS -Wno-enum-constexpr-conversion -Wno-deprecated-declarations)
+    endif()
   endif()
 else()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")