Merge pull request #32 from laraPPr/update_OpenFOAM

Update OpenFOAM demo
EESSI · May 8, 2024 · 684ef83 · 684ef83
2 parents 821b4c2 + 4741c27
commit 684ef83
Show file tree

Hide file tree

Showing 4 changed files with 212 additions and 93 deletions.
diff --git a/OpenFOAM/README.md b/OpenFOAM/README.md
@@ -2,6 +2,6 @@ OpenFOAM example
 
 motorBike tutorial case
 
-can be scaled up/down to N cores by changing X, Y, Z in run.sh
+can be scaled up/down to N cores by changing X, Y, Z in the bike scripts.
 
 Runtime (2M cells): ~5min on 8 cores (Intel Skylake)
diff --git a/OpenFOAM/bike_OpenFOAM_v11.sh b/OpenFOAM/bike_OpenFOAM_v11.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+module load OpenFOAM/11-foss-2023a
+
+which ssh &> /dev/null
+if [ $? -ne 0 ]; then
+    # if ssh is not available, set plm_rsh_agent to empty value to avoid OpenMPI failing over it
+    # that's OK, because this is a single-node run
+    export OMPI_MCA_plm_rsh_agent=''
+fi
+
+source $FOAM_BASH
+
+if [ -z $EBROOTOPENFOAM ]; then
+    echo "ERROR: OpenFOAM module not loaded?" >&2
+    exit 1
+fi
+
+# Allow users to define the WORKDIR externally (for example a shared FS for multinode runs)
+export WORKDIR="${WORKDIR:-/tmp/$USER/$$}"
+echo "WORKDIR: $WORKDIR"
+mkdir -p $WORKDIR
+cd $WORKDIR
+pwd
+
+# motorBike, 2M cells
+BLOCKMESH_DIMENSIONS="100 40 40"
+# motorBike, 150M cells
+#BLOCKMESH_DIMENSIONS="200 80 80"
+
+# X*Y*Z should be equal to total number of available cores (across all nodes)
+X=${X:-4}
+Y=${Y:-2}
+Z=${Z:-1}
+# number of nodes
+NODES=${NODES:-1}
+# total number of cores
+NP=$((X * Y * Z))
+# cores per node
+PPN=$(((NP + NODES -1)/NODES))
+
+CASE_NAME=motorBike
+
+if [ -d $CASE_NAME ]; then
+    echo "$CASE_NAME already exists in $PWD!" >&2
+    exit 1
+fi
+
+cp -r $WM_PROJECT_DIR/tutorials/incompressibleFluid/motorBike $CASE_NAME
+chmod -R u+w $CASE_NAME
+cd $CASE_NAME/$CASE_NAME
+pwd
+
+# generate mesh
+# All Foam dictionary sub entries are accssed using / (<main entry>/<sub entry>) rather than . (<main entry>.<sub entry>)
+echo "generating mesh..."
+# Needed to reduce this to a smaller value 200 million is too big for 8 processes, therefore setting to 8 million.
+foamDictionary -entry castellatedMeshControls/maxGlobalCells -set 8000000 system/snappyHexMeshDict
+foamDictionary -entry blocks -set "( hex ( 0 1 2 3 4 5 6 7 ) ( $BLOCKMESH_DIMENSIONS ) simpleGrading ( 1 1 1 ) )" system/blockMeshDict
+foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict
+foamDictionary -entry hierarchicalCoeffs/n -set "($X $Y $Z)" system/decomposeParDict
+
+# this needs to be moved to constant/geometry and not constant/triSurface/
+cp $WM_PROJECT_DIR/tutorials/resources/geometry/motorBike.obj.gz constant/geometry/
+#surfaceFeaturesDict not available.
+# surfaceFeatures 2>&1 | tee log.surfaceFeatures 
+blockMesh 2>&1 | tee log.blockMesh
+decomposePar -copyZero 2>&1 | tee log.decomposePar
+mpirun -np $NP -npernode $PPN snappyHexMesh -parallel -overwrite 2>&1 | tee log.snappyHexMesh
+reconstructPar -constant
+rm -rf ./processor*
+renumberMesh -constant -overwrite 2>&1 | tee log.renumberMesh
+
+# decompose mesh
+echo "decomposing..."
+foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict
+foamDictionary -entry decomposer -set multiLevel system/decomposeParDict # keyword method changed to decomposer
+foamDictionary -entry multiLevelCoeffs -set "{}" system/decomposeParDict
+foamDictionary -entry scotchCoeffs -set "{}" system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs/level0 -set "{}" system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs/level0/numberOfSubdomains -set $NODES system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs/level0/method -set scotch system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs/level1 -set "{}" system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs/level1/numberOfSubdomains -set $PPN system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs/level1/method -set scotch system/decomposeParDict
+
+decomposePar -copyZero 2>&1 | tee log.decomposeParMultiLevel
+
+# run simulation
+echo "running..."
+# limit run to first 200 time steps
+foamDictionary -entry endTime -set 200 system/controlDict
+foamDictionary -entry writeInterval -set 1000 system/controlDict
+foamDictionary -entry runTimeModifiable -set "false" system/controlDict
+foamDictionary -entry functions -set "{}" system/controlDict
+
+mpirun -np $NP potentialFoam -parallel 2>&1 | tee log.potentialFoam
+time mpirun -np $NP simpleFoam -parallel 2>&1 | tee log.simpleFoam
+
+echo "cleanup..."
+rm -rf $WORKDIR
diff --git a/OpenFOAM/bike_OpenFOAM_v8.sh b/OpenFOAM/bike_OpenFOAM_v8.sh
@@ -0,0 +1,96 @@
+#!/bin/bash
+
+module load OpenFOAM/8-foss-2020a
+
+which ssh &> /dev/null
+if [ $? -ne 0 ]; then
+    # if ssh is not available, set plm_rsh_agent to empty value to avoid OpenMPI failing over it
+    # that's OK, because this is a single-node run
+    export OMPI_MCA_plm_rsh_agent=''
+fi
+
+source $FOAM_BASH
+
+if [ -z $EBROOTOPENFOAM ]; then
+    echo "ERROR: OpenFOAM module not loaded?" >&2
+    exit 1
+fi
+
+# Allow users to define the WORKDIR externally (for example a shared FS for multinode runs)
+export WORKDIR="${WORKDIR:-/tmp/$USER/$$}"
+echo "WORKDIR: $WORKDIR"
+mkdir -p $WORKDIR
+cd $WORKDIR
+pwd
+
+# motorBike, 2M cells
+BLOCKMESH_DIMENSIONS="100 40 40"
+# motorBike, 150M cells
+#BLOCKMESH_DIMENSIONS="200 80 80"
+
+# X*Y*Z should be equal to total number of available cores (across all nodes)
+X=${X:-4}
+Y=${Y:-2}
+Z=${Z:-1}
+# number of nodes
+NODES=${NODES:-1}
+# total number of cores
+NP=$((X * Y * Z))
+# cores per node
+PPN=$(((NP + NODES -1)/NODES))
+
+CASE_NAME=motorBike
+
+if [ -d $CASE_NAME ]; then
+    echo "$CASE_NAME already exists in $PWD!" >&2
+    exit 1
+fi
+
+cp -r $WM_PROJECT_DIR/tutorials/incompressible/simpleFoam/motorBike $CASE_NAME
+cd $CASE_NAME
+pwd
+
+# generate mesh
+echo "generating mesh..."
+foamDictionary  -entry castellatedMeshControls.maxGlobalCells -set 200000000 system/snappyHexMeshDict
+foamDictionary -entry blocks -set "( hex ( 0 1 2 3 4 5 6 7 ) ( $BLOCKMESH_DIMENSIONS ) simpleGrading ( 1 1 1 ) )" system/blockMeshDict
+foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict
+foamDictionary -entry hierarchicalCoeffs.n -set "($X $Y $Z)" system/decomposeParDict
+
+cp $WM_PROJECT_DIR/tutorials/resources/geometry/motorBike.obj.gz constant/triSurface/
+surfaceFeatures 2>&1 | tee log.surfaceFeatures
+blockMesh 2>&1 | tee log.blockMesh
+decomposePar -copyZero 2>&1 | tee log.decomposePar
+mpirun -np $NP -ppn $PPN snappyHexMesh -parallel -overwrite 2>&1 | tee log.snappyHexMesh
+reconstructParMesh -constant
+rm -rf ./processor*
+renumberMesh -constant -overwrite 2>&1 | tee log.renumberMesh
+
+# decompose mesh
+echo "decomposing..."
+foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict
+foamDictionary -entry method -set multiLevel system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs -set "{}" system/decomposeParDict
+foamDictionary -entry scotchCoeffs -set "{}" system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs.level0 -set "{}" system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs.level0.numberOfSubdomains -set $NODES system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs.level0.method -set scotch system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs.level1 -set "{}" system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs.level1.numberOfSubdomains -set $PPN system/decomposeParDict
+foamDictionary -entry multiLevelCoeffs.level1.method -set scotch system/decomposeParDict
+
+decomposePar -copyZero 2>&1 | tee log.decomposeParMultiLevel
+
+# run simulation
+echo "running..."
+# limit run to first 200 time steps
+foamDictionary -entry endTime -set 200 system/controlDict
+foamDictionary -entry writeInterval -set 1000 system/controlDict
+foamDictionary -entry runTimeModifiable -set "false" system/controlDict
+foamDictionary -entry functions -set "{}" system/controlDict
+
+mpirun -np $NP potentialFoam -parallel 2>&1 | tee log.potentialFoam
+time mpirun -np $NP simpleFoam -parallel 2>&1 | tee log.simpleFoam
+
+echo "cleanup..."
+rm -rf $WORKDIR
diff --git a/OpenFOAM/run.sh b/OpenFOAM/run.sh
@@ -1,96 +1,18 @@
 #!/bin/bash
 
-module load OpenFOAM/8-foss-2020a
-
-which ssh &> /dev/null
-if [ $? -ne 0 ]; then
-    # if ssh is not available, set plm_rsh_agent to empty value to avoid OpenMPI failing over it
-    # that's OK, because this is a single-node run
-    export OMPI_MCA_plm_rsh_agent=''
-fi
-
-source $FOAM_BASH
-
-if [ -z $EBROOTOPENFOAM ]; then
-    echo "ERROR: OpenFOAM module not loaded?" >&2
+if [[ $EESSI_CVMFS_REPO == "/cvmfs/software.eessi.io" ]] && [[ $EESSI_VERSION == "2023.06" ]]; then
+    echo "Running demo for OpenFOAM v11 ..."
+    export OMPI_MCA_rmaps_base_oversubscribe=true
+    ./bike_OpenFOAM_v11.sh
+elif [[ $EESSI_CVMFS_REPO == "/cvmfs/pilot.eessi-hpc.org" ]] && [[ $EESSI_PILOT_VERSION == "2021.12" ]]; then
+    echo "Running demo for OpenFOAM v8 ..."
+    export OMPI_MCA_rmaps_base_oversubscribe=true
+    ./bike_OpenFOAM_v8.sh
+elif [[ $EESSI_CVMFS_REPO == "/cvmfs/pilot.eessi-hpc.org" ]] && [[ $EESSI_PILOT_VERSION == "2023.06" ]]; then 
+    echo There is no demo for OpenFOAM in "/cvmfs/pilot.eessi-hpc.org/versions/2023.06". <&2
+    echo Please use the EESSI production repo "/cvmfs/software.eessi.io".<&2
+    exit 1;
+else 
+    echo "Don't know which OpenFOAM module to load for ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}$EESSI_PILOT_VERSION" >&2
     exit 1
 fi
-
-# Allow users to define the WORKDIR externally (for example a shared FS for multinode runs)
-export WORKDIR="${WORKDIR:-/tmp/$USER/$$}"
-echo "WORKDIR: $WORKDIR"
-mkdir -p $WORKDIR
-cd $WORKDIR
-pwd
-
-# motorBike, 2M cells
-BLOCKMESH_DIMENSIONS="100 40 40"
-# motorBike, 150M cells
-#BLOCKMESH_DIMENSIONS="200 80 80"
-
-# X*Y*Z should be equal to total number of available cores (across all nodes)
-X=${X:-4}
-Y=${Y:-2}
-Z=${Z:-1}
-# number of nodes
-NODES=${NODES:-1}
-# total number of cores
-NP=$((X * Y * Z))
-# cores per node
-PPN=$(((NP + NODES -1)/NODES))
-
-CASE_NAME=motorBike
-
-if [ -d $CASE_NAME ]; then
-    echo "$CASE_NAME already exists in $PWD!" >&2
-    exit 1
-fi
-
-cp -r $WM_PROJECT_DIR/tutorials/incompressible/simpleFoam/motorBike $CASE_NAME
-cd $CASE_NAME
-pwd
-
-# generate mesh
-echo "generating mesh..."
-foamDictionary  -entry castellatedMeshControls.maxGlobalCells -set 200000000 system/snappyHexMeshDict
-foamDictionary -entry blocks -set "( hex ( 0 1 2 3 4 5 6 7 ) ( $BLOCKMESH_DIMENSIONS ) simpleGrading ( 1 1 1 ) )" system/blockMeshDict
-foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict
-foamDictionary -entry hierarchicalCoeffs.n -set "($X $Y $Z)" system/decomposeParDict
-
-cp $WM_PROJECT_DIR/tutorials/resources/geometry/motorBike.obj.gz constant/triSurface/
-surfaceFeatures 2>&1 | tee log.surfaceFeatures
-blockMesh 2>&1 | tee log.blockMesh
-decomposePar -copyZero 2>&1 | tee log.decomposePar
-mpirun -np $NP -ppn $PPN -hostfile hostlist snappyHexMesh -parallel -overwrite 2>&1 | tee log.snappyHexMesh
-reconstructParMesh -constant
-rm -rf ./processor*
-renumberMesh -constant -overwrite 2>&1 | tee log.renumberMesh
-
-# decompose mesh
-echo "decomposing..."
-foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict
-foamDictionary -entry method -set multiLevel system/decomposeParDict
-foamDictionary -entry multiLevelCoeffs -set "{}" system/decomposeParDict
-foamDictionary -entry scotchCoeffs -set "{}" system/decomposeParDict
-foamDictionary -entry multiLevelCoeffs.level0 -set "{}" system/decomposeParDict
-foamDictionary -entry multiLevelCoeffs.level0.numberOfSubdomains -set $NODES system/decomposeParDict
-foamDictionary -entry multiLevelCoeffs.level0.method -set scotch system/decomposeParDict
-foamDictionary -entry multiLevelCoeffs.level1 -set "{}" system/decomposeParDict
-foamDictionary -entry multiLevelCoeffs.level1.numberOfSubdomains -set $PPN system/decomposeParDict
-foamDictionary -entry multiLevelCoeffs.level1.method -set scotch system/decomposeParDict
-
-decomposePar -copyZero 2>&1 | tee log.decomposeParMultiLevel
-
-# run simulation
-echo "running..."
-# limit run to first 200 time steps
-foamDictionary -entry endTime -set 200 system/controlDict
-foamDictionary -entry writeInterval -set 1000 system/controlDict
-foamDictionary -entry runTimeModifiable -set "false" system/controlDict
-foamDictionary -entry functions -set "{}" system/controlDict
-
-mpirun --oversubscribe -np $NP potentialFoam -parallel 2>&1 | tee log.potentialFoam
-time mpirun --oversubscribe -np $NP simpleFoam -parallel 2>&1 | tee log.simpleFoam
-
-echo "cleanup..."
-rm -rf $WORKDIR