diff --git a/OpenFOAM/README.md b/OpenFOAM/README.md index 36c35b5..2b51d5e 100644 --- a/OpenFOAM/README.md +++ b/OpenFOAM/README.md @@ -2,6 +2,6 @@ OpenFOAM example motorBike tutorial case -can be scaled up/down to N cores by changing X, Y, Z in run.sh +can be scaled up/down to N cores by changing X, Y, Z in the bike scripts. Runtime (2M cells): ~5min on 8 cores (Intel Skylake) diff --git a/OpenFOAM/bike_OpenFOAM_v11.sh b/OpenFOAM/bike_OpenFOAM_v11.sh new file mode 100755 index 0000000..22ddd27 --- /dev/null +++ b/OpenFOAM/bike_OpenFOAM_v11.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +module load OpenFOAM/11-foss-2023a + +which ssh &> /dev/null +if [ $? -ne 0 ]; then + # if ssh is not available, set plm_rsh_agent to empty value to avoid OpenMPI failing over it + # that's OK, because this is a single-node run + export OMPI_MCA_plm_rsh_agent='' +fi + +source $FOAM_BASH + +if [ -z $EBROOTOPENFOAM ]; then + echo "ERROR: OpenFOAM module not loaded?" >&2 + exit 1 +fi + +# Allow users to define the WORKDIR externally (for example a shared FS for multinode runs) +export WORKDIR="${WORKDIR:-/tmp/$USER/$$}" +echo "WORKDIR: $WORKDIR" +mkdir -p $WORKDIR +cd $WORKDIR +pwd + +# motorBike, 2M cells +BLOCKMESH_DIMENSIONS="100 40 40" +# motorBike, 150M cells +#BLOCKMESH_DIMENSIONS="200 80 80" + +# X*Y*Z should be equal to total number of available cores (across all nodes) +X=${X:-4} +Y=${Y:-2} +Z=${Z:-1} +# number of nodes +NODES=${NODES:-1} +# total number of cores +NP=$((X * Y * Z)) +# cores per node +PPN=$(((NP + NODES -1)/NODES)) + +CASE_NAME=motorBike + +if [ -d $CASE_NAME ]; then + echo "$CASE_NAME already exists in $PWD!" >&2 + exit 1 +fi + +cp -r $WM_PROJECT_DIR/tutorials/incompressibleFluid/motorBike $CASE_NAME +chmod -R u+w $CASE_NAME +cd $CASE_NAME/$CASE_NAME +pwd + +# generate mesh +# All Foam dictionary sub entries are accssed using / (
/) rather than . (
.) +echo "generating mesh..." +# Needed to reduce this to a smaller value 200 million is too big for 8 processes, therefore setting to 8 million. +foamDictionary -entry castellatedMeshControls/maxGlobalCells -set 8000000 system/snappyHexMeshDict +foamDictionary -entry blocks -set "( hex ( 0 1 2 3 4 5 6 7 ) ( $BLOCKMESH_DIMENSIONS ) simpleGrading ( 1 1 1 ) )" system/blockMeshDict +foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict +foamDictionary -entry hierarchicalCoeffs/n -set "($X $Y $Z)" system/decomposeParDict + +# this needs to be moved to constant/geometry and not constant/triSurface/ +cp $WM_PROJECT_DIR/tutorials/resources/geometry/motorBike.obj.gz constant/geometry/ +#surfaceFeaturesDict not available. +# surfaceFeatures 2>&1 | tee log.surfaceFeatures +blockMesh 2>&1 | tee log.blockMesh +decomposePar -copyZero 2>&1 | tee log.decomposePar +mpirun -np $NP -npernode $PPN snappyHexMesh -parallel -overwrite 2>&1 | tee log.snappyHexMesh +reconstructPar -constant +rm -rf ./processor* +renumberMesh -constant -overwrite 2>&1 | tee log.renumberMesh + +# decompose mesh +echo "decomposing..." +foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict +foamDictionary -entry decomposer -set multiLevel system/decomposeParDict # keyword method changed to decomposer +foamDictionary -entry multiLevelCoeffs -set "{}" system/decomposeParDict +foamDictionary -entry scotchCoeffs -set "{}" system/decomposeParDict +foamDictionary -entry multiLevelCoeffs/level0 -set "{}" system/decomposeParDict +foamDictionary -entry multiLevelCoeffs/level0/numberOfSubdomains -set $NODES system/decomposeParDict +foamDictionary -entry multiLevelCoeffs/level0/method -set scotch system/decomposeParDict +foamDictionary -entry multiLevelCoeffs/level1 -set "{}" system/decomposeParDict +foamDictionary -entry multiLevelCoeffs/level1/numberOfSubdomains -set $PPN system/decomposeParDict +foamDictionary -entry multiLevelCoeffs/level1/method -set scotch system/decomposeParDict + +decomposePar -copyZero 2>&1 | tee log.decomposeParMultiLevel + +# run simulation +echo "running..." +# limit run to first 200 time steps +foamDictionary -entry endTime -set 200 system/controlDict +foamDictionary -entry writeInterval -set 1000 system/controlDict +foamDictionary -entry runTimeModifiable -set "false" system/controlDict +foamDictionary -entry functions -set "{}" system/controlDict + +mpirun -np $NP potentialFoam -parallel 2>&1 | tee log.potentialFoam +time mpirun -np $NP simpleFoam -parallel 2>&1 | tee log.simpleFoam + +echo "cleanup..." +rm -rf $WORKDIR diff --git a/OpenFOAM/bike_OpenFOAM_v8.sh b/OpenFOAM/bike_OpenFOAM_v8.sh new file mode 100755 index 0000000..fd0b101 --- /dev/null +++ b/OpenFOAM/bike_OpenFOAM_v8.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +module load OpenFOAM/8-foss-2020a + +which ssh &> /dev/null +if [ $? -ne 0 ]; then + # if ssh is not available, set plm_rsh_agent to empty value to avoid OpenMPI failing over it + # that's OK, because this is a single-node run + export OMPI_MCA_plm_rsh_agent='' +fi + +source $FOAM_BASH + +if [ -z $EBROOTOPENFOAM ]; then + echo "ERROR: OpenFOAM module not loaded?" >&2 + exit 1 +fi + +# Allow users to define the WORKDIR externally (for example a shared FS for multinode runs) +export WORKDIR="${WORKDIR:-/tmp/$USER/$$}" +echo "WORKDIR: $WORKDIR" +mkdir -p $WORKDIR +cd $WORKDIR +pwd + +# motorBike, 2M cells +BLOCKMESH_DIMENSIONS="100 40 40" +# motorBike, 150M cells +#BLOCKMESH_DIMENSIONS="200 80 80" + +# X*Y*Z should be equal to total number of available cores (across all nodes) +X=${X:-4} +Y=${Y:-2} +Z=${Z:-1} +# number of nodes +NODES=${NODES:-1} +# total number of cores +NP=$((X * Y * Z)) +# cores per node +PPN=$(((NP + NODES -1)/NODES)) + +CASE_NAME=motorBike + +if [ -d $CASE_NAME ]; then + echo "$CASE_NAME already exists in $PWD!" >&2 + exit 1 +fi + +cp -r $WM_PROJECT_DIR/tutorials/incompressible/simpleFoam/motorBike $CASE_NAME +cd $CASE_NAME +pwd + +# generate mesh +echo "generating mesh..." +foamDictionary -entry castellatedMeshControls.maxGlobalCells -set 200000000 system/snappyHexMeshDict +foamDictionary -entry blocks -set "( hex ( 0 1 2 3 4 5 6 7 ) ( $BLOCKMESH_DIMENSIONS ) simpleGrading ( 1 1 1 ) )" system/blockMeshDict +foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict +foamDictionary -entry hierarchicalCoeffs.n -set "($X $Y $Z)" system/decomposeParDict + +cp $WM_PROJECT_DIR/tutorials/resources/geometry/motorBike.obj.gz constant/triSurface/ +surfaceFeatures 2>&1 | tee log.surfaceFeatures +blockMesh 2>&1 | tee log.blockMesh +decomposePar -copyZero 2>&1 | tee log.decomposePar +mpirun -np $NP -ppn $PPN snappyHexMesh -parallel -overwrite 2>&1 | tee log.snappyHexMesh +reconstructParMesh -constant +rm -rf ./processor* +renumberMesh -constant -overwrite 2>&1 | tee log.renumberMesh + +# decompose mesh +echo "decomposing..." +foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict +foamDictionary -entry method -set multiLevel system/decomposeParDict +foamDictionary -entry multiLevelCoeffs -set "{}" system/decomposeParDict +foamDictionary -entry scotchCoeffs -set "{}" system/decomposeParDict +foamDictionary -entry multiLevelCoeffs.level0 -set "{}" system/decomposeParDict +foamDictionary -entry multiLevelCoeffs.level0.numberOfSubdomains -set $NODES system/decomposeParDict +foamDictionary -entry multiLevelCoeffs.level0.method -set scotch system/decomposeParDict +foamDictionary -entry multiLevelCoeffs.level1 -set "{}" system/decomposeParDict +foamDictionary -entry multiLevelCoeffs.level1.numberOfSubdomains -set $PPN system/decomposeParDict +foamDictionary -entry multiLevelCoeffs.level1.method -set scotch system/decomposeParDict + +decomposePar -copyZero 2>&1 | tee log.decomposeParMultiLevel + +# run simulation +echo "running..." +# limit run to first 200 time steps +foamDictionary -entry endTime -set 200 system/controlDict +foamDictionary -entry writeInterval -set 1000 system/controlDict +foamDictionary -entry runTimeModifiable -set "false" system/controlDict +foamDictionary -entry functions -set "{}" system/controlDict + +mpirun -np $NP potentialFoam -parallel 2>&1 | tee log.potentialFoam +time mpirun -np $NP simpleFoam -parallel 2>&1 | tee log.simpleFoam + +echo "cleanup..." +rm -rf $WORKDIR diff --git a/OpenFOAM/run.sh b/OpenFOAM/run.sh index d3febd0..3c7c0b3 100755 --- a/OpenFOAM/run.sh +++ b/OpenFOAM/run.sh @@ -1,96 +1,18 @@ #!/bin/bash -module load OpenFOAM/8-foss-2020a - -which ssh &> /dev/null -if [ $? -ne 0 ]; then - # if ssh is not available, set plm_rsh_agent to empty value to avoid OpenMPI failing over it - # that's OK, because this is a single-node run - export OMPI_MCA_plm_rsh_agent='' -fi - -source $FOAM_BASH - -if [ -z $EBROOTOPENFOAM ]; then - echo "ERROR: OpenFOAM module not loaded?" >&2 +if [[ $EESSI_CVMFS_REPO == "/cvmfs/software.eessi.io" ]] && [[ $EESSI_VERSION == "2023.06" ]]; then + echo "Running demo for OpenFOAM v11 ..." + export OMPI_MCA_rmaps_base_oversubscribe=true + ./bike_OpenFOAM_v11.sh +elif [[ $EESSI_CVMFS_REPO == "/cvmfs/pilot.eessi-hpc.org" ]] && [[ $EESSI_PILOT_VERSION == "2021.12" ]]; then + echo "Running demo for OpenFOAM v8 ..." + export OMPI_MCA_rmaps_base_oversubscribe=true + ./bike_OpenFOAM_v8.sh +elif [[ $EESSI_CVMFS_REPO == "/cvmfs/pilot.eessi-hpc.org" ]] && [[ $EESSI_PILOT_VERSION == "2023.06" ]]; then + echo There is no demo for OpenFOAM in "/cvmfs/pilot.eessi-hpc.org/versions/2023.06". <&2 + echo Please use the EESSI production repo "/cvmfs/software.eessi.io".<&2 + exit 1; +else + echo "Don't know which OpenFOAM module to load for ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}$EESSI_PILOT_VERSION" >&2 exit 1 fi - -# Allow users to define the WORKDIR externally (for example a shared FS for multinode runs) -export WORKDIR="${WORKDIR:-/tmp/$USER/$$}" -echo "WORKDIR: $WORKDIR" -mkdir -p $WORKDIR -cd $WORKDIR -pwd - -# motorBike, 2M cells -BLOCKMESH_DIMENSIONS="100 40 40" -# motorBike, 150M cells -#BLOCKMESH_DIMENSIONS="200 80 80" - -# X*Y*Z should be equal to total number of available cores (across all nodes) -X=${X:-4} -Y=${Y:-2} -Z=${Z:-1} -# number of nodes -NODES=${NODES:-1} -# total number of cores -NP=$((X * Y * Z)) -# cores per node -PPN=$(((NP + NODES -1)/NODES)) - -CASE_NAME=motorBike - -if [ -d $CASE_NAME ]; then - echo "$CASE_NAME already exists in $PWD!" >&2 - exit 1 -fi - -cp -r $WM_PROJECT_DIR/tutorials/incompressible/simpleFoam/motorBike $CASE_NAME -cd $CASE_NAME -pwd - -# generate mesh -echo "generating mesh..." -foamDictionary -entry castellatedMeshControls.maxGlobalCells -set 200000000 system/snappyHexMeshDict -foamDictionary -entry blocks -set "( hex ( 0 1 2 3 4 5 6 7 ) ( $BLOCKMESH_DIMENSIONS ) simpleGrading ( 1 1 1 ) )" system/blockMeshDict -foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict -foamDictionary -entry hierarchicalCoeffs.n -set "($X $Y $Z)" system/decomposeParDict - -cp $WM_PROJECT_DIR/tutorials/resources/geometry/motorBike.obj.gz constant/triSurface/ -surfaceFeatures 2>&1 | tee log.surfaceFeatures -blockMesh 2>&1 | tee log.blockMesh -decomposePar -copyZero 2>&1 | tee log.decomposePar -mpirun -np $NP -ppn $PPN -hostfile hostlist snappyHexMesh -parallel -overwrite 2>&1 | tee log.snappyHexMesh -reconstructParMesh -constant -rm -rf ./processor* -renumberMesh -constant -overwrite 2>&1 | tee log.renumberMesh - -# decompose mesh -echo "decomposing..." -foamDictionary -entry numberOfSubdomains -set $NP system/decomposeParDict -foamDictionary -entry method -set multiLevel system/decomposeParDict -foamDictionary -entry multiLevelCoeffs -set "{}" system/decomposeParDict -foamDictionary -entry scotchCoeffs -set "{}" system/decomposeParDict -foamDictionary -entry multiLevelCoeffs.level0 -set "{}" system/decomposeParDict -foamDictionary -entry multiLevelCoeffs.level0.numberOfSubdomains -set $NODES system/decomposeParDict -foamDictionary -entry multiLevelCoeffs.level0.method -set scotch system/decomposeParDict -foamDictionary -entry multiLevelCoeffs.level1 -set "{}" system/decomposeParDict -foamDictionary -entry multiLevelCoeffs.level1.numberOfSubdomains -set $PPN system/decomposeParDict -foamDictionary -entry multiLevelCoeffs.level1.method -set scotch system/decomposeParDict - -decomposePar -copyZero 2>&1 | tee log.decomposeParMultiLevel - -# run simulation -echo "running..." -# limit run to first 200 time steps -foamDictionary -entry endTime -set 200 system/controlDict -foamDictionary -entry writeInterval -set 1000 system/controlDict -foamDictionary -entry runTimeModifiable -set "false" system/controlDict -foamDictionary -entry functions -set "{}" system/controlDict - -mpirun --oversubscribe -np $NP potentialFoam -parallel 2>&1 | tee log.potentialFoam -time mpirun --oversubscribe -np $NP simpleFoam -parallel 2>&1 | tee log.simpleFoam - -echo "cleanup..." -rm -rf $WORKDIR