Skip to content

Commit

Permalink
Merge pull request #672 from GEOS-ESM/develop
Browse files Browse the repository at this point in the history 
Sync develop into main
  • Loading branch information
@sdrabenh
sdrabenh authored Nov 17, 2022
2 parents b6730bf + 8e6662a commit 4961a99
Show file tree
Hide file tree
Showing 22 changed files with 2,991 additions and 4,443 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ version: 2.1

# Anchors to prevent forgetting to update a version
baselibs_version: &baselibs_version v7.5.0
bcs_version: &bcs_version v10.22.5
bcs_version: &bcs_version v10.23.0

orbs:
ci: geos-esm/circleci-tools@1
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/workflow.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:
if: "!contains(github.event.pull_request.labels.*.name, '0 diff trivial')"
runs-on: ubuntu-latest
container:
image: gmao/ubuntu20-geos-env-mkl:v7.5.0-openmpi_4.1.2-gcc_11.2.0
image: gmao/ubuntu20-geos-env-mkl:v7.5.0-openmpi_4.1.4-gcc_12.1.0
credentials:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
Expand Down
883 changes: 25 additions & 858 deletions GEOSagcm_GridComp/GEOSphysics_GridComp/GEOSgwd_GridComp/GEOS_GwdGridComp.F90
View file

Large diffs are not rendered by default.

179 changes: 18 additions & 161 deletions GEOSagcm_GridComp/GEOSphysics_GridComp/GEOSgwd_GridComp/gw_drag.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,106 +20,16 @@ module gw_drag
implicit none

!save
#ifndef _CUDA
private ! Make default type private to the module
!
! PUBLIC: interfaces
!
public gw_intr ! interface to actual parameterization
#endif

!
! PRIVATE: Rest of the data and interfaces are private to this module
!

#ifdef _CUDA

! Inputs
! ------

real, allocatable, dimension(:,:), device :: pint_dev ! pressure at the layer edges
real, allocatable, dimension(:,:), device :: t_dev ! temperature at layers
real, allocatable, dimension(:,:), device :: u_dev ! zonal wind at layers
real, allocatable, dimension(:,:), device :: v_dev ! meridional wind at layers
real, allocatable, dimension(: ), device :: sgh_dev ! standard deviation of orography
real, allocatable, dimension(: ), device :: pref_dev ! reference pressure at the layeredges
real, allocatable, dimension(:,:), device :: pmid_dev ! pressure at the layers
real, allocatable, dimension(:,:), device :: pdel_dev ! pressure thickness at the layers
real, allocatable, dimension(:,:), device :: rpdel_dev ! 1.0 / pdel
real, allocatable, dimension(:,:), device :: lnpint_dev ! log(pint)
real, allocatable, dimension(:,:), device :: zm_dev ! height above surface at layers
real, allocatable, dimension(: ), device :: rlat_dev ! latitude in radian

! Inoutputs
! ---------

! Outputs
! -------

real, allocatable, dimension(:,:), device :: dudt_gwd_dev ! zonal wind tendency at layer
real, allocatable, dimension(:,:), device :: dvdt_gwd_dev ! meridional wind tendency at layer
real, allocatable, dimension(:,:), device :: dtdt_gwd_dev ! temperature tendency at layer
real, allocatable, dimension(:,:), device :: dudt_org_dev ! zonal wind tendency at layer due to orography GWD
real, allocatable, dimension(:,:), device :: dvdt_org_dev ! meridional wind tendency at layer due to orography GWD
real, allocatable, dimension(:,:), device :: dtdt_org_dev ! temperature tendency at layer due to orography GWD
real, allocatable, dimension(: ), device :: taugwdx_dev ! zonal gravity wave surface stress
real, allocatable, dimension(: ), device :: taugwdy_dev ! meridional gravity wave surface stress
real, allocatable, dimension(:,:), device :: tauox_dev ! zonal orographic gravity wave stress
real, allocatable, dimension(:,:), device :: tauoy_dev ! meridional orographic gravity wave stress
real, allocatable, dimension(:,:), device :: feo_dev ! energy flux of orographic gravity waves
real, allocatable, dimension(:,:), device :: fepo_dev ! pseudoenergy flux of orographic gravity waves
real, allocatable, dimension(: ), device :: taubkgx_dev ! zonal gravity wave background stress
real, allocatable, dimension(: ), device :: taubkgy_dev ! meridional gravity wave background stress
real, allocatable, dimension(:,:), device :: taubx_dev ! zonal background gravity wave stress
real, allocatable, dimension(:,:), device :: tauby_dev ! meridional background gravity wave stress
real, allocatable, dimension(:,:), device :: feb_dev ! energy flux of background gravity waves
real, allocatable, dimension(:,:), device :: fepb_dev ! pseudoenergy flux of background gravity waves
real, allocatable, dimension(:,:), device :: utbsrc_dev ! dU/dt below background launch level
real, allocatable, dimension(:,:), device :: vtbsrc_dev ! dV/dt below background launch level
real, allocatable, dimension(:,:), device :: ttbsrc_dev ! dT/dt below background launch level

! ------------
! GEOPOTENTIAL
! ------------

! Inputs
! ------

real, allocatable, dimension(:,:), device :: pmln_dev ! Log midpoint pressures
real, allocatable, dimension(:,:), device :: q_dev ! specific humidity

! Outputs
! -------

real, allocatable, dimension(:,:), device :: zi_dev ! Height above surface at interfaces

! --------
! POSTINTR
! --------

! Outputs
! -------

real, allocatable, dimension(:,:), device :: dudt_tot_dev ! Tendency of eastward wind due to GWD
real, allocatable, dimension(:,:), device :: dvdt_tot_dev ! Tendency of northward wind due to GWD
real, allocatable, dimension(:,:), device :: dtdt_tot_dev ! Tendency of air temperature due to GWD

real, allocatable, dimension(:,:), device :: dudt_rah_dev ! Tendency of eastward wind due to Rayleigh friction
real, allocatable, dimension(:,:), device :: dvdt_rah_dev ! Tendency of northward wind due to Rayleigh friction
real, allocatable, dimension(:,:), device :: dtdt_rah_dev ! Tendency of air temperature due to Rayleigh friction

real, allocatable, dimension(: ), device :: pegwd_dev ! Potential energy tendency across GWD
real, allocatable, dimension(: ), device :: peoro_dev ! Potential energy tendency due to orographic gravity
real, allocatable, dimension(: ), device :: peray_dev ! Potential energy tendency due to Rayleigh friction
real, allocatable, dimension(: ), device :: pebkg_dev ! Potential energy tendency due to gw background
real, allocatable, dimension(: ), device :: kegwd_dev ! Kinetic energy tendency across GWD
real, allocatable, dimension(: ), device :: keoro_dev ! Kinetic energy tendency due to orographic gravity
real, allocatable, dimension(: ), device :: keray_dev ! Kinetic energy tendency due to Rayleigh friction
real, allocatable, dimension(: ), device :: kebkg_dev ! Kinetic energy tendency due to gw background
real, allocatable, dimension(: ), device :: keres_dev ! Kinetic energy residual for total energy conservation
real, allocatable, dimension(: ), device :: bkgerr_dev ! Kinetic energy residual for BKG energy conservation
#endif

real, parameter :: KWVB = 6.28e-5 ! effective horizontal wave number for background
real, parameter :: KWVBEQ = 6.28e-5/7. ! effective horizontal wave number for background
real, parameter :: KWVO = 6.28e-5 ! effective horizontal wave number for orographic
Expand All @@ -146,14 +56,6 @@ module gw_drag

!===============================================================================

! GPU: The GPU call has a different structure only due to limitations of
! CUDA. CUDA doesn't support calls with too many variables, so we must
! use USE association to get around this. Hopefully this will change
! in the future.

#ifdef _CUDA
attributes(global) subroutine gw_intr (pcols,pver,dt,pgwv, bgstressmax, effgworo, effgwbkg)
#else
subroutine gw_intr (pcols, pver, dt, pgwv, &
pint_dev, t_dev, u_dev, v_dev, sgh_dev, pref_dev, &
pmid_dev, pdel_dev, rpdel_dev, lnpint_dev, zm_dev, rlat_dev, &
Expand All @@ -163,22 +65,11 @@ subroutine gw_intr (pcols, pver, dt, pgwv,
taubkgx_dev, taubkgy_dev, taubx_dev, tauby_dev, feb_dev, &
fepo_dev, fepb_dev, utbsrc_dev, vtbsrc_dev, ttbsrc_dev, &
bgstressmax, effgworo, effgwbkg, rc )
#endif

!-----------------------------------------------------------------------
! Interface for multiple gravity wave drag parameterization.
!-----------------------------------------------------------------------

#ifdef _CUDA
!------------------------------Arguments--------------------------------
integer, intent(in), value :: pcols ! number of columns
integer, intent(in), value :: pver ! number of vertical layers
real, intent(in), value :: dt ! time step
integer, intent(in), value :: pgwv ! number of waves allowed (Default = 4, 0 nullifies)
real , intent(in), value :: bgstressmax ! Max of equatorial profile of BG stress factor
real, intent(in), value :: effgwbkg ! tendency efficiency for background gwd (Default = 0.125)
real, intent(in), value :: effgworo ! tendency efficiency for orographic gwd (Default = 0.125)
#else
!------------------------------Arguments--------------------------------
integer, intent(in ) :: pcols ! number of columns
integer, intent(in ) :: pver ! number of vertical layers
Expand Down Expand Up @@ -223,18 +114,6 @@ subroutine gw_intr (pcols, pver, dt, pgwv,
real, intent( out) :: ttbsrc_dev(pcols,pver) ! dT/dt below background launch level

integer, optional, intent(out) :: RC ! return code
#endif

! GPU: CUDA must know the size of local arrays during compile-time. Thus, the automatic
! arrays below are static. These ifdefs make them automatic on the CPU.

#ifndef GPU_MAXLEVS
#define GPU_MAXLEVS pver
#endif

#ifndef MAXPGWV
#define MAXPGWV pgwv
#endif

!---------------------------Local storage-------------------------------

Expand All @@ -247,33 +126,33 @@ subroutine gw_intr (pcols, pver, dt, pgwv,
integer :: ksrc ! index of top interface of source region
integer :: ksrcmn ! min value of ksrc

real :: ttgw(GPU_MAXLEVS) ! temperature tendency
real :: utgw(GPU_MAXLEVS) ! zonal wind tendency
real :: vtgw(GPU_MAXLEVS) ! meridional wind tendency
real :: ttgw(pver) ! temperature tendency
real :: utgw(pver) ! zonal wind tendency
real :: vtgw(pver) ! meridional wind tendency

real :: ni(0:GPU_MAXLEVS) ! interface Brunt-Vaisalla frequency
real :: nm(GPU_MAXLEVS) ! midpoint Brunt-Vaisalla frequency
real :: ni(0:pver) ! interface Brunt-Vaisalla frequency
real :: nm(pver) ! midpoint Brunt-Vaisalla frequency
real :: rdpldv ! 1/dp across low level divergence region
real :: rhoi(0:GPU_MAXLEVS) ! interface density
real :: tau(-MAXPGWV:MAXPGWV,0:GPU_MAXLEVS) ! wave Reynolds stress
real :: rhoi(0:pver) ! interface density
real :: tau(-pgwv:pgwv,0:pver) ! wave Reynolds stress
real :: tau0x ! c=0 sfc. stress (zonal)
real :: tau0y ! c=0 sfc. stress (meridional)
real :: ti(0:GPU_MAXLEVS) ! interface temperature
real :: ubi(0:GPU_MAXLEVS) ! projection of wind at interfaces
real :: ubm(GPU_MAXLEVS) ! projection of wind at midpoints
real :: ti(0:pver) ! interface temperature
real :: ubi(0:pver) ! projection of wind at interfaces
real :: ubm(pver) ! projection of wind at midpoints
real :: xv ! unit vectors of source wind (x)
real :: yv ! unit vectors of source wind (y)

real :: utosrc(GPU_MAXLEVS)
real :: vtosrc(GPU_MAXLEVS)
real :: ttosrc(GPU_MAXLEVS)
real :: utosrc(pver)
real :: vtosrc(pver)
real :: ttosrc(pver)

real :: alpha(0:GPU_MAXLEVS) ! newtonian cooling coefficients
real :: dback(0:GPU_MAXLEVS) ! newtonian cooling coefficients
real :: c (-MAXPGWV:MAXPGWV) ! wave phase speeds
real :: alpha(0:pver) ! newtonian cooling coefficients
real :: dback(0:pver) ! newtonian cooling coefficients
real :: c (-pgwv:pgwv) ! wave phase speeds
real :: c4
real :: cw (-MAXPGWV:MAXPGWV) ! wave phase speeds
real :: cw4(-MAXPGWV:MAXPGWV) ! wave phase speeds
real :: cw (-pgwv:pgwv) ! wave phase speeds
real :: cw4(-pgwv:pgwv) ! wave phase speeds

!-----------------------------------------------------------------------------

Expand All @@ -296,13 +175,7 @@ subroutine gw_intr (pcols, pver, dt, pgwv,

cw = cw*(sum(cw4)/sum(cw))

#ifdef _CUDA
i = (blockidx%x - 1) * blockdim%x + threadidx%x

I_LOOP: if ( i <= pcols ) then
#else
I_LOOP: do i = 1, pcols
#endif

! Assign newtonian cooling coefficients
! -------------------------------------
Expand Down Expand Up @@ -433,20 +306,13 @@ subroutine gw_intr (pcols, pver, dt, pgwv,
taugwdx_dev(i) = tau0x
taugwdy_dev(i) = tau0y

#ifndef _CUDA
end do I_LOOP
rc = 0
#else
end if I_LOOP
#endif

return
end subroutine gw_intr

!================================================================================
#ifdef _CUDA
attributes(device) &
#endif
subroutine gw_prof (i, k, pcols, pver, u, v, t, pm, pi, rhoi, ni, ti, nm)
!-----------------------------------------------------------------------
! Compute profiles of background state quantities for the multiple
Expand Down Expand Up @@ -516,9 +382,6 @@ end subroutine gw_prof

!================================================================================

#ifdef _CUDA
attributes(device) &
#endif
subroutine gw_oro (i, pcols, pver, pgwv, &
u, v, t, sgh, pm, pi, dpm, zm, nm, &
kldv, kldvmn, ksrc, ksrcmn, rdpldv, &
Expand Down Expand Up @@ -709,9 +572,6 @@ subroutine gw_oro (i, pcols, pver, pgwv, &
end subroutine gw_oro

!===============================================================================
#ifdef _CUDA
attributes(device) &
#endif
subroutine gw_bgnd (i, pcols, pver, cw, &
u, v, t, pm, pi, dpm, rdpm, piln, rlat, &
kldv, kldvmn, ksrc, ksrcmn, rdpldv, tau, &
Expand Down Expand Up @@ -850,9 +710,6 @@ subroutine gw_bgnd (i, pcols, pver, cw, &
end subroutine gw_bgnd

!===============================================================================
#ifdef _CUDA
attributes(device) &
#endif
subroutine gw_drag_prof (i, pcols, pver, &
pgwv, ngwv, kbot, ktop, c, u, &
v, t, pi, dpm, rdpm, piln, &
Expand Down
2 changes: 1 addition & 1 deletion GEOSagcm_GridComp/GEOSphysics_GridComp/GEOSmoist_GridComp/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ esma_add_library (${this}
SRCS ${srcs}
DEPENDENCIES GEOS_Shared GMAO_mpeu MAPL Chem_Shared Chem_Base esmf)

get_target_property (extra_incs fms_r4 INCLUDE_DIRECTORIES)
get_target_property (extra_incs fms_r4 INTERFACE_INCLUDE_DIRECTORIES)
target_include_directories(${this} PRIVATE
$<BUILD_INTERFACE:${extra_incs}>
)
Expand Down
2 changes: 1 addition & 1 deletion ...ridComp/GEOSphysics_GridComp/GEOSsurface_GridComp/GEOSland_GridComp/GEOS_LandGridComp.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ subroutine SetServices ( GC, RC )

tmp = ''
if (NUM_LDAS_ENSEMBLE >1) then
!landxxxx
! land_exxxx
tmp(1:ens_id_width)=COMP_NAME(5:5+ens_id_width-1)
endif

Expand Down
17 changes: 15 additions & 2 deletions ...Comp/GEOSsurface_GridComp/GEOSland_GridComp/GEOScatchCN_GridComp/GEOS_CatchCNGridComp.F90
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ subroutine SetServices ( GC, RC )
character(len=ESMF_MAXSTR) :: SURFRC
type(ESMF_Config) :: SCF
integer :: DO_GOSWIM, LSM_CHOICE, ATM_CO2
character(len=ESMF_MAXSTR) :: tmp
integer :: NUM_LDAS_ENSEMBLE, ens_id_width

! Begin...
! --------
Expand All @@ -63,6 +65,17 @@ subroutine SetServices ( GC, RC )
call MAPL_GetObjectFromGC(gc, MAPL, rc=status)
VERIFY_(status)

call MAPL_GetResource ( MAPL, NUM_LDAS_ENSEMBLE, Label="NUM_LDAS_ENSEMBLE:", DEFAULT=1, RC=STATUS)
VERIFY_(STATUS)
call MAPL_GetResource ( MAPL, ens_id_width, Label="ENS_ID_WIDTH:", DEFAULT=0, RC=STATUS)
VERIFY_(STATUS)

tmp = ''
if (NUM_LDAS_ENSEMBLE >1) then
!catchcn_exxxx
tmp(1:ens_id_width)=COMP_NAME(8:8+ens_id_width-1)
endif

call MAPL_GetResource ( MAPL, LSM_CHOICE, Label="LSM_CHOICE:", DEFAULT=2, RC=STATUS)
VERIFY_(STATUS)
call MAPL_GetResource ( MAPL, SURFRC, label = 'SURFRC:', default = 'GEOS_SurfaceGridComp.rc', RC=STATUS) ; VERIFY_(STATUS)
Expand All @@ -72,10 +85,10 @@ subroutine SetServices ( GC, RC )
call ESMF_ConfigGetAttribute (SCF, label='N_CONST_LAND4SNWALB:' , value=DO_GOSWIM , DEFAULT=0, RC=STATUS); VERIFY_(STATUS)

if ( LSM_CHOICE == 2 ) then
CATCHCN = MAPL_AddChild('CATCHCNCLM40', 'setservices_', parentGC=GC, sharedObj='libGEOScatchCNCLM40_GridComp.so', RC=STATUS)
CATCHCN = MAPL_AddChild('CATCHCNCLM40'//trim(tmp), 'setservices_', parentGC=GC, sharedObj='libGEOScatchCNCLM40_GridComp.so', RC=STATUS)
VERIFY_(STATUS)
else if ( LSM_CHOICE == 3 ) then
CATCHCN = MAPL_AddChild('CATCHCNCLM45', 'setservices_', parentGC=GC, sharedObj='libGEOScatchCNCLM45_GridComp.so', RC=STATUS)
CATCHCN = MAPL_AddChild('CATCHCNCLM45'//trim(tmp), 'setservices_', parentGC=GC, sharedObj='libGEOScatchCNCLM45_GridComp.so', RC=STATUS)
VERIFY_(STATUS)
else
_ASSERT( .false., " LSM_CHOICE should equal 2 (CLM40) or 3 (CLM45)")
Expand Down
Loading

0 comments on commit 4961a99

Please sign in to comment.