Skip to content

Remove the decomposition-dependent reprosum "recompute" fallback - fixes Nag subcol test #1702

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jimmielin wants to merge 2 commits into from
Closed

Remove the decomposition-dependent reprosum "recompute" fallback - fixes Nag subcol test #1702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e74431f
Remove the decomposition-dependent reprosum "recompute" fallbacks
hplin-ucar Oct 4, 2026
8c4bb63
Remove all reprosum_recompute=.true. as they are no-ops now
hplin-ucar Oct 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/cam4_port/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/cam4_port5d/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/cam5_port_f45/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/cam5_port_ne30/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/cam6_port_f09/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_dust/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_meteor_impact/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_meteor_smoke/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_mixed_sulfate/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_pmc/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_sea_salt/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_sulfate/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_test_growth/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_test_passive/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_test_radiative/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_test_swelling/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_test_tracers/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/carma_test_tracers2/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/co2rmp/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/dartcambigens/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/fire_emis/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/nudging/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq1d_amie/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq1d_newyear/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq24h_port/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq3s_aqw/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq3s_nudging_f10_L26/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq3s_nudging_ne5_L26/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq3s_sathist/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq3s_subcol/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq3s_zmean_nudging/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq9s_clubbmf/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq9s_kessler/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/outfrq9s_ocnemis/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/reduced_hist5d/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/scm_prep/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,3 @@ orb_obliq = 1.e36
orb_mvelp = 1.e36
orb_mode = "fixed_year"
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/scm_prep_c6/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,3 @@ orb_obliq = 1.e36
orb_mvelp = 1.e36
orb_mode = "fixed_year"
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/scmarm/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/ttrac_loadbal0/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/ttrac_loadbal1/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/ttrac_loadbal3/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
1 change: 0 additions & 1 deletion cime_config/testdefs/testmods_dirs/cam/waccmx_weimer/user_nl_cpl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,2 +1 @@
reprosum_diffmax=1.0e-14
reprosum_recompute=.true.
76 changes: 13 additions & 63 deletions src/dynamics/fv/d2a3dikj.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,11 @@ subroutine d2a3dikj(grid, am_geom_crrct, u, v, ua, va)
use dynamics_vars, only : t_fvdycore_grid

#if defined( SPMD )
use parutilitiesmodule, only : parcollective, sumop
use mod_comm, only: mp_send3d, mp_recv3d
#endif

use shr_reprosum_mod, only : shr_reprosum_calc, &
shr_reprosum_tolExceeded, &
shr_reprosum_reldiffmax, &
shr_reprosum_recompute
shr_reprosum_tolExceeded
use cam_logfile, only : iulog
use perf_mod

Expand Down Expand Up @@ -67,22 +64,21 @@ subroutine d2a3dikj(grid, am_geom_crrct, u, v, ua, va)
!EOP
!-----------------------------------------------------------------------
!BOC
real(r8), parameter :: D0_0 = 0.0_r8
real(r8), parameter :: D0_5 = 0.5_r8

integer :: imh, i, j, k, m, itot, jtot, ltot, ik
integer :: imh, i, j, k, itot, jtot, ltot, ik
real(r8) :: veast(grid%jfirstxy:grid%jlastxy,grid%km)
real(r8) :: unorth(grid%ifirstxy:grid%ilastxy,grid%km)

real(r8) :: uva(grid%ifirstxy:grid%ilastxy,grid%km,2)
real(r8) :: uvn(grid%km,2), uvs(grid%km,2)
real(r8) :: rel_diff(2,grid%km,2)
real(r8),allocatable :: uva_tmp(:)

integer :: ifirstxy, ilastxy, jfirstxy, jlastxy, im, jm, km
integer :: myidxy_y, myidxy_x, nprxy_x, iam

logical :: write_warning
logical :: tol_exceeded

real(r8), pointer :: coslon(:),sinlon(:) ! Sine and cosine in longitude

Expand Down Expand Up @@ -230,36 +226,13 @@ subroutine d2a3dikj(grid, am_geom_crrct, u, v, ua, va)
commid=grid%commxy_x, rel_diff=rel_diff)
call t_stopf("d2a3dikj_reprosum")

! check that "fast" reproducible sum is accurate enough. If not, calculate
! using old method
! Warn if the nonreproducible floating point check sum differs from the
! integer vector sum by more than reprosum_diffmax. The integer vector sum is
! exact, so the result is kept regardless.
write_warning = .false.
if (myidxy_x == 0) write_warning = .true.
if ( shr_reprosum_tolExceeded('d2a3dikj/South Pole', 2*km, write_warning, &
iulog, rel_diff) ) then
if ( shr_reprosum_recompute ) then
call t_startf("d2a3dikj_sumfix")
allocate( uva_tmp(im) )
do m = 1,2
do k = 1,km
if (rel_diff(1,k,m) > shr_reprosum_reldiffmax) then
uva_tmp(:) = D0_0
do i = ifirstxy,ilastxy
uva_tmp(i) = uva(i,k,m)
enddo
#if defined(SPMD)
call parcollective(grid%commxy_x,sumop,im,uva_tmp)
#endif
uvs(k,m) = D0_0
do i = 1,im
uvs(k,m) = uvs(k,m) + uva_tmp(i)
enddo
endif
enddo
enddo
deallocate( uva_tmp )
call t_stopf("d2a3dikj_sumfix")
endif
endif
tol_exceeded = shr_reprosum_tolExceeded('d2a3dikj/South Pole', 2*km, write_warning, &
iulog, rel_diff)

!$omp parallel do private(i,k)
do k = 1,km
Expand Down Expand Up @@ -303,36 +276,13 @@ subroutine d2a3dikj(grid, am_geom_crrct, u, v, ua, va)
commid=grid%commxy_x, rel_diff=rel_diff)
call t_stopf("d2a3dikj_reprosum")

! check that "fast" reproducible sum is accurate enough. If not, calculate
! using old method
! Warn if the nonreproducible floating point check sum differs from the
! integer vector sum by more than reprosum_diffmax. The integer vector sum is
! exact, so the result is kept regardless.
write_warning = .false.
if (myidxy_x == 0) write_warning = .true.
if ( shr_reprosum_tolExceeded('d2a3dikj/Nouth Pole', 2*km, write_warning, &
iulog, rel_diff) ) then
if ( shr_reprosum_recompute ) then
call t_startf("d2a3dikj_sumfix")
allocate( uva_tmp(im) )
do m = 1,2
do k = 1,km
if (rel_diff(1,k,m) > shr_reprosum_reldiffmax) then
uva_tmp(:) = D0_0
do i = ifirstxy,ilastxy
uva_tmp(i) = uva(i,k,m)
enddo
#if defined(SPMD)
call parcollective(grid%commxy_x,sumop,im,uva_tmp)
#endif
uvn(k,m) = D0_0
do i = 1,im
uvn(k,m) = uvn(k,m) + uva_tmp(i)
enddo
endif
enddo
enddo
deallocate( uva_tmp )
call t_stopf("d2a3dikj_sumfix")
endif
endif
tol_exceeded = shr_reprosum_tolExceeded('d2a3dikj/North Pole', 2*km, write_warning, &
iulog, rel_diff)

!$omp parallel do private(i,k)
do k = 1,km
Expand Down
46 changes: 8 additions & 38 deletions src/dynamics/fv/mean_module.F90
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
module mean_module

use shr_kind_mod, only : r8 => shr_kind_r8
use shr_reprosum_mod, only : shr_reprosum_calc, shr_reprosum_tolExceeded, &
shr_reprosum_recompute
use shr_reprosum_mod, only : shr_reprosum_calc, shr_reprosum_tolExceeded
use perf_mod
use cam_logfile, only : iulog

Expand Down Expand Up @@ -97,10 +96,6 @@ subroutine gmeanxy(grid, q, qmean)
use commap, only : w
use dynamics_vars, only : T_FVDYCORE_GRID

#if defined( SPMD )
use parutilitiesmodule, only : parcollective, sumop
#endif

implicit none

! !INPUT PARAMETERS:
Expand Down Expand Up @@ -134,13 +129,13 @@ subroutine gmeanxy(grid, q, qmean)

real(r8) :: q_tmp(grid%ifirstxy:grid%ilastxy, &
grid%jfirstxy:grid%jlastxy)
real(r8) :: rel_diff(2), qmean_tmp(1), xsum
real(r8), allocatable :: q_global(:,:)
real(r8) :: rel_diff(2), qmean_tmp(1)

integer :: i, j, im, jm, ifirstxy, ilastxy, jfirstxy, jlastxy
integer :: lim, ljm, lijm

logical :: write_warning
logical :: tol_exceeded

im = grid%im
jm = grid%jm
Expand All @@ -165,38 +160,13 @@ subroutine gmeanxy(grid, q, qmean)
qmean = qmean_tmp(1)
call t_stopf("gmeanxy_reprosum")

! check that "fast" reproducible sum is accurate enough. If not, calculate
! using old method
! Warn if the nonreproducible floating point check sum differs from the
! integer vector sum by more than reprosum_diffmax. The integer vector sum is
! exact, so the result is kept regardless.
write_warning = .false.
if (grid%iam == 0) write_warning = .true.
if ( shr_reprosum_tolExceeded('gmeanxy', 1, write_warning, &
iulog, rel_diff) ) then
if ( shr_reprosum_recompute ) then
call t_startf("gmeanxy_sumfix")
allocate( q_global(im,jm) )
q_global = D0_0
do j=jfirstxy,jlastxy
do i=ifirstxy,ilastxy
q_global(i,j) = q_tmp(i,j)
enddo
enddo

#if defined( SPMD )
call parcollective( grid%commxy, sumop, im, jm, q_global )
#endif
qmean = D0_0
do j=1,jm
xsum = D0_0
do i=1,im
xsum = xsum + q_global(i,j)
enddo
qmean = qmean + xsum
enddo

deallocate( q_global )
call t_stopf("gmeanxy_sumfix")
endif
endif
tol_exceeded = shr_reprosum_tolExceeded('gmeanxy', 1, write_warning, &
iulog, rel_diff)

qmean = qmean / (2*im)

Expand Down
Loading
Loading