From 4a4f0102d59c5683ff51c5825a60901d23788914 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 16 Jul 2026 13:44:05 -0500 Subject: [PATCH 1/2] perf: grind uses minimum wall-clock per-stage time, not running-mean cpu_time --- src/simulation/m_rhs.fpp | 15 +-------------- src/simulation/m_time_steppers.fpp | 20 +++++++++++++++++++- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/simulation/m_rhs.fpp b/src/simulation/m_rhs.fpp index 0ab59a8454..369aa765e6 100644 --- a/src/simulation/m_rhs.fpp +++ b/src/simulation/m_rhs.fpp @@ -463,8 +463,7 @@ contains end subroutine s_initialize_rhs_module !> Compute the right-hand side of the semi-discrete governing equations for a single time stage - impure subroutine s_compute_rhs(q_cons_vf, q_T_sf, q_prim_vf, bc_type, rhs_vf, pb_in, rhs_pb, mv_in, rhs_mv, t_step, & - & time_avg, stage) + impure subroutine s_compute_rhs(q_cons_vf, q_T_sf, q_prim_vf, bc_type, rhs_vf, pb_in, rhs_pb, mv_in, rhs_mv, t_step, stage) type(scalar_field), dimension(sys_size), intent(inout) :: q_cons_vf type(scalar_field), intent(inout) :: q_T_sf @@ -478,9 +477,7 @@ contains real(stp), dimension(idwbuff(1)%beg:,idwbuff(2)%beg:,idwbuff(3)%beg:,1:,1:), intent(inout) :: mv_in real(wp), dimension(idwbuff(1)%beg:,idwbuff(2)%beg:,idwbuff(3)%beg:,1:,1:), intent(inout) :: rhs_mv integer, intent(in) :: t_step - real(wp), intent(inout) :: time_avg integer, intent(in) :: stage - real(wp) :: t_start, t_finish integer :: id integer(kind=8) :: i, j, k, l, q !< Generic loop iterators @@ -488,8 +485,6 @@ contains call nvtxStartRange("COMPUTE-RHS") - call cpu_time(t_start) - if (.not. igr) then ! Association/Population of Working Variables $:GPU_PARALLEL_LOOP(private='[i, j, k, l]', collapse=4) @@ -849,14 +844,6 @@ contains end if end if - call cpu_time(t_finish) - - if (t_step >= 2) then - time_avg = (abs(t_finish - t_start) + (t_step - 2)*time_avg)/(t_step - 1) - else - time_avg = 0._wp - end if - call nvtxEndRange end subroutine s_compute_rhs diff --git a/src/simulation/m_time_steppers.fpp b/src/simulation/m_time_steppers.fpp index 66b0951525..b4afd8b9aa 100644 --- a/src/simulation/m_time_steppers.fpp +++ b/src/simulation/m_time_steppers.fpp @@ -459,6 +459,9 @@ contains integer, intent(in) :: nstage integer :: i, j, k, l, q, s !< Generic loop iterator real(wp) :: start, finish + integer(kind=8) :: stage_t0, stage_t1, clock_rate + real(wp) :: stage_time + integer, parameter :: n_warmup = 2 !< stages excluded before the timing floor (warmup/JIT/first-touch) call cpu_time(start) call nvtxStartRange("TIMESTEP") @@ -467,8 +470,9 @@ contains if (adap_dt) call s_adaptive_dt_bubble(1) do s = 1, nstage + call system_clock(stage_t0) call s_compute_rhs(q_cons_ts(1)%vf, q_T_sf, q_prim_vf, bc_type, rhs_vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, & - & t_step, time_avg, s) + & t_step, s) if (s == 1) then if (run_time_info) then @@ -566,6 +570,20 @@ contains call s_ibm_correct_state(q_cons_ts(1)%vf, q_prim_vf) end if end if + + ! Grind: minimum wall-clock time of a full RK stage (compute + halo H2D/D2H + + ! update + IBM correction, aside from I/O) over steady-state stages. Wall clock + ! (not cpu_time, which counts MPI spin-wait) and the minimum (jitter only adds + ! time) make it reproducible; the min over stages drops the I/O-bearing s==1 stage. + call system_clock(stage_t1, clock_rate) + stage_time = real(stage_t1 - stage_t0, wp)/real(clock_rate, wp) + if (t_step - t_step_start < n_warmup) then + time_avg = 0._wp + else if (time_avg <= 0._wp) then + time_avg = stage_time + else + time_avg = min(time_avg, stage_time) + end if end do if (ib) then From f3a068d4d6933dddbcf06c9ecbcc40e0899cfa01 Mon Sep 17 00:00:00 2001 From: Spencer Bryngelson Date: Thu, 16 Jul 2026 15:03:30 -0500 Subject: [PATCH 2/2] perf: correct system_clock wrap-around in grind timer; fix n_warmup comment --- src/simulation/m_time_steppers.fpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/simulation/m_time_steppers.fpp b/src/simulation/m_time_steppers.fpp index b4afd8b9aa..076df05563 100644 --- a/src/simulation/m_time_steppers.fpp +++ b/src/simulation/m_time_steppers.fpp @@ -459,9 +459,9 @@ contains integer, intent(in) :: nstage integer :: i, j, k, l, q, s !< Generic loop iterator real(wp) :: start, finish - integer(kind=8) :: stage_t0, stage_t1, clock_rate + integer(kind=8) :: stage_t0, stage_t1, clock_rate, clock_max real(wp) :: stage_time - integer, parameter :: n_warmup = 2 !< stages excluded before the timing floor (warmup/JIT/first-touch) + integer, parameter :: n_warmup = 2 !< time steps excluded before the timing floor (warmup/JIT/first-touch) call cpu_time(start) call nvtxStartRange("TIMESTEP") @@ -575,8 +575,13 @@ contains ! update + IBM correction, aside from I/O) over steady-state stages. Wall clock ! (not cpu_time, which counts MPI spin-wait) and the minimum (jitter only adds ! time) make it reproducible; the min over stages drops the I/O-bearing s==1 stage. - call system_clock(stage_t1, clock_rate) - stage_time = real(stage_t1 - stage_t0, wp)/real(clock_rate, wp) + call system_clock(stage_t1, clock_rate, clock_max) + ! Correct the delta for the rare case of the clock counter wrapping past clock_max. + if (stage_t1 >= stage_t0) then + stage_time = real(stage_t1 - stage_t0, wp)/real(clock_rate, wp) + else + stage_time = real(stage_t1 - stage_t0 + clock_max + 1_8, wp)/real(clock_rate, wp) + end if if (t_step - t_step_start < n_warmup) then time_avg = 0._wp else if (time_avg <= 0._wp) then