Reduce CAM initialization memory footprint that is important for ne1024 simulation- #4102

bld/build-namelist

@@ -705,6 +705,7 @@ if (!($simple_phys or $aqua_mode)) {
   my $chem_nitrodep = chem_has_species($cfg, 'NO') and chem_has_species($cfg, 'NH3');
   if ((!$chem_nitrodep) or ($chem =~ /geoschem/)) {
     add_default($nl, 'stream_ndep_mesh_filename');
+    add_default($nl, 'stream_ndep_mapalgo');
     add_default($nl, 'stream_ndep_data_filename', 'sim_year'=>$sim_year);
     add_default($nl, 'stream_ndep_year_first', 'val'=>$sim_year_first);
     add_default($nl, 'stream_ndep_year_last',  'val'=>$sim_year_last);

bld/namelist_files/namelist_defaults_cam.xml

@@ -2117,6 +2117,7 @@
 
 <!-- nitrogen deposition to surface models -->
 <stream_ndep_mesh_filename>share/meshes/fv0.9x1.25_141008_polemod_ESMFmesh.nc</stream_ndep_mesh_filename>
+<stream_ndep_mapalgo>bilinear</stream_ndep_mapalgo>
 
 <stream_ndep_data_filename >lnd/clm2/ndepdata/fndep_clm_hist_b.e21.BWHIST.f09_g17.CMIP6-historical-WACCM.ensmean_1849-2015_monthly_0.9x1.25_c180926.nc</stream_ndep_data_filename>
 <stream_ndep_data_filename sim_year="1850">atm/cam/chem/trop_strat_mam5_ts4_aero_spinup/ndep/fndep_clm_B1850C_MT4s.1850monthly.ne30_c251220.nc</stream_ndep_data_filename>

bld/namelist_files/namelist_definition.xml

@@ -8248,6 +8248,15 @@ Nitrogen deposition stream data filename.
 Grid mesh file corresponding to stream_ndep_data_filename.
 </entry>
 
+<entry id="stream_ndep_mapalgo" type="char*40" category="Nitrogen Deposition"
+       group="ndep_stream_nl" valid_values="bilinear,nn,consf,consd,redist,none" >
+Spatial mapping (regrid) method used to map the nitrogen deposition stream data from
+the stream grid to the model grid.  Default 'bilinear'.  Use 'redist' (pure index-based
+redistribution -- no interpolation weights or ESMF route handle) when the ndep data file
+is already on the model grid; this greatly reduces memory and setup cost at high
+resolution (e.g. ne1024pg2).
+</entry>
+
 <entry id="megan_factors_file" type="char*256" input_pathname="abs" category="VOC_emissions"
        group="megan_emis_nl" valid_values="" >
 File containing MEGAN emissions factors.

src/chemistry/aerosol/soil_erod_mod.F90

@@ -24,26 +24,36 @@ subroutine soil_erod_init( dust_emis_fact, soil_erod_file )
     use interpolate_data, only: lininterp_init, lininterp, lininterp_finish, interp_type
     use ppgrid,           only: begchunk, endchunk, pcols
     use mo_constants,     only: pi, d2r
-    use pio,              only: file_desc_t,pio_inq_dimid,pio_inq_dimlen,pio_get_var,pio_inq_varid, PIO_NOWRITE
     use phys_grid,        only: get_ncols_p, get_rlat_all_p, get_rlon_all_p
-    use cam_pio_utils,    only: cam_pio_openfile
     use ioFileMod,        only: getfil
+    use netcdf,           only: nf90_open, nf90_nowrite, nf90_close, nf90_noerr,  &
+                                nf90_inq_dimid, nf90_inquire_dimension,           &
+                                nf90_inq_varid, nf90_get_var
+    use cam_shmem_mod,    only: cam_shmem_alloc_r8_2d, cam_shmem_fence,           &
+                                cam_shmem_free, cam_shmem_is_leader,              &
+                                cam_shmem_leader_comm, cam_shmem_npes_per_node
+#ifdef SPMD
+    use mpishorthand,     only: mpicom, mpiint, mpir8
+#endif
 
     real(r8),         intent(in) :: dust_emis_fact
     character(len=*), intent(in) :: soil_erod_file
 
-    real(r8), allocatable ::  soil_erodibility_in(:,:)  ! temporary input array
+    real(r8), pointer     ::  soil_erodibility_in(:,:) => null()  ! node-shared input field
     real(r8), allocatable :: dst_lons(:)
     real(r8), allocatable :: dst_lats(:)
     character(len=cl)     :: infile
     integer :: did, vid, nlat, nlon
-    type(file_desc_t) :: ncid
+    integer :: ncid, iret
+    integer :: win_serod        ! per-node shared-memory window for soil_erodibility_in
 
     type(interp_type) :: lon_wgts, lat_wgts
     real(r8) :: to_lats(pcols), to_lons(pcols)
     integer :: c, ncols, ierr
     real(r8), parameter :: zero=0._r8, twopi=2._r8*pi
 
+    win_serod = -1
+
     soil_erod_fact = dust_emis_fact
 
     ! Summary to log file
@@ -57,27 +67,56 @@ subroutine soil_erod_init( dust_emis_fact, soil_erod_file )
 
     ! Get file name.  
     call getfil(soil_erod_file, infile, 0)
-    call cam_pio_openfile (ncid, trim(infile), PIO_NOWRITE)
 
-    ! Get input data resolution.
-    ierr = pio_inq_dimid( ncid, 'lon', did )
-    ierr = pio_inq_dimlen( ncid, did, nlon )
+    ! Read the source-grid soil erodibility on masterproc only (serial NetCDF) and
+    ! hold it in per-node MPI-3 shared memory: one physical copy per node instead
+    ! of one per MPI rank.  At high resolution (e.g. ne1024) this removes
+    ! (ranks_per_node - 1) redundant nlon*nlat copies of the input field.
+    if (masterproc) then
+       iret = nf90_open(trim(infile), NF90_NOWRITE, ncid)
+       if (iret /= nf90_noerr) call endrun('soil_erod_init: failed to open '//trim(infile))
+       iret = nf90_inq_dimid( ncid, 'lon', did )
+       iret = nf90_inquire_dimension( ncid, did, len=nlon )
+       iret = nf90_inq_dimid( ncid, 'lat', did )
+       iret = nf90_inquire_dimension( ncid, did, len=nlat )
+    end if
 
-    ierr = pio_inq_dimid( ncid, 'lat', did )
-    ierr = pio_inq_dimlen( ncid, did, nlat )
+#ifdef SPMD
+    call mpibcast( nlon, 1, mpiint, 0, mpicom )
+    call mpibcast( nlat, 1, mpiint, 0, mpicom )
+#endif
 
     allocate(dst_lons(nlon))
     allocate(dst_lats(nlat))
-    allocate(soil_erodibility_in(nlon,nlat))
+    call cam_shmem_alloc_r8_2d( soil_erodibility_in, win_serod, nlon, nlat )
+    if (masterproc) then
+       write(iulog,*) 'soil_erod_mod: soil_erodibility_in held in per-node shared memory; ', &
+            cam_shmem_npes_per_node()-1, ' redundant copies/node avoided'
+    end if
 
-    ierr = pio_inq_varid( ncid, 'lon', vid )
-    ierr = pio_get_var( ncid, vid, dst_lons  )
+    ! Open the window epoch, fill on the node leader (masterproc), then publish.
+    call cam_shmem_fence( win_serod )
 
-    ierr = pio_inq_varid( ncid, 'lat', vid )
-    ierr = pio_get_var( ncid, vid, dst_lats  )
+    if (masterproc) then
+       iret = nf90_inq_varid( ncid, 'lon', vid )
+       iret = nf90_get_var( ncid, vid, dst_lons )
+       iret = nf90_inq_varid( ncid, 'lat', vid )
+       iret = nf90_get_var( ncid, vid, dst_lats )
+       iret = nf90_inq_varid( ncid, 'mbl_bsn_fct_geo', vid )
+       iret = nf90_get_var( ncid, vid, soil_erodibility_in )
+       iret = nf90_close( ncid )
+    end if
 
-    ierr = pio_inq_varid( ncid, 'mbl_bsn_fct_geo', vid )
-    ierr = pio_get_var( ncid, vid, soil_erodibility_in )
+#ifdef SPMD
+    ! Source coordinates to every rank; the large field to every node leader.  The
+    ! closing fence then publishes it to every rank on each node.
+    call mpibcast( dst_lons, nlon, mpir8, 0, mpicom )
+    call mpibcast( dst_lats, nlat, mpir8, 0, mpicom )
+    if (cam_shmem_is_leader()) then
+       call mpibcast( soil_erodibility_in, nlon*nlat, mpir8, 0, cam_shmem_leader_comm() )
+    end if
+#endif
+    call cam_shmem_fence( win_serod )
 
     !-----------------------------------------------------------------------
     !     	... convert to radians and setup regridding
@@ -107,11 +146,8 @@ subroutine soil_erod_init( dust_emis_fact, soil_erod_file )
        call lininterp_finish(lat_wgts)
        call lininterp_finish(lon_wgts)
     end do
-    deallocate( soil_erodibility_in, stat=ierr )
-    if( ierr /= 0 ) then
-       write(iulog,*) 'soil_erod_init: failed to deallocate soil_erodibility_in, ierr = ',ierr
-       call endrun('soil_erod_init: failed to deallocate soil_erodibility_in')
-    end if
+    ! Release the node-shared input field (collective over the node communicator).
+    call cam_shmem_free( soil_erodibility_in, win_serod )
 
     deallocate( dst_lats )
     deallocate( dst_lons )

src/chemistry/mozart/chemistry.F90

@@ -1371,6 +1371,7 @@ subroutine chem_final()
     use species_sums_diags, only: species_sums_final
     use mo_tuvx, only: tuvx_finalize, tuvx_active
     use short_lived_species, only: short_lived_species_final
+    use mo_photo, only: photo_final
 
     call mee_ion_final()
     call rate_diags_final()
@@ -1379,6 +1380,7 @@ subroutine chem_final()
        call tuvx_finalize()
     end if
     call short_lived_species_final()
+    call photo_final()
 
   end subroutine chem_final