REMORA Class Reference

#include <REMORA.H>

Inheritance diagram for REMORA:

Collaboration diagram for REMORA:

Public Member Functions
	REMORA ()
virtual	~REMORA ()
void	Evolve ()
virtual void	ErrorEst (int lev, amrex::TagBoxArray &tags, amrex::Real time, int ngrow) override
void	InitData ()
void	init_only (int lev, amrex::Real time)
void	restart ()
void	post_timestep (int nstep, amrex::Real time, amrex::Real dt_lev)
void	sum_integrated_quantities (amrex::Real time)
amrex::Real	volWgtSumMF (int lev, const amrex::MultiFab &mf, int comp, bool local, bool finemask)
bool	is_it_time_for_action (int nstep, amrex::Real time, amrex::Real dt, int action_interval, amrex::Real action_per)
virtual void	MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
virtual void	RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
virtual void	ClearLevel (int lev) override
virtual void	MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
amrex::Real	estTimeStep (int lev) const
void	remora_advance (int level, amrex::MultiFab &cons_old, amrex::MultiFab &cons_new, amrex::MultiFab &xvel_old, amrex::MultiFab &yvel_old, amrex::MultiFab &zvel_old, amrex::MultiFab &xvel_new, amrex::MultiFab &yvel_new, amrex::MultiFab &zvel_new, amrex::MultiFab &source, const amrex::Geometry fine_geom, const amrex::Real dt, const amrex::Real time)
amrex::MultiFab &	build_fine_mask (int lev)
void	WritePlotFile (int which, amrex::Vector< std::string > plot_var_names)
void	WriteMultiLevelPlotfileWithBathymetry (const std::string &plotfilename, int nlevels, const amrex::Vector< const amrex::MultiFab * > &mf, const amrex::Vector< const amrex::MultiFab * > &mf_nd, const amrex::Vector< std::string > &varnames, amrex::Real time, const amrex::Vector< int > &level_steps, const std::string &versionName="HyperCLaw-V1.1", const std::string &levelPrefix="Level_", const std::string &mfPrefix="Cell", const amrex::Vector< std::string > &extra_dirs=amrex::Vector< std::string >()) const
void	WriteGenericPlotfileHeaderWithBathymetry (std::ostream &HeaderFile, int nlevels, const amrex::Vector< amrex::BoxArray > &bArray, const amrex::Vector< std::string > &varnames, amrex::Real time, const amrex::Vector< int > &level_steps, const std::string &versionName, const std::string &levelPrefix, const std::string &mfPrefix) const
void	Advance (int lev, amrex::Real time, amrex::Real dt_lev, int iteration, int ncycle)
void	setup_step (int lev, amrex::Real time, amrex::Real dt_lev)
void	advance_3d_ml (int lev, amrex::Real dt_lev)
void	advance_2d_onestep (int lev, amrex::Real dt_lev, amrex::Real dtfast_lev, int my_iif, int nfast_counter)
void	advance_2d (int lev, amrex::MultiFab const *mf_rhoS, amrex::MultiFab const *mf_rhoA, amrex::MultiFab *mf_ru, amrex::MultiFab *mf_rv, amrex::MultiFab *mf_rufrc, amrex::MultiFab *mf_rvfrc, amrex::MultiFab *mf_Zt_avg1, std::unique_ptr< amrex::MultiFab > &mf_DU_avg1, std::unique_ptr< amrex::MultiFab > &mf_DU_avg2, std::unique_ptr< amrex::MultiFab > &mf_DV_avg1, std::unique_ptr< amrex::MultiFab > &mf_DV_avg2, std::unique_ptr< amrex::MultiFab > &mf_rubar, std::unique_ptr< amrex::MultiFab > &mf_rvbar, std::unique_ptr< amrex::MultiFab > &mf_rzeta, std::unique_ptr< amrex::MultiFab > &mf_ubar, std::unique_ptr< amrex::MultiFab > &mf_vbar, amrex::MultiFab *mf_zeta, amrex::MultiFab const *mf_h, amrex::MultiFab const *mf_pm, amrex::MultiFab const *mf_pn, amrex::MultiFab const *mf_fcor, amrex::MultiFab const *mf_visc2_p, amrex::MultiFab const *mf_visc2_r, const amrex::Real dtfast_lev, const bool predictor_2d_step, const bool first_2d_step, int my_iif, int &next_indx1)
void	advance_3d (int lev, amrex::MultiFab &mf_cons, amrex::MultiFab &mf_u, amrex::MultiFab &mf_v, amrex::MultiFab *mf_sstore, amrex::MultiFab *mf_ru, amrex::MultiFab *mf_rv, std::unique_ptr< amrex::MultiFab > &mf_DU_avg1, std::unique_ptr< amrex::MultiFab > &mf_DU_avg2, std::unique_ptr< amrex::MultiFab > &mf_DV_avg1, std::unique_ptr< amrex::MultiFab > &mf_DV_avg2, std::unique_ptr< amrex::MultiFab > &mf_ubar, std::unique_ptr< amrex::MultiFab > &mf_vbar, std::unique_ptr< amrex::MultiFab > &mf_Akv, std::unique_ptr< amrex::MultiFab > &mf_Akt, std::unique_ptr< amrex::MultiFab > &mf_Hz, std::unique_ptr< amrex::MultiFab > &mf_Huon, std::unique_ptr< amrex::MultiFab > &mf_Hvom, std::unique_ptr< amrex::MultiFab > &mf_z_w, amrex::MultiFab const *mf_h, amrex::MultiFab const *mf_pm, amrex::MultiFab const *mf_pn, const int N, const amrex::Real dt_lev)
void	prestep (int lev, amrex::MultiFab &mf_uold, amrex::MultiFab &mf_vold, amrex::MultiFab &mf_u, amrex::MultiFab &mf_v, amrex::MultiFab *mf_ru, amrex::MultiFab *mf_rv, amrex::MultiFab &S_old, amrex::MultiFab &S_new, amrex::MultiFab &mf_W, amrex::MultiFab &mf_DC, const amrex::MultiFab *mf_z_r, const amrex::MultiFab *mf_z_w, const amrex::MultiFab *mf_h, const amrex::MultiFab *mf_pm, const amrex::MultiFab *mf_pn, const amrex::MultiFab *mf_sustr, const amrex::MultiFab *mf_svstr, const amrex::MultiFab *mf_bustr, const amrex::MultiFab *mf_bvstr, const int iic, const int nfirst, const int nnew, int nstp, int nrhs, int N, const amrex::Real dt_lev)
void	prestep_t_advection (const amrex::Box &tbx, const amrex::Box &gbx, const amrex::Array4< amrex::Real > &tempold, const amrex::Array4< amrex::Real > &tempcache, const amrex::Array4< amrex::Real > &Hz, const amrex::Array4< amrex::Real > &Huon, const amrex::Array4< amrex::Real > &Hvom, const amrex::Array4< amrex::Real > &W, const amrex::Array4< amrex::Real > &DC, const amrex::Array4< amrex::Real > &FC, const amrex::Array4< amrex::Real > &sstore, const amrex::Array4< amrex::Real const > &z_w, const amrex::Array4< amrex::Real const > &h, const amrex::Array4< amrex::Real const > &pm, const amrex::Array4< amrex::Real const > &pn, int iic, int ntfirst, int nrhs, int N, const amrex::Real dt_lev)
void	rhs_t_3d (const amrex::Box &bx, const amrex::Box &gbx, const amrex::Array4< amrex::Real > &t, const amrex::Array4< amrex::Real const > &tempstore, const amrex::Array4< amrex::Real const > &Huon, const amrex::Array4< amrex::Real const > &Hvom, const amrex::Array4< amrex::Real const > &Hz, const amrex::Array4< amrex::Real const > &pn, const amrex::Array4< amrex::Real const > &pm, const amrex::Array4< amrex::Real const > &W, const amrex::Array4< amrex::Real > &FC, int nrhs, int nnew, int N, const amrex::Real dt_lev)
void	rhs_uv_3d (const amrex::Box &xbx, const amrex::Box &ybx, const amrex::Array4< amrex::Real const > &uold, const amrex::Array4< amrex::Real const > &vold, const amrex::Array4< amrex::Real > &ru, const amrex::Array4< amrex::Real > &rv, const amrex::Array4< amrex::Real > &rufrc, const amrex::Array4< amrex::Real > &rvfrc, const amrex::Array4< amrex::Real const > &sustr, const amrex::Array4< amrex::Real const > &svstr, const amrex::Array4< amrex::Real const > &bustr, const amrex::Array4< amrex::Real const > &bvstr, const amrex::Array4< amrex::Real const > &Huon, const amrex::Array4< amrex::Real const > &Hvom, const amrex::Array4< amrex::Real const > &pm, const amrex::Array4< amrex::Real const > &pn, const amrex::Array4< amrex::Real const > &W, const amrex::Array4< amrex::Real > &FC, int nrhs, int N)
void	rhs_uv_2d (const amrex::Box &xbx, const amrex::Box &ybx, const amrex::Array4< amrex::Real const > &uold, const amrex::Array4< amrex::Real const > &vold, const amrex::Array4< amrex::Real > &ru, const amrex::Array4< amrex::Real > &rv, const amrex::Array4< amrex::Real const > &Duon, const amrex::Array4< amrex::Real const > &Dvom, const int nrhs)
void	rho_eos (const amrex::Box &bx, const amrex::Array4< amrex::Real const > &state, const amrex::Array4< amrex::Real > &rho, const amrex::Array4< amrex::Real > &rhoA, const amrex::Array4< amrex::Real > &rhoS, const amrex::Array4< amrex::Real const > &Hz, const amrex::Array4< amrex::Real const > &z_w, const amrex::Array4< amrex::Real const > &h, const int N)
void	prsgrd (const amrex::Box &bx, const amrex::Box &gbx, const amrex::Box &utbx, const amrex::Box &vtbx, const amrex::Array4< amrex::Real > &ru, const amrex::Array4< amrex::Real > &rv, const amrex::Array4< amrex::Real const > &pn, const amrex::Array4< amrex::Real const > &pm, const amrex::Array4< amrex::Real const > &rho, const amrex::Array4< amrex::Real > &FC, const amrex::Array4< amrex::Real const > &Hz, const amrex::Array4< amrex::Real const > &z_r, const amrex::Array4< amrex::Real const > &z_w, const int nrhs, const int N)
void	prestep_diffusion (const amrex::Box &bx, const amrex::Box &gbx, const int ioff, const int joff, const amrex::Array4< amrex::Real > &vel, const amrex::Array4< amrex::Real const > &vel_old, const amrex::Array4< amrex::Real > &rvel, const amrex::Array4< amrex::Real const > &Hz, const amrex::Array4< amrex::Real const > &Akv, const amrex::Array4< amrex::Real > &DC, const amrex::Array4< amrex::Real > &FC, const amrex::Array4< amrex::Real const > &sstr, const amrex::Array4< amrex::Real const > &bstr, const amrex::Array4< amrex::Real const > &z_r, const amrex::Array4< amrex::Real const > &pm, const amrex::Array4< amrex::Real const > &pn, const int iic, const int ntfirst, const int nnew, int nstp, int nrhs, int N, const amrex::Real lambda, const amrex::Real dt_lev)
void	vert_visc_3d (const amrex::Box &bx, const int ioff, const int joff, const amrex::Array4< amrex::Real > &phi, const amrex::Array4< amrex::Real const > &Hz, const amrex::Array4< amrex::Real > &Hzk, const amrex::Array4< amrex::Real > &AK, const amrex::Array4< amrex::Real > &Akv, const amrex::Array4< amrex::Real > &BC, const amrex::Array4< amrex::Real > &DC, const amrex::Array4< amrex::Real > &FC, const amrex::Array4< amrex::Real > &CF, const int nnew, const int N, const amrex::Real dt_lev)
void	update_massflux_3d (const amrex::Box &bx, const int ioff, const int joff, const amrex::Array4< amrex::Real > &phi, const amrex::Array4< amrex::Real > &phibar, const amrex::Array4< amrex::Real > &Hphi, const amrex::Array4< amrex::Real const > &Hz, const amrex::Array4< amrex::Real const > &pm_or_pn, const amrex::Array4< amrex::Real const > &Dphi1, const amrex::Array4< amrex::Real const > &Dphi2, const amrex::Array4< amrex::Real > &DC, const amrex::Array4< amrex::Real > &FC, const int nnew)
void	vert_mean_3d (const amrex::Box &bx, const int ioff, const int joff, const amrex::Array4< amrex::Real > &phi, const amrex::Array4< amrex::Real const > &Hz, const amrex::Array4< amrex::Real const > &Dphi_avg1, const amrex::Array4< amrex::Real > &DC, const amrex::Array4< amrex::Real > &CF, const amrex::Array4< amrex::Real const > &dxlen, const int nnew, const int N)
void	uv3dmix (const amrex::Box &xbx, const amrex::Box &ybx, const amrex::Array4< amrex::Real > &u, const amrex::Array4< amrex::Real > &v, const amrex::Array4< amrex::Real const > &uold, const amrex::Array4< amrex::Real const > &vold, const amrex::Array4< amrex::Real > &rufrc, const amrex::Array4< amrex::Real > &rvfrc, const amrex::Array4< amrex::Real const > &visc2_p, const amrex::Array4< amrex::Real const > &visc2_r, const amrex::Array4< amrex::Real const > &Hz, const amrex::Array4< amrex::Real const > &pm, const amrex::Array4< amrex::Real const > &pn, int nrhs, int nnew, const amrex::Real dt_lev)
void	t3dmix (const amrex::Box &bx, const amrex::Array4< amrex::Real > &state, const amrex::Array4< amrex::Real const > &diff2, const amrex::Array4< amrex::Real const > &Hz, const amrex::Array4< amrex::Real const > &pm, const amrex::Array4< amrex::Real const > &pn, const amrex::Real dt_lev, const int ncomp)
void	coriolis (const amrex::Box &xbx, const amrex::Box &ybx, const amrex::Array4< amrex::Real const > &uold, const amrex::Array4< amrex::Real const > &vold, const amrex::Array4< amrex::Real > &ru, const amrex::Array4< amrex::Real > &rv, const amrex::Array4< amrex::Real const > &Hz, const amrex::Array4< amrex::Real const > &fomn, int nrhs, int nr)
void	set_2darrays (int lev)
void	set_bathymetry (int lev)
void	set_coriolis (int lev)
void	stretch_transform (int lev)
void	set_vmix (int lev)
void	set_smflux (int lev, amrex::Real time)
void	set_hmixcoef (int lev)
void	set_drag (int lev)
void	set_weights (int lev)
void	FillPatch (int lev, amrex::Real time, amrex::MultiFab &mf_to_be_filled, amrex::Vector< amrex::MultiFab * > const &mfs, const int bdy_var_type=BdyVars::null)
void	fill_from_bdyfiles (amrex::MultiFab &mf_to_fill, const amrex::Real time, const int bdy_var_type)
void	init_beta_plane_coriolis (int lev)
void	writeJobInfo (const std::string &dir) const

Static Public Member Functions
static void	writeBuildInfo (std::ostream &os)

Public Attributes
std::string	pp_prefix {"remora"}
amrex::Vector< amrex::MultiFab * >	cons_old
amrex::Vector< amrex::MultiFab * >	xvel_old
amrex::Vector< amrex::MultiFab * >	yvel_old
amrex::Vector< amrex::MultiFab * >	zvel_old
amrex::Vector< amrex::MultiFab * >	cons_new
amrex::Vector< amrex::MultiFab * >	xvel_new
amrex::Vector< amrex::MultiFab * >	yvel_new
amrex::Vector< amrex::MultiFab * >	zvel_new
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_hOfTheConfusingName
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_Hz
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_Huon
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_Hvom
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_ru
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_rv
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_rufrc
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_rvfrc
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_Akv
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_Akt
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_visc2_p
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_visc2_r
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_diff2
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_x_r
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_y_r
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_z_r
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_z_w
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_s_r
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_z_phys_nd
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_Zt_avg1
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_sustr
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_svstr
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_rdrag
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_bustr
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_bvstr
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_DU_avg1
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_DU_avg2
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_DV_avg1
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_DV_avg2
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_rubar
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_rvbar
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_rzeta
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_ubar
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_vbar
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_zeta
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_pm
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_pn
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_fcor
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_sstore
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_visc3d_r
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_rhoS
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	vec_rhoA
amrex::Vector< amrex::Real >	vec_weight1
amrex::Vector< amrex::Real >	vec_weight2

Private Member Functions
void	ReadParameters ()
void	AverageDown ()
void	init1DArrays ()
void	init_bcs ()
void	init_custom (int lev)
void	init_stuff (int lev, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm)
void	resize_stuff (int lev)
void	AverageDownTo (int crse_lev)
void	FillCoarsePatch (int lev, amrex::Real time, amrex::MultiFab *mf_fine, amrex::MultiFab *mf_crse)
TimeInterpolatedData	GetDataAtTime (int lev, amrex::Real time)
void	timeStep (int lev, amrex::Real time, int iteration)
void	timeStepML (amrex::Real time, int iteration)
void	initHSE ()
	Initialize HSE. More...
void	initRayleigh ()
	Initialize Rayleigh damping profiles. More...
void	ComputeDt ()
std::string	PlotFileName (int lev) const
amrex::Vector< std::string >	PlotFileVarNames (amrex::Vector< std::string > plot_var_names) const
void	setPlotVariables (const std::string &pp_plot_var_names, amrex::Vector< std::string > &plot_var_names)
void	WriteCheckpointFile () const
void	ReadCheckpointFile ()
void	InitializeFromFile ()
void	InitializeLevelFromData (int lev, const amrex::MultiFab &initial_data)
void	post_update (amrex::MultiFab &state_mf, const amrex::Real time, const amrex::Geometry &geom)
void	fill_rhs (amrex::MultiFab &rhs_mf, const amrex::MultiFab &state_mf, const amrex::Real time, const amrex::Geometry &geom)
void	refinement_criteria_setup ()
AMREX_FORCE_INLINE int	ComputeGhostCells (const int &spatial_order)
AMREX_FORCE_INLINE amrex::YAFluxRegister *	getAdvFluxReg (int lev)
AMREX_FORCE_INLINE std::ostream &	DataLog (int i)
AMREX_FORCE_INLINE int	NumDataLogs () noexcept
amrex::Real	getCPUTime () const
void	setRecordDataInfo (int i, const std::string &filename)
const std::string	DataLogName (int i) const noexcept
	The filename of the ith datalog file. More...

Static Private Member Functions
static void	GotoNextLine (std::istream &is)

Private Attributes
amrex::Vector< int >	num_boxes_at_level
amrex::Vector< int >	num_files_at_level
amrex::Vector< amrex::Vector< amrex::Box > >	boxes_at_level
amrex::Vector< int >	istep
amrex::Vector< int >	nsubsteps
amrex::Vector< amrex::Real >	t_new
amrex::Vector< amrex::Real >	t_old
amrex::Vector< amrex::Real >	dt
amrex::Vector< std::unique_ptr< REMORAPhysBCFunct > >	physbcs
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mapfac_m
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mapfac_u
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	mapfac_v
amrex::Vector< std::unique_ptr< amrex::MultiFab > >	sst
amrex::Vector< amrex::YAFluxRegister * >	advflux_reg
amrex::Vector< amrex::BCRec >	domain_bcs_type
amrex::Gpu::DeviceVector< amrex::BCRec >	domain_bcs_type_d
amrex::Array< std::string, 2 *AMREX_SPACEDIM >	domain_bc_type
amrex::Array< amrex::Array< amrex::Real, AMREX_SPACEDIM *2 >, AMREX_SPACEDIM+NCONS >	m_bc_extdir_vals
amrex::GpuArray< REMORA_BC, AMREX_SPACEDIM *2 >	phys_bc_type
int	last_plot_file_step_1
int	last_plot_file_step_2
int	last_check_file_step
int	plot_file_on_restart = 1
int	max_step = std::numeric_limits<int>::max()
amrex::Real	stop_time = std::numeric_limits<amrex::Real>::max()
std::string	restart_chkfile = ""
int	nfast
int	do_substep
int	regrid_int = 2
std::string	plot_file_1 {"plt_1_"}
std::string	plot_file_2 {"plt_2_"}
int	plot_int_1 = -1
int	plot_int_2 = -1
std::string	check_file {"chk"}
int	check_int = -1
amrex::Vector< std::string >	plot_var_names_1
amrex::Vector< std::string >	plot_var_names_2
const amrex::Vector< std::string >	cons_names {"temp", "salt", "scalar"}
const amrex::Vector< std::string >	derived_names {"dpdx", "dpdy"}
amrex::Vector< amrex::Real >	h_havg_density
amrex::Vector< amrex::Real >	h_havg_temperature
amrex::Vector< amrex::Real >	h_havg_pressure
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_density
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_temperature
amrex::Gpu::DeviceVector< amrex::Real >	d_havg_pressure
amrex::MultiFab	fine_mask
amrex::Vector< std::unique_ptr< std::fstream > >	datalog
amrex::Vector< std::string >	datalogname

Static Private Attributes
static amrex::Real	cfl = 0.8
static amrex::Real	init_shrink = 1.0
static amrex::Real	change_max = 1.1
static amrex::Real	fixed_dt = -1.0
static amrex::Real	fixed_fast_dt = -1.0
static int	fixed_ndtfast_ratio = 0
static SolverChoice	solverChoice
static int	verbose = 0
static bool	use_tracer_particles
static int	sum_interval = -1
static amrex::Real	sum_per = -1.0
static PlotfileType	plotfile_type = PlotfileType::amrex
static amrex::Vector< amrex::Vector< std::string > >	nc_init_file
static amrex::Vector< amrex::Vector< std::string > >	nc_grid_file
static std::string	nc_bdry_file
static std::string	input_sounding_file
static int	output_1d_column
static int	column_interval
static amrex::Real	column_per
static amrex::Real	column_loc_x
static amrex::Real	column_loc_y
static std::string	column_file_name
static int	output_bndry_planes
static int	bndry_output_planes_interval
static amrex::Real	bndry_output_planes_per
static amrex::Real	bndry_output_planes_start_time
static int	input_bndry_planes
static amrex::Vector< amrex::AMRErrorTag >	ref_tags
static amrex::Real	startCPUTime = 0.0
static amrex::Real	previousCPUTimeUsed = 0.0

Constructor & Destructor Documentation

REMORA()

REMORA::REMORA

(

)

constructor - reads in parameters from inputs file

• sizes multilevel arrays and data structures

{
    if (ParallelDescriptor::IOProcessor()) {
        const char* remora_hash = amrex::buildInfoGetGitHash(1);
        const char* amrex_hash = amrex::buildInfoGetGitHash(2);
        const char* buildgithash = amrex::buildInfoGetBuildGitHash();
        const char* buildgitname = amrex::buildInfoGetBuildGitName();
 
        if (strlen(remora_hash) > 0) {
          amrex::Print() << "\n"
                         << "REMORA git hash: " << remora_hash << "\n";
        }
        if (strlen(amrex_hash) > 0) {
          amrex::Print() << "AMReX git hash: " << amrex_hash << "\n";
        }
        if (strlen(buildgithash) > 0) {
          amrex::Print() << buildgitname << " git hash: " << buildgithash << "\n";
        }
 
        amrex::Print() << "\n";
    }
 
    ReadParameters();
    const std::string& pv1 = "plot_vars_1"; setPlotVariables(pv1,plot_var_names_1);
    const std::string& pv2 = "plot_vars_2"; setPlotVariables(pv2,plot_var_names_2);
 
    amrex_probinit(geom[0].ProbLo(),geom[0].ProbHi());
 
    // Geometry on all levels has been defined already.
 
    // No valid BoxArray and DistributionMapping have been defined.
    // But the arrays for them have been resized.
 
    int nlevs_max = max_level + 1;
 
    istep.resize(nlevs_max, 0);
    nsubsteps.resize(nlevs_max, 1);
    for (int lev = 1; lev <= max_level; ++lev) {
        nsubsteps[lev] = do_substep ? MaxRefRatio(lev-1) : 1;
    }
 
    physbcs.resize(nlevs_max);
 
    t_new.resize(nlevs_max, 0.0);
    t_old.resize(nlevs_max, -1.e100);
    dt.resize(nlevs_max, 1.e100);
 
    cons_new.resize(nlevs_max);
    cons_old.resize(nlevs_max);
    xvel_new.resize(nlevs_max);
    xvel_old.resize(nlevs_max);
    yvel_new.resize(nlevs_max);
    yvel_old.resize(nlevs_max);
    zvel_new.resize(nlevs_max);
    zvel_old.resize(nlevs_max);
 
    advflux_reg.resize(nlevs_max);
 
    // Initialize tagging criteria for mesh refinement
    refinement_criteria_setup();
 
    // We have already read in the ref_Ratio (via amr.ref_ratio =) but we need to enforce
    //     that there is no refinement in the vertical so we test on that here.
    for (int lev = 0; lev < max_level; ++lev)
    {
       amrex::Print() << "Refinement ratio at level " << lev << " set to be " <<
          ref_ratio[lev][0]  << " " << ref_ratio[lev][1]  <<  " " << ref_ratio[lev][2] << std::endl;
 
       if (ref_ratio[lev][2] != 1)
       {
           amrex::Error("We don't allow refinement in the vertical -- make sure to set ref_ratio = 1 in z");
       }
    }
}

Here is the call graph for this function:

~REMORA()

REMORA::~REMORA ( )

virtual

{
}

Member Function Documentation

Advance()

void REMORA::Advance	(	int	lev,
		amrex::Real	time,
		amrex::Real	dt_lev,
		int	iteration,
		int	ncycle
	)

advance a single level for a single time step

{
    BL_PROFILE("REMORA::Advance()");
 
    setup_step(lev, time, dt_lev);
 
    if (solverChoice.use_barotropic)
    {
        int nfast_counter=nfast + 1;
 
        //***************************************************
        //Compute fast timestep from dt_lev and ratio
        //***************************************************
        Real dtfast_lev=dt_lev/Real(fixed_ndtfast_ratio);
 
        //***************************************************
        //Advance nfast_counter steps of the 2d integrator
        //***************************************************
        for (int my_iif = 0; my_iif < nfast_counter; my_iif++) {
            advance_2d_onestep(lev, dt_lev, dtfast_lev, my_iif, nfast_counter);
        }
    }
 
    //***************************************************
    //Advance one step of the 3d integrator
    //***************************************************
    advance_3d_ml(lev, dt_lev);
}

advance_2d()

void REMORA::advance_2d	(	int	lev,
		amrex::MultiFab const *	mf_rhoS,
		amrex::MultiFab const *	mf_rhoA,
		amrex::MultiFab *	mf_ru,
		amrex::MultiFab *	mf_rv,
		amrex::MultiFab *	mf_rufrc,
		amrex::MultiFab *	mf_rvfrc,
		amrex::MultiFab *	mf_Zt_avg1,
		std::unique_ptr< amrex::MultiFab > &	mf_DU_avg1,
		std::unique_ptr< amrex::MultiFab > &	mf_DU_avg2,
		std::unique_ptr< amrex::MultiFab > &	mf_DV_avg1,
		std::unique_ptr< amrex::MultiFab > &	mf_DV_avg2,
		std::unique_ptr< amrex::MultiFab > &	mf_rubar,
		std::unique_ptr< amrex::MultiFab > &	mf_rvbar,
		std::unique_ptr< amrex::MultiFab > &	mf_rzeta,
		std::unique_ptr< amrex::MultiFab > &	mf_ubar,
		std::unique_ptr< amrex::MultiFab > &	mf_vbar,
		amrex::MultiFab *	mf_zeta,
		amrex::MultiFab const *	mf_h,
		amrex::MultiFab const *	mf_pm,
		amrex::MultiFab const *	mf_pn,
		amrex::MultiFab const *	mf_fcor,
		amrex::MultiFab const *	mf_visc2_p,
		amrex::MultiFab const *	mf_visc2_r,
		const amrex::Real	dtfast_lev,
		const bool	predictor_2d_step,
		const bool	first_2d_step,
		int	my_iif,
		int &	next_indx1
	)

Perform a 2D predictor (predictor_2d_step=True) or corrector (predictor_2d_step=False) step

Function that coordinates the evolution across levels – this calls Advance to do the actual advance at this level, then recursively calls itself at finer levels

Parameters

[in]	lev	level of refinement (coarsest level is 0)
[in]	mf_rhoS
[in]	mf_rhoA
[in]	mf_ru
[in]	mf_rv
[in,out]	mf_rufrc
[in,out]	mf_rvfrc
[in,out]	mf_Zt_avg1
[in,out]	mf_DU_avg1
[in,out]	mf_DU_avg2
[in,out]	mf_DV_avg1
[in,out]	mf_DV_avg2
[in,out]	mf_rubar
[in,out]	mf_rvbar
[in,out]	mf_zeta
[in]	mf_h
[in,out]	mf_visc2_p
[in,out]	mf_visc2_f
[in]	dtfast_lev
[in]	predictor_2d_step
[in]	first_2d_step
[in]	my_iif
[in]	next_indx1

{
    int iic = istep[lev];
    const int nnew  = 0;
    const int nstp  = 0;
    int ntfirst = 0;
 
    int knew = 3;
    int krhs = (my_iif + iic) % 2 + 1;
    int kstp = my_iif <=1 ? iic % 2 + 1 : (iic % 2 + my_iif % 2 + 1) % 2 + 1;
    int indx1 = krhs;
    if (predictor_2d_step) {
        next_indx1 = 3 - indx1;
    } else {
        knew = next_indx1;
        kstp = 3 - knew;
        krhs = 3;
        //If it's not the auxiliary time step, set indx1 to next_indx1
        // NOTE: should this ever not execute?
        if (my_iif<nfast+1)
            indx1=next_indx1;
    }
    int ptsk = 3-kstp;
    knew-=1;
    krhs-=1;
    kstp-=1;
    indx1-=1;
    ptsk-=1;
    auto ba = mf_h->boxArray();
    auto dm = mf_h->DistributionMap();
 
    MultiFab mf_DUon(convert(ba,IntVect(1,0,0)),dm,1,IntVect(NGROW,NGROW,0));
    MultiFab mf_DVom(convert(ba,IntVect(0,1,0)),dm,1,IntVect(NGROW,NGROW,0));
 
    for ( MFIter mfi(*mf_rhoS, TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
        Array4<Real      > const& ubar = mf_ubar->array(mfi);
        Array4<Real      > const& vbar = mf_vbar->array(mfi);
        Array4<Real      > const& zeta = mf_zeta->array(mfi);
        Array4<Real const> const& h    = mf_h->const_array(mfi);
 
        Array4<Real const> const& pm   = mf_pm->const_array(mfi);
        Array4<Real const> const& pn   = mf_pn->const_array(mfi);
 
        Box bx = mfi.tilebox();
        Box gbx = mfi.growntilebox();
        Box gbx1 = mfi.growntilebox(IntVect(NGROW-1,NGROW-1,0));
        Box gbx2 = mfi.growntilebox(IntVect(NGROW,NGROW,0));
        Box xgbx2 = mfi.grownnodaltilebox(0, IntVect(NGROW,NGROW,0));
        Box ygbx2 = mfi.grownnodaltilebox(1, IntVect(NGROW,NGROW,0));
 
        Box tbxp1 = bx;
        Box tbxp11 = bx;
        Box tbxp2 = bx;
        Box tbxp3 = bx;
        tbxp1.grow(IntVect(NGROW-1,NGROW-1,0));
        tbxp2.grow(IntVect(NGROW,NGROW,0));
        tbxp11.grow(IntVect(NGROW-1,NGROW-1,NGROW-1));
        tbxp3.grow(IntVect(NGROW+1,NGROW+1,0));
 
        Box bxD   = bx  ;   bxD.makeSlab(2,0);
        Box gbxD  = gbx ;  gbxD.makeSlab(2,0);
        Box gbx1D = gbx1; gbx1D.makeSlab(2,0);
        Box gbx2D = gbx2; gbx2D.makeSlab(2,0);
 
        Box tbxp2D = tbxp2;
        tbxp2D.makeSlab(2,0);
 
        // step2d work arrays
        FArrayBox fab_Drhs(tbxp3,1,The_Async_Arena());
        auto Drhs=fab_Drhs.array();
 
        auto DUon = mf_DUon.array(mfi);
        auto DVom = mf_DVom.array(mfi);
 
        ParallelFor(makeSlab(tbxp3,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
        {
            Drhs(i,j,0)=zeta(i,j,0,krhs)+h(i,j,0);
        });
 
        ParallelFor(makeSlab(xgbx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
        {
            Real on_u = 2.0_rt / (pn(i,j,0)+pn(i-1,j,0));
            Real cff1= 0.5_rt * on_u *(Drhs(i,j,0)+Drhs(i-1,j,0));
            DUon(i,j,0)=ubar(i,j,0,krhs)*cff1;
        });
 
        ParallelFor(makeSlab(ygbx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
        {
            Real om_v = 2.0_rt / (pm(i,j,0)+pm(i,j-1,0));
            Real cff1= 0.5_rt * om_v * (Drhs(i,j,0)+Drhs(i,j-1,0));
            DVom(i,j,0)=vbar(i,j,0,krhs)*cff1;
        });
    }
 
    // These are needed to pass the tests with bathymetry but I don't quite see why
    mf_DUon.FillBoundary(geom[lev].periodicity());
    mf_DVom.FillBoundary(geom[lev].periodicity());
 
    for ( MFIter mfi(*mf_rhoS, TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
        Array4<Real const> const& rhoS = mf_rhoS->const_array(mfi);
        Array4<Real const> const& rhoA = mf_rhoA->const_array(mfi);
        Array4<Real const> const& h    = mf_h->const_array(mfi);
 
        Array4<Real      > const& rufrc   = mf_rufrc->array(mfi);
        Array4<Real      > const& rvfrc   = mf_rvfrc->array(mfi);
        Array4<Real      > const& Zt_avg1 = mf_Zt_avg1->array(mfi);
        Array4<Real      > const& ubar    = mf_ubar->array(mfi);
        Array4<Real      > const& vbar    = mf_vbar->array(mfi);
        Array4<Real      > const& zeta = mf_zeta->array(mfi);
        Array4<Real      > const& DU_avg1 = (mf_DU_avg1)->array(mfi);
        Array4<Real      > const& DU_avg2 = (mf_DU_avg2)->array(mfi);
        Array4<Real      > const& DV_avg1 = (mf_DV_avg1)->array(mfi);
        Array4<Real      > const& DV_avg2 = (mf_DV_avg2)->array(mfi);
        Array4<Real      > const& ru = (mf_ru)->array(mfi);
        Array4<Real      > const& rv = (mf_rv)->array(mfi);
        Array4<Real      > const& rubar = (mf_rubar)->array(mfi);
        Array4<Real      > const& rvbar = (mf_rvbar)->array(mfi);
        Array4<Real      > const& rzeta = (mf_rzeta)->array(mfi);
        Array4<Real const> const& visc2_p = mf_visc2_p->const_array(mfi);
        Array4<Real const> const& visc2_r = mf_visc2_r->const_array(mfi);
 
        Array4<Real const> const& pm   = mf_pm->const_array(mfi);
        Array4<Real const> const& pn   = mf_pn->const_array(mfi);
        Array4<Real const> const& fcor = mf_fcor->const_array(mfi);
 
        Box bx = mfi.tilebox();
        Box gbx = mfi.growntilebox();
        Box gbx1 = mfi.growntilebox(IntVect(NGROW-1,NGROW-1,0));
        Box gbx2 = mfi.growntilebox(IntVect(NGROW,NGROW,0));
        Box gbx3 = mfi.growntilebox(IntVect(NGROW+1,NGROW+1,0));
        Box xgbx2 = mfi.grownnodaltilebox(0, IntVect(NGROW,NGROW,0));
        Box ygbx2 = mfi.grownnodaltilebox(1, IntVect(NGROW,NGROW,0));
 
        Box xbxD = mfi.nodaltilebox(0);
        xbxD.makeSlab(2,0);
 
        Box ybxD = mfi.nodaltilebox(1);
        ybxD.makeSlab(2,0);
 
        Box tbxp1  = bx;  tbxp1.grow(IntVect(NGROW-1,NGROW-1,0));
        Box tbxp11 = bx; tbxp11.grow(IntVect(NGROW-1,NGROW-1,NGROW-1));
        Box tbxp2  = bx;  tbxp2.grow(IntVect(NGROW,NGROW,0));
        Box tbxp3  = bx;  tbxp3.grow(IntVect(NGROW+1,NGROW+1,0));
 
        Box bxD   = bx;   bxD.makeSlab(2,0);
        Box gbxD  = gbx;  gbxD.makeSlab(2,0);
        Box gbx1D = gbx1; gbx1D.makeSlab(2,0);
        Box gbx2D = gbx2; gbx2D.makeSlab(2,0);
 
        Box tbxp2D = tbxp2;
        tbxp2D.makeSlab(2,0);
 
        FArrayBox fab_fomn(tbxp2,1,The_Async_Arena());
 
        //step2d work arrays
        FArrayBox fab_Drhs(tbxp3,1,The_Async_Arena());
        FArrayBox fab_Dnew(tbxp2,1,The_Async_Arena());
        FArrayBox fab_zwrk(tbxp1,1,The_Async_Arena());
        FArrayBox fab_gzeta(tbxp1,1,The_Async_Arena());
        FArrayBox fab_gzeta2(tbxp1,1,The_Async_Arena());
        FArrayBox fab_gzetaSA(tbxp1,1,The_Async_Arena());
        FArrayBox fab_Dstp(tbxp3,1,The_Async_Arena());
        FArrayBox & fab_DUon=mf_DUon[mfi];
        FArrayBox & fab_DVom=mf_DVom[mfi];
        FArrayBox fab_rhs_ubar(xbxD,1,The_Async_Arena());
        FArrayBox fab_rhs_vbar(ybxD,1,The_Async_Arena());
        FArrayBox fab_rhs_zeta(tbxp1,1,The_Async_Arena());
        FArrayBox fab_zeta_new(tbxp1,1,The_Async_Arena());
 
        auto fomn=fab_fomn.array();
 
        auto Drhs = fab_Drhs.array();
        auto Drhs_const = fab_Drhs.const_array();
        auto Dnew=fab_Dnew.array();
        auto zwrk=fab_zwrk.array();
        auto gzeta=fab_gzeta.array();
        auto gzeta2=fab_gzeta2.array();
        auto gzetaSA=fab_gzetaSA.array();
        auto Dstp=fab_Dstp.array();
        auto DUon=fab_DUon.array();
        auto DVom=fab_DVom.array();
        auto rhs_ubar=fab_rhs_ubar.array();
        auto rhs_vbar=fab_rhs_vbar.array();
        auto rhs_zeta=fab_rhs_zeta.array();
        auto zeta_new=fab_zeta_new.array();
 
        auto weight1 = vec_weight1.dataPtr();
        auto weight2 = vec_weight2.dataPtr();
 
        //From ana_grid.h and metrics.F
        ParallelFor(xbxD, [=] AMREX_GPU_DEVICE (int i, int j, int)
        {
              rhs_ubar(i,j,0)=0.0_rt;
        });
 
        ParallelFor(ybxD, [=] AMREX_GPU_DEVICE (int i, int j, int)
        {
              rhs_vbar(i,j,0)=0.0_rt;
        });
 
        if (solverChoice.use_coriolis) {
            ParallelFor(tbxp2D, [=] AMREX_GPU_DEVICE (int i, int j, int  )
            {
                fomn(i,j,0) = fcor(i,j,0)*(1.0/(pm(i,j,0)*pn(i,j,0)));
            });
        }
 
        ParallelFor(makeSlab(tbxp3,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
        {
            Drhs(i,j,0)=zeta(i,j,0,krhs)+h(i,j,0);
        });
 
        if(predictor_2d_step)
        {
            if(first_2d_step) {
                Real cff2=(-1.0_rt/12.0_rt)*weight2[my_iif+1];
                ParallelFor(makeSlab(gbx3,2,0),
                [=] AMREX_GPU_DEVICE (int i, int j, int)
                {
                    Zt_avg1(i,j,0)=0.0_rt;
                });
                ParallelFor(makeSlab(xgbx2,2,0),
                [=] AMREX_GPU_DEVICE (int i, int j, int)
                {
                    DU_avg1(i,j,0)=0.0_rt;
                    DU_avg2(i,j,0)=cff2*DUon(i,j,0);
                });
                ParallelFor(makeSlab(ygbx2,2,0),
                [=] AMREX_GPU_DEVICE (int i, int j, int)
                {
                    DV_avg1(i,j,0)=0.0_rt;
                    DV_avg2(i,j,0)=cff2*DVom(i,j,0);
                });
            }
            else {
                Real cff1_wt1 = weight1[my_iif-1];
                Real cff2_wt1 = (8.0_rt/12.0_rt)*weight2[my_iif]-
                                (1.0_rt/12.0_rt)*weight2[my_iif+1];
 
                ParallelFor(makeSlab(gbx3,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
                {
                    Zt_avg1(i,j,0) += cff1_wt1*zeta(i,j,0,krhs);
                });
 
                ParallelFor(makeSlab(xgbx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
                {
                    DU_avg1(i,j,0) += cff1_wt1*DUon(i,j,0);
                    DU_avg2(i,j,0) += cff2_wt1*DUon(i,j,0);
                });
 
                ParallelFor(makeSlab(ygbx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
                {
                    DV_avg1(i,j,0) += cff1_wt1*DVom(i,j,0);
                    DV_avg2(i,j,0) += cff2_wt1*DVom(i,j,0);
                });
            }
        }
        else {
            Real cff2_wt2;
 
            if (first_2d_step) {
                cff2_wt2=weight2[my_iif];
            } else {
                cff2_wt2=5.0_rt/12.0_rt*weight2[my_iif];
            }
 
            ParallelFor(makeSlab(xgbx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
            {
                DU_avg2(i,j,0)=DU_avg2(i,j,0)+cff2_wt2*DUon(i,j,0);
            });
 
            ParallelFor(makeSlab(ygbx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
            {
                DV_avg2(i,j,0)=DV_avg2(i,j,0)+cff2_wt2*DVom(i,j,0);
            });
        }
        //
        //  Do not perform the actual time stepping during the auxiliary
        //  (nfast(ng)+1) time step. Jump to next box
        //
 
        if (my_iif>=nfast) {
            continue; }
        //Load new free-surface values into shared array at both predictor
        //and corrector steps
        //
        //=======================================================================
        //  Time step free-surface equation.
        //=======================================================================
        //
        //  During the first time-step, the predictor step is Forward-Euler
        //  and the corrector step is Backward-Euler. Otherwise, the predictor
        //  step is Leap-frog and the corrector step is Adams-Moulton.
        //
 
        // todo: gzeta
 
        // todo: HACKHACKHACK Should use rho0 from prob.H
        Real fac=1000.0_rt/1025.0_rt;
 
        if (my_iif==0) {
            Real cff1=dtfast_lev;
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                rhs_zeta(i,j,0) = (DUon(i,j,0)-DUon(i+1,j,0))+
                                  (DVom(i,j,0)-DVom(i,j+1,0));
                zeta_new(i,j,0) = zeta(i,j,0,kstp)+ pm(i,j,0)*pn(i,j,0)*cff1*rhs_zeta(i,j,0);
                Dnew(i,j,0) = zeta_new(i,j,0)+h(i,j,0);
 
                //Pressure gradient terms:
                zwrk(i,j,0)=0.5_rt*(zeta(i,j,0,kstp)+zeta_new(i,j,0));
                gzeta(i,j,0)=(fac+rhoS(i,j,0))*zwrk(i,j,0);
                gzeta2(i,j,0)=gzeta(i,j,0)*zwrk(i,j,0);
                gzetaSA(i,j,0)=zwrk(i,j,0)*(rhoS(i,j,0)-rhoA(i,j,0));
            });
 
        } else if (predictor_2d_step) {
 
            Real cff1=2.0_rt * dtfast_lev;
            Real cff4=4.0_rt / 25.0_rt;
            Real cff5=1.0_rt - 2.0_rt*cff4;
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                rhs_zeta(i,j,0)=(DUon(i,j,0)-DUon(i+1,j,0))+
                                (DVom(i,j,0)-DVom(i,j+1,0));
                zeta_new(i,j,0)=zeta(i,j,0,kstp)+
                                pm(i,j,0)*pn(i,j,0)*cff1*rhs_zeta(i,j,0);
                Dnew(i,j,0)=zeta_new(i,j,0)+h(i,j,0);
                //Pressure gradient terms
                zwrk(i,j,0)=cff5*zeta(i,j,0,krhs)+
                    cff4*(zeta(i,j,0,kstp)+zeta_new(i,j,0));
                gzeta(i,j,0)=(fac+rhoS(i,j,0))*zwrk(i,j,0);
                gzeta2(i,j,0)=gzeta(i,j,0)*zwrk(i,j,0);
                gzetaSA(i,j,0)=zwrk(i,j,0)*(rhoS(i,j,0)-rhoA(i,j,0));
            });
 
        } else if (!predictor_2d_step) { //AKA if(corrector_2d_step)
 
            Real cff1=dtfast_lev * 5.0_rt/12.0_rt;
            Real cff2=dtfast_lev * 8.0_rt/12.0_rt;
            Real cff3=dtfast_lev * 1.0_rt/12.0_rt;
            Real cff4=2.0_rt/5.0_rt;
            Real cff5=1.0_rt-cff4;
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                Real cff=cff1*((DUon(i,j,0)-DUon(i+1,j,0))+
                               (DVom(i,j,0)-DVom(i,j+1,0)));
                zeta_new(i,j,0)=zeta(i,j,0,kstp)+
                    pm(i,j,0)*pn(i,j,0)*(cff+
                                         cff2*rzeta(i,j,0,kstp)-
                                         cff3*rzeta(i,j,0,ptsk));
                Dnew(i,j,0)=zeta_new(i,j,0)+h(i,j,0);
                //Pressure gradient terms
                zwrk(i,j,0)=cff5*zeta_new(i,j,0)+cff4*zeta(i,j,0,krhs);
                gzeta(i,j,0)=(fac+rhoS(i,j,0))*zwrk(i,j,0);
                gzeta2(i,j,0)=gzeta(i,j,0)*zwrk(i,j,0);
                gzetaSA(i,j,0)=zwrk(i,j,0)*(rhoS(i,j,0)-rhoA(i,j,0));
            });
        }
 
        //
        //  Load new free-surface values into shared array at both predictor
        //  and corrector steps.
        //
        //// zeta(knew) only valid at zeta_new, i.e. tbxp1
        ParallelFor(gbx1,
        [=] AMREX_GPU_DEVICE (int i, int j, int )
        {
            zeta(i,j,0,knew) = zeta_new(i,j,0);
        });
 
        //
        //  If predictor step, load right-side-term into shared array.
        //
        if (predictor_2d_step) {
            ParallelFor(gbx1, [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                rzeta(i,j,0,krhs)=rhs_zeta(i,j,0);
            });
        }
 
        //
        //=======================================================================
        //  Compute right-hand-side for the 2D momentum equations.
        //=======================================================================
        //
/*
!
!-----------------------------------------------------------------------
!  Compute pressure gradient terms.
!-----------------------------------------------------------------------
!
*/
 
        Real cff1 = 0.5_rt * solverChoice.g; // Should be the variable gravitational field strength
        Real cff2 = 1.0_rt / 3.0_rt;
        ParallelFor(xbxD,
        [=] AMREX_GPU_DEVICE (int i, int j, int )
        {
          Real on_u = 2.0_rt / (pn(i,j,0)+pn(i-1,j,0));
          rhs_ubar(i,j,0)=cff1 * on_u *
                        ((    h(i-1,j,0) +     h(i,j,0))*
                         (gzeta(i-1,j,0) - gzeta(i,j,0))+
                         (    h(i-1,j,0) -     h(i,j,0))*
                         (      gzetaSA(i-1,j,0) + gzetaSA(i,j,0)+
                          cff2*(   rhoA(i-1,j,0) -    rhoA(i,j,0))*
                               (   zwrk(i-1,j,0) -    zwrk(i,j,0)))+
                         (gzeta2(i-1,j,0)- gzeta2(i  ,j,0)));
        });
 
        ParallelFor(ybxD,
        [=] AMREX_GPU_DEVICE (int i, int j, int )
        {
            Real om_v = 2.0_rt / (pm(i,j,0)+pm(i,j-1,0));
            rhs_vbar(i,j,0) = cff1*om_v *
                          ((    h(i,j-1,0) +     h(i,j,0))*
                           (gzeta(i,j-1,0) - gzeta(i,j,0))+
                           (    h(i,j-1,0) -     h(i,j,0))*
                           (gzetaSA(i,j-1,0)+ gzetaSA(i,j  ,0)+
                            cff2*(rhoA(i,j-1,0)- rhoA(i,j  ,0))*
                                 (zwrk(i,j-1,0)- zwrk(i,j  ,0)))+
                           (gzeta2(i,j-1,0)- gzeta2(i,j  ,0)));
        });
 
       // Advection terms for 2d ubar, vbar added to rhs_ubar and rhs_vbar
       //
       //-----------------------------------------------------------------------
       // rhs_uv_2d
       //-----------------------------------------------------------------------
       //
       Array4<Real const> const& ubar_const = mf_ubar->const_array(mfi);
       Array4<Real const> const& vbar_const = mf_vbar->const_array(mfi);
       rhs_uv_2d(xbxD, ybxD, ubar_const, vbar_const, rhs_ubar, rhs_vbar, DUon, DVom, krhs);
 
       //-----------------------------------------------------------------------
       // Add Coriolis forcing
       //-----------------------------------------------------------------------
       if (solverChoice.use_coriolis) {
            // Coriolis terms for 2d ubar, vbar added to rhs_ubar and rhs_vbar
            //
            //-----------------------------------------------------------------------
            // coriolis
            //-----------------------------------------------------------------------
            //
            coriolis(xbxD, ybxD, ubar_const, vbar_const, rhs_ubar, rhs_vbar, Drhs, fomn, krhs, 0);
       }
 
       //-----------------------------------------------------------------------
       //Add in horizontal harmonic viscosity.
       // Consider generalizing or copying uv3dmix, where Drhs is used instead of Hz and u=>ubar v=>vbar, drop dt terms
       //-----------------------------------------------------------------------
       uv3dmix(xbxD, ybxD, ubar, vbar, ubar, vbar, rhs_ubar, rhs_vbar,
               visc2_p, visc2_r, Drhs_const,
               pn, pm, krhs, nnew, 0.0_rt);
 
       //-----------------------------------------------------------------------
       // Coupling from 3d to 2d
       //-----------------------------------------------------------------------
       if (first_2d_step&&predictor_2d_step)
       {
            if (iic==ntfirst) {
 
                ParallelFor(xbxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
                {
                    rufrc(i,j,0)    -= rhs_ubar(i,j,0);
                    rhs_ubar(i,j,0) += rufrc(i,j,0);
                    ru(i,j,-1,nstp)  = rufrc(i,j,0);
                });
 
                ParallelFor(ybxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
                {
                    rvfrc(i,j,0)    -= rhs_vbar(i,j,0);
                    rhs_vbar(i,j,0) += rvfrc(i,j,0);
                    rv(i,j,-1,nstp)  = rvfrc(i,j,0);
                });
 
            } else if (iic==(ntfirst+1)) {
 
                ParallelFor(xbxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
                {
                    rufrc(i,j,0)=rufrc(i,j,0)-rhs_ubar(i,j,0);
                    rhs_ubar(i,j,0)=rhs_ubar(i,j,0)+1.5_rt*rufrc(i,j,0)-0.5_rt*ru(i,j,-1,0);
                    ru(i,j,-1,1)=rufrc(i,j,0);
                    Real r_swap= ru(i,j,-1,1);
                    ru(i,j,-1,1) = ru(i,j,-1,0);
                    ru(i,j,-1,0) = r_swap;
                });
 
                ParallelFor(ybxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
                {
                    rvfrc(i,j,0)=rvfrc(i,j,0)-rhs_vbar(i,j,0);
                    rhs_vbar(i,j,0)=rhs_vbar(i,j,0)+1.5_rt*rvfrc(i,j,0)-0.5_rt*rv(i,j,-1,0);
                    rv(i,j,-1,1)=rvfrc(i,j,0);
                    Real r_swap= rv(i,j,-1,1);
                    rv(i,j,-1,1) = rv(i,j,-1,0);
                    rv(i,j,-1,0) = r_swap;
                });
 
            } else {
                cff1=23.0_rt/12.0_rt;
                cff2=16.0_rt/12.0_rt;
                Real cff3= 5.0_rt/12.0_rt;
 
                ParallelFor(xbxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
                {
                    rufrc(i,j,0)=rufrc(i,j,0)-rhs_ubar(i,j,0);
                    rhs_ubar(i,j,0)=rhs_ubar(i,j,0)+
                        cff1*rufrc(i,j,0)-
                        cff2*ru(i,j,-1,0)+
                        cff3*ru(i,j,-1,1);
                    ru(i,j,-1,1)=rufrc(i,j,0);
                    Real r_swap= ru(i,j,-1,1);
                    ru(i,j,-1,1) = ru(i,j,-1,0);
                    ru(i,j,-1,0) = r_swap;
                });
 
                ParallelFor(ybxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
                {
                    rvfrc(i,j,0)=rvfrc(i,j,0)-rhs_vbar(i,j,0);
                    rhs_vbar(i,j,0)=rhs_vbar(i,j,0)+
                          cff1*rvfrc(i,j,0)-
                          cff2*rv(i,j,-1,0)+
                          cff3*rv(i,j,-1,1);
                    rv(i,j,-1,1)=rvfrc(i,j,0);
 
                    Real r_swap= rv(i,j,-1,1);
                    rv(i,j,-1,1) = rv(i,j,-1,0);
                    rv(i,j,-1,0) = r_swap;
                });
            }
       } else {
 
            ParallelFor(xbxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                rhs_ubar(i,j,0) += rufrc(i,j,0);
            });
 
            ParallelFor(ybxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                rhs_vbar(i,j,0) += rvfrc(i,j,0);
            });
        }
 
        //
        //=======================================================================
        //  Time step 2D momentum equations.
        //=======================================================================
        //
        //  Compute total water column depth.
        //
        ParallelFor(makeSlab(tbxp3,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int )
        {
              Dstp(i,j,0)=zeta(i,j,0,kstp)+h(i,j,0);
        });
 
        //
        //  During the first time-step, the predictor step is Forward-Euler
        //  and the corrector step is Backward-Euler. Otherwise, the predictor
        //  step is Leap-frog and the corrector step is Adams-Moulton.
        //
        if (my_iif==0) {
            cff1=0.5_rt*dtfast_lev;
            ParallelFor(xbxD,
            [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                Real cff=(pm(i,j,0)+pm(i-1,j,0))*(pn(i,j,0)+pn(i-1,j,0));
                Real Dnew_avg =1.0_rt/(Dnew(i,j,0)+Dnew(i-1,j,0));
                ubar(i,j,0,knew)=(ubar(i,j,0,kstp)*
                                 (Dstp(i,j,0)+Dstp(i-1,j,0))+
                                  cff*cff1*rhs_ubar(i,j,0))*Dnew_avg;
            });
            ParallelFor(ybxD,
            [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                Real cff=(pm(i,j,0)+pm(i,j-1,0))*(pn(i,j,0)+pn(i,j-1,0));
                Real Dnew_avg=1.0_rt/(Dnew(i,j,0)+Dnew(i,j-1,0));
                vbar(i,j,0,knew)=(vbar(i,j,0,kstp)*
                                 (Dstp(i,j,0)+Dstp(i,j-1,0))+
                                  cff*cff1*rhs_vbar(i,j,0))*Dnew_avg;
            });
 
        } else if (predictor_2d_step) {
 
            cff1=dtfast_lev;
            ParallelFor(xbxD,
            [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                Real cff=(pm(i,j,0)+pm(i-1,j,0))*(pn(i,j,0)+pn(i-1,j,0));
                Real Dnew_avg=1.0_rt/(Dnew(i,j,0)+Dnew(i-1,j,0));
                ubar(i,j,0,knew)=(ubar(i,j,0,kstp)*
                                 (Dstp(i,j,0)+Dstp(i-1,j,0))+
                                  cff*cff1*rhs_ubar(i,j,0))*Dnew_avg;
            });
            ParallelFor(ybxD,
            [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                Real cff=(pm(i,j,0)+pm(i,j-1,0))*(pn(i,j,0)+pn(i,j-1,0));
                Real Dnew_avg=1.0_rt/(Dnew(i,j,0)+Dnew(i,j-1,0));
                vbar(i,j,0,knew)=(vbar(i,j,0,kstp)*
                                 (Dstp(i,j,0)+Dstp(i,j-1,0))+
                                  cff*cff1*rhs_vbar(i,j,0))*Dnew_avg;
            });
 
        } else if ((!predictor_2d_step)) {
 
            cff1=0.5_rt*dtfast_lev*5.0_rt/12.0_rt;
            cff2=0.5_rt*dtfast_lev*8.0_rt/12.0_rt;
            Real cff3=0.5_rt*dtfast_lev*1.0_rt/12.0_rt;
            ParallelFor(xbxD,
            [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                Real cff=(pm(i,j,0)+pm(i-1,j,0))*(pn(i,j,0)+pn(i-1,j,0));
                Real Dnew_avg=1.0_rt/(Dnew(i,j,0)+Dnew(i-1,j,0));
                ubar(i,j,0,knew)=(ubar(i,j,0,kstp)*
                                 (Dstp(i,j,0)+Dstp(i-1,j,0))+
                                 cff*(cff1*rhs_ubar(i,j,0)+
                                      cff2*rubar(i,j,0,kstp)-
                                      cff3*rubar(i,j,0,ptsk)))*Dnew_avg;
            });
            ParallelFor(ybxD,
            [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                Real cff=(pm(i,j,0)+pm(i,j-1,0))*(pn(i,j,0)+pn(i,j-1,0));
                Real Dnew_avg=1.0_rt/(Dnew(i,j,0)+Dnew(i,j-1,0));
                vbar(i,j,0,knew)=(vbar(i,j,0,kstp)*
                                 (Dstp(i,j,0)+Dstp(i,j-1,0))+
                                 cff*(cff1*rhs_vbar(i,j,0)+
                                      cff2*rvbar(i,j,0,kstp)-
                                      cff3*rvbar(i,j,0,ptsk)))*Dnew_avg;
            });
        }
 
        //store rhs_ubar and rhs_vbar to save later
        //
        //  If predictor step, load right-side-term into shared arrays for
        //  future use during the subsequent corrector step.
        //
 
        if (predictor_2d_step) {
            ParallelFor(xbxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                rubar(i,j,0,krhs)=rhs_ubar(i,j,0);
            });
            ParallelFor(ybxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                rvbar(i,j,0,krhs)=rhs_vbar(i,j,0);
            });
        }
    }
}

advance_2d_onestep()

void REMORA::advance_2d_onestep	(	int	lev,
		amrex::Real	dt_lev,
		amrex::Real	dtfast_lev,
		int	my_iif,
		int	nfast_counter
	)

2D advance, one predictor/corrector step

{
    bool first_2d_step=(my_iif==0);
 
    // These are needed to pass ctests
    Real dummy_time = 0.0;
    FillPatch(lev, dummy_time, *vec_ubar[lev], GetVecOfPtrs(vec_ubar), BdyVars::ubar);
    FillPatch(lev, dummy_time, *vec_vbar[lev], GetVecOfPtrs(vec_vbar), BdyVars::vbar);
 
    //Predictor
    bool predictor_2d_step=true;
    int next_indx1 = 0;
    advance_2d(lev,
               vec_rhoS[lev].get() , vec_rhoA[lev].get(),
               vec_ru[lev].get()   , vec_rv[lev].get(),
               vec_rufrc[lev].get(), vec_rvfrc[lev].get(),
               vec_Zt_avg1[lev].get(),
               vec_DU_avg1[lev], vec_DU_avg2[lev],
               vec_DV_avg1[lev], vec_DV_avg2[lev],
               vec_rubar[lev], vec_rvbar[lev], vec_rzeta[lev],
                vec_ubar[lev],  vec_vbar[lev],  vec_zeta[lev].get(),
               vec_hOfTheConfusingName[lev].get(),
               vec_pm[lev].get(), vec_pn[lev].get(),
               vec_fcor[lev].get(),
               vec_visc2_p[lev].get(), vec_visc2_r[lev].get(),
               dtfast_lev, predictor_2d_step, first_2d_step, my_iif, next_indx1);
 
    //Corrector. Skip it on last fast step
    predictor_2d_step=false;
    if (my_iif < nfast_counter - 1) {
 
        // These are needed to pass ctests
        FillPatch(lev, dummy_time, *vec_ubar[lev], GetVecOfPtrs(vec_ubar), BdyVars::ubar);
        FillPatch(lev, dummy_time, *vec_vbar[lev], GetVecOfPtrs(vec_vbar), BdyVars::vbar);
 
        advance_2d(lev,
                   vec_rhoS[lev].get(), vec_rhoA[lev].get(),
                   vec_ru[lev].get(), vec_rv[lev].get(),
                   vec_rufrc[lev].get(), vec_rvfrc[lev].get(),
                   vec_Zt_avg1[lev].get(),
                   vec_DU_avg1[lev], vec_DU_avg2[lev],
                   vec_DV_avg1[lev], vec_DV_avg2[lev],
                   vec_rubar[lev], vec_rvbar[lev], vec_rzeta[lev],
                    vec_ubar[lev],  vec_vbar[lev],  vec_zeta[lev].get(),
                   vec_hOfTheConfusingName[lev].get(),
                   vec_pm[lev].get(), vec_pn[lev].get(),
                    vec_fcor[lev].get(),
                   vec_visc2_p[lev].get(), vec_visc2_r[lev].get(),
                   dtfast_lev, predictor_2d_step, first_2d_step, my_iif, next_indx1);
    }
}

advance_3d()

void REMORA::advance_3d	(	int	lev,
		amrex::MultiFab &	mf_cons,
		amrex::MultiFab &	mf_u,
		amrex::MultiFab &	mf_v,
		amrex::MultiFab *	mf_sstore,
		amrex::MultiFab *	mf_ru,
		amrex::MultiFab *	mf_rv,
		std::unique_ptr< amrex::MultiFab > &	mf_DU_avg1,
		std::unique_ptr< amrex::MultiFab > &	mf_DU_avg2,
		std::unique_ptr< amrex::MultiFab > &	mf_DV_avg1,
		std::unique_ptr< amrex::MultiFab > &	mf_DV_avg2,
		std::unique_ptr< amrex::MultiFab > &	mf_ubar,
		std::unique_ptr< amrex::MultiFab > &	mf_vbar,
		std::unique_ptr< amrex::MultiFab > &	mf_Akv,
		std::unique_ptr< amrex::MultiFab > &	mf_Akt,
		std::unique_ptr< amrex::MultiFab > &	mf_Hz,
		std::unique_ptr< amrex::MultiFab > &	mf_Huon,
		std::unique_ptr< amrex::MultiFab > &	mf_Hvom,
		std::unique_ptr< amrex::MultiFab > &	mf_z_w,
		amrex::MultiFab const *	mf_h,
		amrex::MultiFab const *	mf_pm,
		amrex::MultiFab const *	mf_pn,
		const int	N,
		const amrex::Real	dt_lev
	)

Advance the 3D variables

{
    const int nrhs  = 0;
    const int nnew  = 0;
 
    int iic = istep[lev];
    int ntfirst = 0;
 
    // Because zeta may have changed
    stretch_transform(lev);
 
    // These temporaries used to be made in advance_3d_ml and passed in;
    // now we make them here
 
    const BoxArray&            ba = mf_cons.boxArray();
    const DistributionMapping& dm = mf_cons.DistributionMap();
 
    //Only used locally, probably should be rearranged into FArrayBox declaration
    MultiFab mf_AK (ba,dm,1,IntVect(NGROW,NGROW,0));       //2d missing j coordinate
    MultiFab mf_DC (ba,dm,1,IntVect(NGROW,NGROW,NGROW-1)); //2d missing j coordinate
    MultiFab mf_Hzk(ba,dm,1,IntVect(NGROW,NGROW,NGROW-1)); //2d missing j coordinate
 
    for ( MFIter mfi(mf_cons, TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
        Array4<Real      > const& u = mf_u.array(mfi);
        Array4<Real      > const& v = mf_v.array(mfi);
 
        Array4<Real      > const& ru = mf_ru->array(mfi);
        Array4<Real      > const& rv = mf_rv->array(mfi);
 
        Array4<Real      > const& AK = mf_AK.array(mfi);
        Array4<Real      > const& DC = mf_DC.array(mfi);
 
        Array4<Real      > const& Hzk = mf_Hzk.array(mfi);
        Array4<Real      > const& Akv = mf_Akv->array(mfi);
 
        Array4<Real const> const& Hz  = mf_Hz->const_array(mfi);
 
        Array4<Real const> const& DU_avg1  = mf_DU_avg1->const_array(mfi);
        Array4<Real const> const& DV_avg1  = mf_DV_avg1->const_array(mfi);
 
        Array4<Real> const& DU_avg2  = mf_DU_avg2->array(mfi);
        Array4<Real> const& DV_avg2  = mf_DV_avg2->array(mfi);
 
        Array4<Real> const& ubar = mf_ubar->array(mfi);
        Array4<Real> const& vbar = mf_vbar->array(mfi);
 
        Array4<Real> const& Huon = mf_Huon->array(mfi);
        Array4<Real> const& Hvom = mf_Hvom->array(mfi);
 
        Array4<Real const> const& pm  = mf_pm->const_array(mfi);
        Array4<Real const> const& pn  = mf_pn->const_array(mfi);
 
        Box bx = mfi.tilebox();
        Box gbx2 = mfi.growntilebox(IntVect(NGROW,NGROW,0));
        Box gbx21 = mfi.growntilebox(IntVect(NGROW,NGROW,NGROW-1));
 
        Box xbx = mfi.nodaltilebox(0);
        Box ybx = mfi.nodaltilebox(1);
 
        Box gbx2D = gbx2;
        gbx2D.makeSlab(2,0);
 
        Box tbxp1 = bx;
        Box tbxp11 = bx;
        Box tbxp2 = bx;
        tbxp1.grow(IntVect(NGROW-1,NGROW-1,0));
        tbxp2.grow(IntVect(NGROW,NGROW,0));
        tbxp11.grow(IntVect(NGROW-1,NGROW-1,NGROW-1));
 
        FArrayBox fab_FC(gbx2,1,amrex::The_Async_Arena());
        FArrayBox fab_BC(gbx2,1,amrex::The_Async_Arena());
        FArrayBox fab_CF(gbx21,1,amrex::The_Async_Arena());
        FArrayBox fab_W(tbxp2,1,amrex::The_Async_Arena());
 
        auto FC = fab_FC.array();
        auto BC = fab_BC.array();
        auto CF = fab_CF.array();
 
        Real cff;
        if (iic==ntfirst) {
          cff=0.25*dt_lev;
        } else if (iic==ntfirst+1) {
          cff=0.25*dt_lev*3.0/2.0;
        } else {
          cff=0.25*dt_lev*23.0/12.0;
        }
 
        ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            u(i,j,k) += cff * (pm(i,j,0)+pm(i-1,j,0)) * (pn(i,j,0)+pn(i-1,j,0)) * ru(i,j,k,nrhs);
            u(i,j,k) *= 2.0 / (Hz(i-1,j,k) + Hz(i,j,k));
        });
 
        ParallelFor(ybx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            v(i,j,k) += cff * (pm(i,j,0)+pm(i,j-1,0)) * (pn(i,j,0)+pn(i,j-1,0)) * rv(i,j,k,nrhs);
            v(i,j,k) *= 2.0 / (Hz(i,j-1,k) + Hz(i,j,k));
        });
 
        // NOTE: DC is only used as scratch in vert_visc_3d -- no need to pass or return a value
        // NOTE: may not actually need to set these to zero
 
        // Reset to zero on the box on which they'll be used
        mf_DC[mfi].template setVal<RunOn::Device>(0.,xbx);
        fab_CF.template     setVal<RunOn::Device>(0.,xbx);
 
        vert_visc_3d(xbx,1,0,u,Hz,Hzk,AK,Akv,BC,DC,FC,CF,nnew,N,dt_lev);
 
        // Reset to zero on the box on which they'll be used
        mf_DC[mfi].template setVal<RunOn::Device>(0.,ybx);
        fab_CF.template     setVal<RunOn::Device>(0.,ybx);
 
        vert_visc_3d(ybx,0,1,v,Hz,Hzk,AK,Akv,BC,DC,FC,CF,nnew,N,dt_lev);
 
        // Reset to zero on the box on which they'll be used
        mf_DC[mfi].template setVal<RunOn::Device>(0.,xbx);
        fab_CF.template     setVal<RunOn::Device>(0.,xbx);
 
        vert_mean_3d(xbx,1,0,u,Hz,DU_avg1,DC,CF,pm,nnew,N);
 
        // Reset to zero on the box on which they'll be used
        mf_DC[mfi].template setVal<RunOn::Device>(0.,ybx);
        fab_CF.template     setVal<RunOn::Device>(0.,ybx);
 
        vert_mean_3d(ybx,0,1,v,Hz,DV_avg1,DC,CF,pn,nnew,N);
 
#if 0
        // Reset to zero on the box on which they'll be used
        mf_DC[mfi].template setVal<RunOn::Device>(0.,grow(xbx,IntVect(0,0,1)));
        fab_CF.template     setVal<RunOn::Device>(0.,grow(xbx,IntVect(0,0,1)));
 
        update_massflux_3d(xbx,1,0,u,ubar,Huon,Hz,pn,DU_avg1,DU_avg2,DC,FC,nnew);
 
        // Reset to zero on the box on which they'll be used
        mf_DC[mfi].template setVal<RunOn::Device>(0.,grow(ybx,IntVect(0,0,1)));
        fab_CF.template     setVal<RunOn::Device>(0.,grow(ybx,IntVect(0,0,1)));
 
        update_massflux_3d(ybx,0,1,v,vbar,Hvom,Hz,pm,DV_avg1,DV_avg2,DC,FC,nnew);
 
#else
        // Reset to zero on the box on which they'll be used
        fab_FC.template setVal<RunOn::Device>(0.,gbx2);
        mf_DC[mfi].template setVal<RunOn::Device>(0.,grow(gbx2,IntVect(0,0,1)));
        update_massflux_3d(gbx2,1,0,u,ubar,Huon,Hz,pn,DU_avg1,DU_avg2,DC,FC,nnew);
 
        // Reset to zero on the box on which they'll be used
        fab_FC.template     setVal<RunOn::Device>(0.,gbx2);
        mf_DC[mfi].template setVal<RunOn::Device>(0.,grow(gbx2,IntVect(0,0,1)));
        update_massflux_3d(gbx2,0,1,v,vbar,Hvom,Hz,pm,DV_avg1,DV_avg2,DC,FC,nnew);
#endif
    }
 
    // WE BELIEVE THESE VALUES SHOULD ALREADY BE FILLED
    // mf_Huon->FillBoundary(geom[lev].periodicity());
    // mf_Hvom->FillBoundary(geom[lev].periodicity());
 
    // ************************************************************************
    // This should fill both temp and salt with temp/salt currently in cons_old
    // ************************************************************************
 
    for ( MFIter mfi(mf_cons, TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
        Array4<Real> const& Hz  = mf_Hz->array(mfi);
 
        Array4<Real> const& Huon = mf_Huon->array(mfi);
        Array4<Real> const& Hvom = mf_Hvom->array(mfi);
 
        Array4<Real const> const& z_w = mf_z_w->const_array(mfi);
        Array4<Real const> const& h   = mf_h->const_array(mfi);
        Array4<Real const> const& pm  = mf_pm->const_array(mfi);
        Array4<Real const> const& pn  = mf_pn->const_array(mfi);
 
        Box bx = mfi.tilebox();
        Box gbx = mfi.growntilebox();
        Box gbx1 = mfi.growntilebox(IntVect(NGROW-1,NGROW-1,0));
        Box gbx2 = mfi.growntilebox(IntVect(NGROW,NGROW,0));
        Box gbx21 = mfi.growntilebox(IntVect(NGROW,NGROW,NGROW-1));
 
        Box tbxp1 = bx;
        Box tbxp11 = bx;
        Box tbxp2 = bx;
        tbxp1.grow(IntVect(NGROW-1,NGROW-1,0));
        tbxp2.grow(IntVect(NGROW,NGROW,0));
        tbxp11.grow(IntVect(NGROW-1,NGROW-1,NGROW-1));
 
        FArrayBox fab_FC(gbx2,1,amrex::The_Async_Arena());
        FArrayBox fab_BC(gbx2,1,amrex::The_Async_Arena());
        FArrayBox fab_CF(gbx21,1,amrex::The_Async_Arena());
        FArrayBox fab_W(tbxp2,1,amrex::The_Async_Arena());
 
        auto FC  = fab_FC.array();
        auto W   = fab_W.array();
 
        //
        //------------------------------------------------------------------------
        //  Vertically integrate horizontal mass flux divergence.
        //------------------------------------------------------------------------
        //
        //Should really use gbx3uneven
        //TODO: go over these boxes and compare to other spots where we do the same thing
        Box gbx1D = gbx1;
        gbx1D.makeSlab(2,0);
 
        //  Starting with zero vertical velocity at the bottom, integrate
        //  from the bottom (k=0) to the free-surface (k=N).  The w(:,:,N(ng))
        //  contains the vertical velocity at the free-surface, d(zeta)/d(t).
        //  Notice that barotropic mass flux divergence is not used directly.
        ParallelFor(gbx1D, [=] AMREX_GPU_DEVICE (int i, int j, int )
        {
            W(i,j,0) = - (Huon(i+1,j,0)-Huon(i,j,0)) - (Hvom(i,j+1,0)-Hvom(i,j,0));
 
            for (int k=1; k<=N; k++) {
                W(i,j,k) = W(i,j,k-1) - (Huon(i+1,j,k)-Huon(i,j,k)) - (Hvom(i,j+1,k)-Hvom(i,j,k));
            }
        });
 
        //  Starting with zero vertical velocity at the bottom, integrate
        //  from the bottom (k=0) to the free-surface (k=N).  The w(:,:,N(ng))
        //  contains the vertical velocity at the free-surface, d(zeta)/d(t).
        //  Notice that barotropic mass flux divergence is not used directly.
        //
        ParallelFor(gbx1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            Real wrk_ij = W(i,j,N) / (z_w(i,j,N)+h(i,j,0,0));
 
            if(k!=N) {
                W(i,j,k) -=  wrk_ij * (z_w(i,j,k)+h(i,j,0,0));
            }
        });
 
        ParallelFor(makeSlab(gbx1,2,N), [=] AMREX_GPU_DEVICE (int i, int j, int)
        {
            W(i,j,N) = 0.0;
        });
 
        //
        //-----------------------------------------------------------------------
        // rhs_t_3d
        //-----------------------------------------------------------------------
        //
        for (int i_comp=0; i_comp < NCONS; i_comp++)
        {
            Array4<Real> const& sstore = mf_sstore->array(mfi, i_comp);
            rhs_t_3d(bx, gbx, mf_cons.array(mfi,i_comp), sstore, Huon, Hvom,
                     Hz, pn, pm, W, FC, nrhs, nnew, N,dt_lev);
        }
 
    } // mfi
 
    FillPatch(lev, t_new[lev], mf_cons, cons_new, BdyVars::t);
 
    for ( MFIter mfi(mf_cons, TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
        Array4<Real> const& AK = mf_AK.array(mfi);
        Array4<Real> const& DC = mf_DC.array(mfi);
 
        Array4<Real> const& Hzk = mf_Hzk.array(mfi);
        Array4<Real const> const& Hz  = mf_Hz->const_array(mfi);
 
        Box bx = mfi.tilebox();
 
        // Copy the tilebox
        Box tbxp1 = bx;
        Box tbxp11 = bx;
        Box tbxp2 = bx;
        Box tbxp21 = bx;
        //make only gbx be grown to match multifabs
        tbxp21.grow(IntVect(NGROW,NGROW,NGROW-1));
        tbxp2.grow(IntVect(NGROW,NGROW,0));
        tbxp1.grow(IntVect(NGROW-1,NGROW-1,0));
        tbxp11.grow(IntVect(NGROW-1,NGROW-1,NGROW-1));
 
        FArrayBox fab_FC(tbxp2,1,amrex::The_Async_Arena());
        FArrayBox fab_BC(tbxp2,1,amrex::The_Async_Arena());
        FArrayBox fab_CF(tbxp21,1,amrex::The_Async_Arena());
        FArrayBox fab_W(tbxp2,1,amrex::The_Async_Arena());
 
        auto FC = fab_FC.array();
        auto BC = fab_BC.array();
        auto CF = fab_CF.array();
 
        for (int i_comp=0; i_comp < NCONS; i_comp++) {
            vert_visc_3d(bx,0,0,mf_cons.array(mfi,i_comp),Hz,Hzk,
                    AK,mf_Akt->array(mfi,i_comp),BC,DC,FC,CF,nnew,N,dt_lev);
        }
    } // MFiter
}

advance_3d_ml()

void REMORA::advance_3d_ml	(	int	lev,
		amrex::Real	dt_lev
	)

3D advance on a single level

{
    // Fill in three ways: 1) interpolate from coarse grid if lev > 0; 2) fill from physical boundaries;
    //                     3) fine-fine fill of ghost cells with FillBoundary call
    FillPatch(lev, t_new[lev], *cons_new[lev], cons_new, BdyVars::t);
    FillPatch(lev, t_new[lev], *xvel_new[lev], xvel_new, BdyVars::u);
    FillPatch(lev, t_new[lev], *yvel_new[lev], yvel_new, BdyVars::v);
    FillPatch(lev, t_new[lev], *zvel_new[lev], zvel_new, BdyVars::null);
 
    FillPatch(lev, t_new[lev], *vec_sstore[lev], GetVecOfPtrs(vec_sstore), BdyVars::t);
 
    auto N = Geom(lev).Domain().size()[2]-1; // Number of vertical "levs" aka, NZ
 
    advance_3d(lev, *cons_new[lev], *xvel_new[lev], *yvel_new[lev],
               vec_sstore[lev].get(),
               vec_ru[lev].get(), vec_rv[lev].get(),
               vec_DU_avg1[lev], vec_DU_avg2[lev],
               vec_DV_avg1[lev], vec_DV_avg2[lev],
               vec_ubar[lev],  vec_vbar[lev],
               vec_Akv[lev], vec_Akt[lev], vec_Hz[lev], vec_Huon[lev], vec_Hvom[lev],
               vec_z_w[lev], vec_hOfTheConfusingName[lev].get(),
               vec_pm[lev].get(), vec_pn[lev].get(),
               N, dt_lev);
 
    FillPatch(lev, t_new[lev], *vec_ubar[lev], GetVecOfPtrs(vec_ubar), BdyVars::ubar);
    FillPatch(lev, t_new[lev], *vec_vbar[lev], GetVecOfPtrs(vec_vbar), BdyVars::vbar);
    FillPatch(lev, t_new[lev], *vec_sstore[lev], GetVecOfPtrs(vec_sstore), BdyVars::t);
 
    // Fill in three ways: 1) interpolate from coarse grid if lev > 0; 2) fill from physical boundaries;
    //                     3) fine-fine fill of ghost cells with FillBoundary call
    // Note that we need the fine-fine and physical bc's in order to correctly move the particles
    FillPatch(lev, t_new[lev], *cons_new[lev], cons_new, BdyVars::t);
    FillPatch(lev, t_new[lev], *xvel_new[lev], xvel_new, BdyVars::u);
    FillPatch(lev, t_new[lev], *yvel_new[lev], yvel_new, BdyVars::v);
    FillPatch(lev, t_new[lev], *zvel_new[lev], zvel_new, BdyVars::null);
 
#ifdef REMORA_USE_PARTICLES
    //***************************************************
    //Advance particles
    //***************************************************
    particleData.advance_particles(lev, dt_lev, xvel_new[lev], yvel_new[lev], zvel_new[lev], vec_z_phys_nd);
#endif
}

AverageDown()

void REMORA::AverageDown ( )

private

set covered coarse cells to be the average of overlying fine cells

{
    for (int lev = finest_level-1; lev >= 0; --lev)
    {
        AverageDownTo(lev);
    }
}

AverageDownTo()

void REMORA::AverageDownTo ( int  crse_lev )

private

more flexible version of AverageDown() that lets you average down across multiple levels

{
    average_down(*cons_new[crse_lev+1], *cons_new[crse_lev],
                 0, cons_new[crse_lev]->nComp(), refRatio(crse_lev));
 
    Array<MultiFab*,AMREX_SPACEDIM>  faces_crse;
    Array<MultiFab*,AMREX_SPACEDIM>  faces_fine;
    faces_crse[0] = xvel_new[crse_lev];
    faces_crse[1] = yvel_new[crse_lev];
    faces_crse[2] = zvel_new[crse_lev];
 
    faces_fine[0] = xvel_new[crse_lev+1];
    faces_fine[1] = yvel_new[crse_lev+1];
    faces_fine[2] = zvel_new[crse_lev+1];
 
    average_down_faces(GetArrOfConstPtrs(faces_fine), faces_crse,
                       refRatio(crse_lev),geom[crse_lev]);
}

build_fine_mask()

MultiFab & REMORA::build_fine_mask

(

int

lev

)

Make mask to zero out covered cells

{
    // Mask for zeroing covered cells
    AMREX_ASSERT(level > 0);
 
    const BoxArray& cba = grids[level-1];
    const DistributionMapping& cdm = dmap[level-1];
 
    // TODO -- we should make a vector of these a member of REMORA class
    fine_mask.define(cba, cdm, 1, 0, MFInfo());
    fine_mask.setVal(1.0);
 
    BoxArray fba = grids[level];
    iMultiFab ifine_mask = makeFineMask(cba, cdm, fba, ref_ratio[level-1], 1, 0);
 
    const auto  fma =  fine_mask.arrays();
    const auto ifma = ifine_mask.arrays();
    ParallelFor(fine_mask, [=] AMREX_GPU_DEVICE(int bno, int i, int j, int k) noexcept
    {
        fma[bno](i,j,k) = ifma[bno](i,j,k);
    });
 
    Gpu::synchronize();
 
    return fine_mask;
}

ClearLevel()

void REMORA::ClearLevel ( int  lev )

overridevirtual

Delete level data Overrides the pure virtual function in AmrCore

{
    delete cons_new[lev]; delete xvel_new[lev];  delete yvel_new[lev];  delete zvel_new[lev];
    delete cons_old[lev]; delete xvel_old[lev];  delete yvel_old[lev];  delete zvel_old[lev];
}

ComputeDt()

void REMORA::ComputeDt ( )

private

a wrapper for estTimeStep()

{
    Vector<Real> dt_tmp(finest_level+1);
 
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        dt_tmp[lev] = estTimeStep(lev);
    }
 
    ParallelDescriptor::ReduceRealMin(&dt_tmp[0], dt_tmp.size());
 
    Real dt_0 = dt_tmp[0];
    int n_factor = 1;
    for (int lev = 0; lev <= finest_level; ++lev) {
        dt_tmp[lev] = amrex::min(dt_tmp[lev], change_max*dt[lev]);
        n_factor *= nsubsteps[lev];
        dt_0 = amrex::min(dt_0, n_factor*dt_tmp[lev]);
    }
 
    // Limit dt's by the value of stop_time.
    const Real eps = 1.e-3*dt_0;
    if (t_new[0] + dt_0 > stop_time - eps) {
        dt_0 = stop_time - t_new[0];
    }
 
    dt[0] = dt_0;
    for (int lev = 1; lev <= finest_level; ++lev) {
        dt[lev] = dt[lev-1] / nsubsteps[lev];
    }
}

ComputeGhostCells()

AMREX_FORCE_INLINE int REMORA::ComputeGhostCells ( const int &  spatial_order )

inlineprivate

                                                {
      int nGhostCells;
 
      switch (spatial_order) {
        case 2:
          nGhostCells = 2; // We need this many to compute the eddy viscosity in the ghost cells
          break;
        case 3:
          nGhostCells = 2;
          break;
        case 4:
          nGhostCells = 2;
          break;
        case 5:
          nGhostCells = 3;
          break;
        case 6:
          nGhostCells = 3;
          break;
        default:
          amrex::Error("Must specify spatial order to be 2,3,4,5 or 6");
      }
 
      return nGhostCells;
    }

coriolis()

void REMORA::coriolis	(	const amrex::Box &	xbx,
		const amrex::Box &	ybx,
		const amrex::Array4< amrex::Real const > &	uold,
		const amrex::Array4< amrex::Real const > &	vold,
		const amrex::Array4< amrex::Real > &	ru,
		const amrex::Array4< amrex::Real > &	rv,
		const amrex::Array4< amrex::Real const > &	Hz,
		const amrex::Array4< amrex::Real const > &	fomn,
		int	nrhs,
		int	nr
	)

{
    //
    //-----------------------------------------------------------------------
    //  Add in Coriolis terms.
    //-----------------------------------------------------------------------
    //
 
    ParallelFor(xbx,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real UFx_i   = 0.5 * Hz(i  ,j,k) * fomn(i  ,j,0) * (vold(i  ,j,k,nrhs)+vold(i  ,j+1,k,nrhs));
        Real UFx_im1 = 0.5 * Hz(i-1,j,k) * fomn(i-1,j,0) * (vold(i-1,j,k,nrhs)+vold(i-1,j+1,k,nrhs));
        ru(i,j,k,nr) += 0.5*(UFx_i + UFx_im1);
    });
 
    ParallelFor(ybx,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real VFe_j   = 0.5 * Hz(i,j  ,k) * fomn(i,j  ,0) * (uold(i,j  ,k,nrhs)+uold(i+1,j  ,k,nrhs));
        Real VFe_jm1 = 0.5 * Hz(i,j-1,k) * fomn(i,j-1,0) * (uold(i,j-1,k,nrhs)+uold(i+1,j-1,k,nrhs));
        rv(i,j,k,nr) -= 0.5*(VFe_j + VFe_jm1);
    });
}

DataLog()

AMREX_FORCE_INLINE std::ostream& REMORA::DataLog ( int  i )

inlineprivate

    {
        return *datalog[i];
    }

DataLogName()

const std::string REMORA::DataLogName ( int  i ) const

inlineprivatenoexcept

The filename of the ith datalog file.

{ return datalogname[i]; }

ErrorEst()

void REMORA::ErrorEst ( int  lev, amrex::TagBoxArray &  tags, amrex::Real  time, int  ngrow  )

overridevirtual

Tag cells for refinement

Function to tag cells for refinement – this overrides the pure virtual function in AmrCore

Parameters

[in]	level	level of refinement (0 is coarsest leve)
[out]	tags	array of tagged cells
[in]	time	current time

{
    const int clearval = TagBox::CLEAR;
    const int   tagval = TagBox::SET;
    for (int j=0; j < ref_tags.size(); ++j)
    {
        std::unique_ptr<MultiFab> mf;
 
        // This allows dynamic refinement based on the value of the scalar
        if (ref_tags[j].Field() == "scalar")
        {
            mf = std::make_unique<MultiFab>(grids[level], dmap[level], 1, 0);
            MultiFab::Copy(*mf,*cons_new[level],Scalar_comp,0,1,0);
        }
 
        ref_tags[j](tags,mf.get(),clearval,tagval,time,level,geom[level]);
  }
}

estTimeStep()

Real REMORA::estTimeStep

(

int

lev

)

const

compute dt from CFL considerations

{
  BL_PROFILE("REMORA::estTimeStep()");
 
  amrex::Real estdt_lowM = 1.e20;
 
  auto const dxinv = geom[level].InvCellSizeArray();
 
  MultiFab ccvel(grids[level],dmap[level],3,0);
 
  average_face_to_cellcenter(ccvel,0,
      Array<const MultiFab*,3>{xvel_new[level], yvel_new[level], zvel_new[level]});
 
   Real estdt_lowM_inv = amrex::ReduceMax(ccvel, 0,
       [=] AMREX_GPU_HOST_DEVICE (Box const& b,
                                  Array4<Real const> const& u) -> Real
       {
           Real new_lm_dt = -1.e100;
           amrex::Loop(b, [=,&new_lm_dt] (int i, int j, int k) noexcept
           {
               new_lm_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0)))*dxinv[0]),
                                      ((amrex::Math::abs(u(i,j,k,1)))*dxinv[1]),
                                      ((amrex::Math::abs(u(i,j,k,2)))*dxinv[2]), new_lm_dt);
           });
           return new_lm_dt;
       });
 
   ParallelDescriptor::ReduceRealMax(estdt_lowM_inv);
   if (estdt_lowM_inv > 0.0_rt)
       estdt_lowM = cfl / estdt_lowM_inv;;
 
  if (verbose) {
    if (fixed_dt <= 0.0) {
        amrex::Print() << "Using cfl = " << cfl << std::endl;
        if (estdt_lowM_inv > 0.0_rt) {
            amrex::Print() << "Slow  dt at level " << level << ":  " << estdt_lowM << std::endl;
        } else {
            amrex::Print() << "Slow  dt at level " << level << ": undefined " << std::endl;
        }
    }
    if (fixed_dt > 0.0) {
        amrex::Print() << "Based on cfl of 1.0 " << std::endl;
        if (estdt_lowM_inv > 0.0_rt) {
            amrex::Print() << "Slow  dt at level " << level << " would be:  " << estdt_lowM/cfl << std::endl;
        } else {
            amrex::Print() << "Slow  dt at level " << level << " would be undefined " << std::endl;
        }
        amrex::Print() << "Fixed dt at level " << level << "       is:  " << fixed_dt << std::endl;
    }
  }
 
  if (fixed_dt > 0.0) {
    return fixed_dt;
  } else {
        return estdt_lowM;
  }
}

Evolve()

void REMORA::Evolve

(

)

Advance solution to final time

{
    Real cur_time = t_new[0];
 
    // Take one coarse timestep by calling timeStep -- which recursively calls timeStep
    //      for finer levels (with or without subcycling)
    for (int step = istep[0]; step < max_step && cur_time < stop_time; ++step)
    {
        amrex::Print() << "\nCoarse STEP " << step+1 << " starts ..." << std::endl;
 
        ComputeDt();
 
        int lev = 0;
        int iteration = 1;
 
        if (max_level == 0) {
            timeStep(lev, cur_time, iteration);
        }
        else {
            timeStepML(cur_time, iteration);
        }
 
        cur_time  += dt[0];
 
        amrex::Print() << "Coarse STEP " << step+1 << " ends." << " TIME = " << cur_time
                       << " DT = " << dt[0]  << std::endl;
 
        if (plot_int_1 > 0 && (step+1) % plot_int_1 == 0) {
            last_plot_file_step_1 = step+1;
            if (plotfile_type == PlotfileType::amrex)
                WritePlotFile(1,plot_var_names_1);
#ifdef REMORA_USE_NETCDF
            else if (plotfile_type == PlotfileType::netcdf)
                WriteNCPlotFile(step);
#endif
        }
        if (plot_int_2 > 0 && (step+1) % plot_int_2 == 0) {
            last_plot_file_step_2 = step+1;
            if (plotfile_type == PlotfileType::amrex)
                WritePlotFile(2,plot_var_names_2);
#ifdef REMORA_USE_NETCDF
            else if (plotfile_type == PlotfileType::netcdf)
                WriteNCPlotFile(step);
#endif
        }
 
        if (check_int > 0 && (step+1) % check_int == 0) {
            last_check_file_step = step+1;
            WriteCheckpointFile();
        }
 
        post_timestep(step, cur_time, dt[0]);
 
#ifdef AMREX_MEM_PROFILING
        {
            std::ostringstream ss;
            ss << "[STEP " << step+1 << "]";
            MemProfiler::report(ss.str());
        }
#endif
 
        if (cur_time >= stop_time - 1.e-6*dt[0]) break;
    }
 
    if (plot_int_1 > 0 && istep[0] > last_plot_file_step_1) {
        if (plotfile_type == PlotfileType::amrex)
            WritePlotFile(1,plot_var_names_1);
#ifdef REMORA_USE_NETCDF
        if (plotfile_type == PlotfileType::netcdf)
            WriteNCPlotFile(istep[0]);
#endif
    }
    if (plot_int_2 > 0 && istep[0] > last_plot_file_step_2) {
        if (plotfile_type == PlotfileType::amrex)
            WritePlotFile(2,plot_var_names_2);
#ifdef REMORA_USE_NETCDF
        else if (plotfile_type == PlotfileType::netcdf)
            WriteNCPlotFile(istep[0]);
#endif
    }
 
    if (check_int > 0 && istep[0] > last_check_file_step) {
        WriteCheckpointFile();
    }
 
}

Referenced by main().

Here is the caller graph for this function:

fill_from_bdyfiles()

void REMORA::fill_from_bdyfiles	(	amrex::MultiFab &	mf_to_fill,
		const amrex::Real	time,
		const int	bdy_var_type
	)

fill_rhs()

void REMORA::fill_rhs ( amrex::MultiFab &  rhs_mf, const amrex::MultiFab &  state_mf, const amrex::Real  time, const amrex::Geometry &  geom  )

private

FillCoarsePatch()

void REMORA::FillCoarsePatch ( int  lev, amrex::Real  time, amrex::MultiFab *  mf_fine, amrex::MultiFab *  mf_crse  )

private

fill an entire multifab by interpolating from the coarser level this comes into play when a new level of refinement appears

{
    BL_PROFILE_VAR("FillCoarsePatch()",FillCoarsePatch);
    AMREX_ASSERT(lev > 0);
 
    int bccomp = 0;
    int  icomp = 0;
    int  ncomp = 1;
    amrex::Interpolater* mapper = nullptr;
 
    Box box_mf = ((*mf_to_fill)[0]).box();
    if (box_mf.ixType() == IndexType(IntVect(0,0,0)))
    {
        bccomp = 0;
        mapper = &cell_cons_interp;
        ncomp = NCONS;
    }
    else if (box_mf.ixType() == IndexType(IntVect(1,0,0)))
    {
        bccomp = BCVars::xvel_bc;
        mapper = &face_linear_interp;
    }
    else if (box_mf.ixType() == IndexType(IntVect(0,1,0)))
    {
        bccomp = BCVars::yvel_bc;
        mapper = &face_linear_interp;
    }
    else if (box_mf.ixType() == IndexType(IntVect(0,0,1)))
    {
        bccomp = BCVars::zvel_bc;
        mapper = &face_linear_interp;
    } else {
          amrex::Abort("Dont recognize this box type in REMORA_FillPatch");
    }
 
#if 0
    TimeInterpolatedData cdata = GetDataAtTime(lev-1, time);
    TimeInterpolatedData fdata = GetDataAtTime(lev  , time);
    Vector<Real> ctime = {cdata.get_time()};
 
    Vector<MultiFab*> cmf = {mf_crse};
    Vector<Real> ctime = {time};
 
    REMORAPhysBCFunct cphysbc(lev-1,geom[lev-1],
                             domain_bcs_type,domain_bcs_type_d,
                             cdata,
                             m_bc_extdir_vals
#ifdef REMORA_USE_NETCDF
                            ,ic_bc_type,bdy_data_xlo,bdy_data_xhi,
                             bdy_data_ylo,bdy_data_yhi,bdy_time_interval
#endif
                            );
    REMORAPhysBCFunct fphysbc(lev,geom[lev],
                             domain_bcs_type,domain_bcs_type_d,
                             fdata,
                             m_bc_extdir_vals
#ifdef REMORA_USE_NETCDF
                            ,ic_bc_type,bdy_data_xlo,bdy_data_xhi,
                             bdy_data_ylo,bdy_data_yhi,bdy_time_interval
#endif
                            );
#endif
 
//  amrex::InterpFromCoarseLevel(mf, time, *cmf[0], 0, icomp, ncomp, geom[lev-1], geom[lev],
//                               cphysbc, 0, fphysbc, 0, refRatio(lev-1),
//                               mapper, domain_bcs_type, bccomp);
    amrex::InterpFromCoarseLevel(*mf_to_fill, time, mf_crse[lev-1], 0, icomp, ncomp, geom[lev-1], geom[lev],
                                 null_bc, 0, null_bc, 0, refRatio(lev-1),
                                 mapper, domain_bcs_type, bccomp);
}

Here is the call graph for this function:

FillPatch()

void REMORA::FillPatch	(	int	lev,
		amrex::Real	time,
		amrex::MultiFab &	mf_to_be_filled,
		amrex::Vector< amrex::MultiFab * > const &	mfs,
		const int	bdy_var_type = BdyVars::null
	)

{
    BL_PROFILE_VAR("REMORA::FillPatch()",REMORA_FillPatch);
    int bccomp;
    amrex::Interpolater* mapper = nullptr;
 
    const int icomp = 0;
    const int ncomp = mf_to_fill.nComp();
 
    Box mf_box(mf_to_fill.boxArray()[0]);
    if (mf_box.ixType() == IndexType(IntVect(0,0,0)))
    {
        bccomp = 0;
        mapper = &cell_cons_interp;
    }
    else if (mf_box.ixType() == IndexType(IntVect(1,0,0)))
    {
        bccomp = BCVars::xvel_bc;
        mapper = &face_linear_interp;
    }
    else if (mf_box.ixType() == IndexType(IntVect(0,1,0)))
    {
        bccomp = BCVars::yvel_bc;
        mapper = &face_linear_interp;
    }
    else if (mf_box.ixType() == IndexType(IntVect(0,0,1)))
    {
        bccomp = BCVars::zvel_bc;
        mapper = &face_linear_interp;
    }
    else
    {
        amrex::Abort("Dont recognize this box type in REMORA_FillPatch");
    }
 
    if (lev == 0)
    {
        Vector<MultiFab*> fmf = {mfs[lev], mfs[lev]};
        Vector<Real> ftime    = {t_old[lev], t_new[lev]};
        amrex::FillPatchSingleLevel(mf_to_fill, time, fmf, ftime, icomp, icomp, ncomp,
                                    geom[lev], null_bc, bccomp);
    }
    else
    {
        Vector<MultiFab*> fmf = {mfs[lev], mfs[lev]};
        Vector<Real> ftime    = {t_old[lev], t_new[lev]};
        Vector<MultiFab*> cmf = {mfs[lev-1], mfs[lev-1]};
        Vector<Real> ctime    = {t_old[lev-1], t_new[lev-1]};
 
        amrex::FillPatchTwoLevels(mf_to_fill, time, cmf, ctime, fmf, ftime,
                                  0, icomp, ncomp, geom[lev-1], geom[lev],
                                  null_bc, bccomp, null_bc, bccomp, refRatio(lev-1),
                                  mapper, domain_bcs_type, bccomp);
    } // lev > 0
 
    // ***************************************************************************
    // Physical bc's at domain boundary
    // ***************************************************************************
 
#ifdef REMORA_USE_NETCDF
    // Fill the data which is stored in the boundary data read from netcdf files
    if ( (solverChoice.ic_bc_type == IC_BC_Type::Real) && (lev==0) &&
         (bdy_var_type != BdyVars::null) )
    {
        fill_from_bdyfiles (mf_to_fill,time,bdy_var_type);
    }
#endif
 
    // Enforce physical boundary conditions
    (*physbcs[lev])(mf_to_fill,0,ncomp,mf_to_fill.nGrowVect(),time,bccomp);
 
    // Also enforce free-slip at top boundary (on xvel or yvel)
    if ( (mf_box.ixType() == IndexType(IntVect(1,0,0))) ||
         (mf_box.ixType() == IndexType(IntVect(0,1,0))) )
    {
        int khi = geom[lev].Domain().bigEnd(2);
        for (MFIter mfi(mf_to_fill); mfi.isValid(); ++mfi)
        {
            Box gbx  = mfi.growntilebox(); // Note this is face-centered since vel is
            gbx.setSmall(2,khi+1);
            if (gbx.ok()) {
                Array4<Real> vel_arr = mf_to_fill.array(mfi);
                ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
                {
                    vel_arr(i,j,k) = vel_arr(i,j,khi);
                });
            }
        }
    }
}

getAdvFluxReg()

AMREX_FORCE_INLINE amrex::YAFluxRegister* REMORA::getAdvFluxReg ( int  lev )

inlineprivate

    {
        return advflux_reg[lev];
    }

getCPUTime()

amrex::Real REMORA::getCPUTime ( ) const

inlineprivate

    {
      int numCores = amrex::ParallelDescriptor::NProcs();
#ifdef _OPENMP
      numCores = numCores * omp_get_max_threads();
#endif
 
      amrex::Real T =
        numCores * (amrex::ParallelDescriptor::second() - startCPUTime) +
        previousCPUTimeUsed;
 
      return T;
    }

Referenced by writeJobInfo().

Here is the caller graph for this function:

GetDataAtTime()

TimeInterpolatedData REMORA::GetDataAtTime ( int  lev, amrex::Real  time  )

private

utility to copy in data from old and/or new state into another multifab

{
    BL_PROFILE_VAR("GetDataAtTime()",GetDataAtTime);
    TimeInterpolatedData data;
 
// HACK HACK HACK
#if 0
    const Real teps = (t_new[lev] - t_old[lev]) * 1.e-3;
 
    if (time > t_new[lev] - teps && time < t_new[lev] + teps)
    {
        for (int i = 0; i < Vars::NumTypes; ++i) {
            data.add_var(&vars_new[lev][i], data.non_owning);
        }
        data.set_time(t_new[lev]);
    }
    else if (time > t_old[lev] - teps && time < t_old[lev] + teps)
    {
        for (int i = 0; i < Vars::NumTypes; ++i) {
            data.add_var(&vars_old[lev][i], data.non_owning);
        }
        data.set_time(t_old[lev]);
    }
    else if (time > t_old[lev] && time < t_new[lev])
    {
        // do first order interpolation in time between [t_old[lev], t_new[lev]]
        // time interpolation includes the ghost cells
        for (int i = 0; i < Vars::NumTypes; ++i) {
            MultiFab* mf_tmp = new MultiFab(vars_new[lev][i].boxArray(),
                                            vars_new[lev][i].DistributionMap(),
                                            vars_new[lev][i].nComp(), vars_new[lev][i].nGrowVect());
            mf_tmp->setVal(0.0_rt);
 
            const Real dt_fraction = (time - t_old[lev]) / (t_new[lev] - t_old[lev]);
            MultiFab::Saxpy(*mf_tmp, 1.0_rt - dt_fraction, vars_old[lev][i], 0, 0, mf_tmp->nComp(), mf_tmp->nGrowVect());
            MultiFab::Saxpy(*mf_tmp,          dt_fraction, vars_new[lev][i], 0, 0, mf_tmp->nComp(), mf_tmp->nGrowVect());
 
            data.add_var(mf_tmp, data.owning);
        }
        data.set_time(time);
    }
    else
    {
        amrex::Error("Requested data at a time outside the interval [t_old, t_new]");
    }
 
    // We need to make sure to fill these before we compute the viscosity
    for (int i = 0; i < Vars::NumTypes; ++i) {
        data.get_var(Vars::xvel).FillBoundary(geom[lev].periodicity());
        data.get_var(Vars::yvel).FillBoundary(geom[lev].periodicity());
        data.get_var(Vars::zvel).FillBoundary(geom[lev].periodicity());
        data.get_var(Vars::cons).FillBoundary(geom[lev].periodicity());
    }
#endif
    return data;
}

Here is the call graph for this function:

GotoNextLine()

void REMORA::GotoNextLine ( std::istream &  is )

staticprivate

{
    constexpr std::streamsize bl_ignore_max { 100000 };
    is.ignore(bl_ignore_max, '\n');
}

init1DArrays()

void REMORA::init1DArrays ( )

private

init_bcs()

void REMORA::init_bcs ( )

private

{
    auto f = [this] (std::string const& bcid, Orientation ori)
    {
        // These are simply defaults for Dirichlet faces -- they should be over-written below
        m_bc_extdir_vals[BCVars::Temp_bc_comp  ][ori] = 1.e19;
        m_bc_extdir_vals[BCVars::Salt_bc_comp  ][ori] = 1.e20;
        m_bc_extdir_vals[BCVars::Scalar_bc_comp][ori] = 1.e21;
 
        m_bc_extdir_vals[BCVars::xvel_bc][ori] = 0.0; // default
        m_bc_extdir_vals[BCVars::yvel_bc][ori] = 0.0;
        m_bc_extdir_vals[BCVars::zvel_bc][ori] = 0.0;
 
        ParmParse pp(bcid);
        std::string bc_type_in = "null";
        pp.query("type", bc_type_in);
        std::string bc_type = amrex::toLower(bc_type_in);
 
        if (bc_type == "symmetry")
        {
            phys_bc_type[ori] = REMORA_BC::symmetry;
            domain_bc_type[ori] = "Symmetry";
        }
        else if (bc_type == "outflow")
        {
            phys_bc_type[ori] = REMORA_BC::outflow;
            domain_bc_type[ori] = "Outflow";
        }
        else if (bc_type == "inflow")
        {
            phys_bc_type[ori] = REMORA_BC::inflow;
            domain_bc_type[ori] = "Inflow";
 
            std::vector<Real> v;
            pp.getarr("velocity", v, 0, AMREX_SPACEDIM);
            m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
            m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
            m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
 
            Real scalar_in = 0.;
            if (pp.query("scalar", scalar_in))
            m_bc_extdir_vals[BCVars::Scalar_bc_comp][ori] = scalar_in;
        }
        else if (bc_type == "noslipwall")
        {
            phys_bc_type[ori] = REMORA_BC::no_slip_wall;
            domain_bc_type[ori] = "NoSlipWall";
 
            std::vector<Real> v;
 
            // The values of m_bc_extdir_vals default to 0.
            // But if we find "velocity" in the inputs file, use those values instead.
            if (pp.queryarr("velocity", v, 0, AMREX_SPACEDIM))
            {
                v[ori.coordDir()] = 0.0;
                m_bc_extdir_vals[BCVars::xvel_bc][ori] = v[0];
                m_bc_extdir_vals[BCVars::yvel_bc][ori] = v[1];
                m_bc_extdir_vals[BCVars::zvel_bc][ori] = v[2];
            }
        }
        else if (bc_type == "slipwall")
        {
            phys_bc_type[ori] = REMORA_BC::slip_wall;
            domain_bc_type[ori] = "SlipWall";
        }
        else
        {
            phys_bc_type[ori] = REMORA_BC::undefined;
        }
 
        if (geom[0].isPeriodic(ori.coordDir()))
        {
            domain_bc_type[ori] = "Periodic";
            if (phys_bc_type[ori] == REMORA_BC::undefined)
            {
                phys_bc_type[ori] = REMORA_BC::periodic;
            } else {
                amrex::Abort("Wrong BC type for periodic boundary");
            }
        }
 
        if (phys_bc_type[ori] == REMORA_BC::undefined)
        {
             amrex::Print() << "BC Type specified for face " << bcid << " is " << bc_type_in << std::endl;
             amrex::Abort("This BC type is unknown");
        }
 
        if ((bcid == "xlo" || bcid == "xhi" ||
             bcid == "ylo" || bcid == "yhi") &&
            solverChoice.ic_bc_type == IC_BC_Type::Real &&
            phys_bc_type[ori] != REMORA_BC::outflow)
        {
            amrex::Abort("BC type must be outflow in x and y when reading BCs from file");
        }
    };
 
    f("xlo", Orientation(Direction::x,Orientation::low));
    f("xhi", Orientation(Direction::x,Orientation::high));
    f("ylo", Orientation(Direction::y,Orientation::low));
    f("yhi", Orientation(Direction::y,Orientation::high));
    f("zlo", Orientation(Direction::z,Orientation::low));
    f("zhi", Orientation(Direction::z,Orientation::high));
 
    // *****************************************************************************
    //
    // Here we translate the physical boundary conditions -- one type per face --
    //     into logical boundary conditions for each velocity component
    //
    // *****************************************************************************
    {
        domain_bcs_type.resize(AMREX_SPACEDIM+NCONS);
        domain_bcs_type_d.resize(AMREX_SPACEDIM+NCONS);
 
        for (OrientationIter oit; oit; ++oit) {
            Orientation ori = oit();
            int dir = ori.coordDir();
            Orientation::Side side = ori.faceDir();
            auto const bct = phys_bc_type[ori];
            if ( bct == REMORA_BC::symmetry )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, REMORABCType::reflect_even);
                    domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, REMORABCType::reflect_odd);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, REMORABCType::reflect_even);
                    domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, REMORABCType::reflect_odd);
                }
            }
            else if (bct == REMORA_BC::outflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, REMORABCType::foextrap);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, REMORABCType::foextrap);
                }
            }
            else if (bct == REMORA_BC::inflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, REMORABCType::ext_dir);
                    }
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++) {
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, REMORABCType::ext_dir);
                    }
                }
            }
            else if (bct == REMORA_BC::no_slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, REMORABCType::ext_dir);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, REMORABCType::ext_dir);
                }
            }
            else if (bct == REMORA_BC::slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, REMORABCType::foextrap);
                    // Only normal direction has ext_dir
                    domain_bcs_type[BCVars::xvel_bc+dir].setLo(dir, REMORABCType::ext_dir);
 
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, REMORABCType::foextrap);
                    // Only normal direction has ext_dir
                    domain_bcs_type[BCVars::xvel_bc+dir].setHi(dir, REMORABCType::ext_dir);
                }
            }
            else if (bct == REMORA_BC::periodic)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setLo(dir, REMORABCType::int_dir);
                } else {
                    for (int i = 0; i < AMREX_SPACEDIM; i++)
                        domain_bcs_type[BCVars::xvel_bc+i].setHi(dir, REMORABCType::int_dir);
                }
            }
        }
    }
 
    // *****************************************************************************
    //
    // Here we translate the physical boundary conditions -- one type per face --
    //     into logical boundary conditions for each cell-centered variable
    //
    // *****************************************************************************
    {
        for (OrientationIter oit; oit; ++oit) {
            Orientation ori = oit();
            int dir = ori.coordDir();
            Orientation::Side side = ori.faceDir();
            auto const bct = phys_bc_type[ori];
            if ( bct == REMORA_BC::symmetry )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NCONS; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, REMORABCType::reflect_even);
                } else {
                    for (int i = 0; i < NCONS; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, REMORABCType::reflect_even);
                }
            }
            else if ( bct == REMORA_BC::outflow )
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NCONS; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, REMORABCType::foextrap);
                } else {
                    for (int i = 0; i < NCONS; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, REMORABCType::foextrap);
                }
            }
            else if ( bct == REMORA_BC::no_slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NCONS; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, REMORABCType::foextrap);
                } else {
                    for (int i = 0; i < NCONS; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, REMORABCType::foextrap);
                }
            }
            else if (bct == REMORA_BC::slip_wall)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NCONS; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, REMORABCType::foextrap);
                } else {
                    for (int i = 0; i < NCONS; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, REMORABCType::foextrap);
                }
            }
            else if (bct == REMORA_BC::inflow)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NCONS; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, REMORABCType::ext_dir);
                    }
                } else {
                    for (int i = 0; i < NCONS; i++) {
                        domain_bcs_type[BCVars::cons_bc+i].setHi(dir, REMORABCType::ext_dir);
                    }
                }
            }
            else if (bct == REMORA_BC::periodic)
            {
                if (side == Orientation::low) {
                    for (int i = 0; i < NCONS; i++)
                        domain_bcs_type[BCVars::cons_bc+i].setLo(dir, REMORABCType::int_dir);
                } else {
                    for (int i = 0; i < NCONS; i++)
                       domain_bcs_type[BCVars::cons_bc+i].setHi(dir, REMORABCType::int_dir);
                }
            }
        }
    }
 
#ifdef AMREX_USE_GPU
    Gpu::htod_memcpy
        (domain_bcs_type_d.data(), domain_bcs_type.data(),
         sizeof(amrex::BCRec)*(NCONS+AMREX_SPACEDIM));
#else
    std::memcpy
        (domain_bcs_type_d.data(), domain_bcs_type.data(),
         sizeof(amrex::BCRec)*(NCONS+AMREX_SPACEDIM));
#endif
}

init_beta_plane_coriolis()

void REMORA::init_beta_plane_coriolis

(

int

lev

)

{
    std::unique_ptr<MultiFab>& mf_fcor = vec_fcor[lev];
    auto geomdata  = Geom(lev).data();
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        auto fcor_arr = (mf_fcor)->array(mfi);
        Real coriolis_f0 = solverChoice.coriolis_f0;
        Real coriolis_beta = solverChoice.coriolis_beta;
        Real Esize = geomdata.ProbHi()[1] - geomdata.ProbLo()[1];
        Real prob_lo = geomdata.ProbLo()[1];
        Real dx = geomdata.CellSize()[1];
 
        ParallelFor(Box(fcor_arr), [=] AMREX_GPU_DEVICE (int i, int j, int )
        {
            Real y = prob_lo + (j + 0.5) * dx;
            fcor_arr(i,j,0) = coriolis_f0 + coriolis_beta * (y - 0.5 * Esize);
        });
    } //mfi
 
    vec_fcor[lev]->FillBoundary(geom[lev].periodicity());
}

init_custom()

void REMORA::init_custom ( int  lev )

private

{
    std::unique_ptr<MultiFab>& mf_z_w = vec_z_w[lev];
    std::unique_ptr<MultiFab>& mf_z_r = vec_z_r[lev];
    std::unique_ptr<MultiFab>& mf_Hz  = vec_Hz[lev];
    std::unique_ptr<MultiFab>& mf_h  = vec_hOfTheConfusingName[lev];
    std::unique_ptr<MultiFab>& mf_Zt_avg1  = vec_Zt_avg1[lev];
 
#ifdef _OPENMP
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
#endif
    for (MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
    {
        const Box &bx = mfi.tilebox();
        const auto &cons_arr = cons_new[lev]->array(mfi);
        const auto &xvel_arr = xvel_new[lev]->array(mfi);
        const auto &yvel_arr = yvel_new[lev]->array(mfi);
        const auto &zvel_arr = zvel_new[lev]->array(mfi);
 
        Array4<const Real> const& z_w_arr = (mf_z_w)->array(mfi);
        Array4<const Real> const& z_r_arr = (mf_z_r)->array(mfi);
        Array4<const Real> const& Hz_arr  = (mf_Hz)->array(mfi);
        Array4<const Real> const& h_arr  = (mf_h)->array(mfi);
        Array4<const Real> const& Zt_avg1_arr  = mf_Zt_avg1->const_array(mfi);
 
        init_custom_prob(bx, cons_arr, xvel_arr, yvel_arr, zvel_arr,
                         z_w_arr, z_r_arr, Hz_arr, h_arr, Zt_avg1_arr, geom[lev].data(),
                         solverChoice);
 
    } //mfi
 
    // Initialize the "pm" and "pn" arrays
    const auto dxi = Geom(lev).InvCellSize();
    vec_pm[lev]->setVal(dxi[0]); vec_pm[lev]->FillBoundary(geom[lev].periodicity());
    vec_pn[lev]->setVal(dxi[1]); vec_pn[lev]->FillBoundary(geom[lev].periodicity());
 
    set_2darrays(lev);
 
}

Here is the call graph for this function:

init_only()

void REMORA::init_only	(	int	lev,
		amrex::Real	time
	)

Init (NOT restart or regrid)

{
    t_new[lev] = time;
    t_old[lev] = time - 1.e200;
 
    cons_new[lev]->setVal(0.0);
    xvel_new[lev]->setVal(0.0);
    yvel_new[lev]->setVal(0.0);
    zvel_new[lev]->setVal(0.0);
 
    if (solverChoice.ic_bc_type == IC_BC_Type::Custom)
    {
        init_custom(lev);
#ifdef REMORA_USE_NETCDF
    } else if (solverChoice.ic_bc_type == IC_BC_Type::Real) {
 
        amrex::Print() << "Calling init_data_from_netcdf " << std::endl;
        init_data_from_netcdf(lev);
        amrex::Print() << "Initial data loaded from netcdf file \n " << std::endl;
 
        amrex::Print() << "Calling init_bdry_from_netcdf " << std::endl;
        init_bdry_from_netcdf();
        amrex::Print() << "Boundary data loaded from netcdf file \n " << std::endl;
#endif
    } else {
        Abort("Need to specify ic_bc_type");
    }
 
    // Ensure that the face-based data are the same on both sides of a periodic domain.
    // The data associated with the lower grid ID is considered the correct value.
    xvel_new[lev]->OverrideSync(geom[lev].periodicity());
    yvel_new[lev]->OverrideSync(geom[lev].periodicity());
    zvel_new[lev]->OverrideSync(geom[lev].periodicity());
}

init_stuff()

void REMORA::init_stuff ( int  lev, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

private

{
    // ********************************************************************************************
    // Initialize the boundary conditions
    // ********************************************************************************************
    physbcs[lev] = std::make_unique<REMORAPhysBCFunct> (lev, geom[lev], domain_bcs_type, domain_bcs_type_d,
                                                       m_bc_extdir_vals);
 
    BoxList bl2d = ba.boxList();
    for (auto& b : bl2d) {
        b.setRange(2,0);
    }
    BoxArray ba2d(std::move(bl2d));
 
    BoxList bl1d = ba.boxList();
    for (auto& b : bl1d) {
        b.setRange(0,0);
        b.setRange(1,0);
    }
    BoxArray ba1d(std::move(bl1d));
 
    // Map factors
    mapfac_m[lev].reset(new MultiFab(ba2d,dm,1,0));
    mapfac_u[lev].reset(new MultiFab(convert(ba2d,IntVect(1,0,0)),dm,1,0));
    mapfac_v[lev].reset(new MultiFab(convert(ba2d,IntVect(0,1,0)),dm,1,0));
    mapfac_m[lev]->setVal(1.);
    mapfac_u[lev]->setVal(1.);
    mapfac_v[lev]->setVal(1.);
 
    BoxArray ba_nd(ba);
    ba_nd.surroundingNodes();
    BoxArray ba_w(ba);
    ba_w.surroundingNodes(2);
 
    vec_z_phys_nd[lev].reset          (new MultiFab(ba_nd,dm,1,IntVect(NGROW,NGROW,0))); // z at psi points (nodes) MIGHT NEED NGROW+1
 
    vec_hOfTheConfusingName[lev].reset(new MultiFab(ba2d ,dm,2,IntVect(NGROW+1,NGROW+1,0))); //2d, depth (double check if negative)
    vec_Zt_avg1[lev].reset            (new MultiFab(ba2d ,dm,1,IntVect(NGROW+1,NGROW+1,0))); //2d, average of the free surface (zeta)
 
    vec_x_r[lev].reset                (new MultiFab(ba2d,dm,1,IntVect(NGROW+1,NGROW+1,0))); // x at r points (cell center)
    vec_y_r[lev].reset                (new MultiFab(ba2d,dm,1,IntVect(NGROW+1,NGROW+1,0))); // y at r points (cell center)
 
    vec_s_r[lev].reset                (new MultiFab(ba1d,dm,1,IntVect(    0,    0,0))); // scaled vertical coordinate [0,1] , transforms to z
 
    vec_z_w[lev].reset                (new MultiFab(ba_w,dm,1,IntVect(NGROW+1,NGROW+1,0))); // z at w points (cell faces)
    vec_z_r[lev].reset                (new MultiFab(ba  ,dm,1,IntVect(NGROW+1,NGROW+1,0))); // z at r points (cell center)
    vec_Hz[lev].reset                 (new MultiFab(ba  ,dm,1,IntVect(NGROW+1,NGROW+1,NGROW+1))); // like in ROMS, thickness of cell in z
 
    vec_Huon[lev].reset               (new MultiFab(convert(ba,IntVect(1,0,0)),dm,1,IntVect(NGROW,NGROW,0))); // mass flux for u component
    vec_Hvom[lev].reset               (new MultiFab(convert(ba,IntVect(0,1,0)),dm,1,IntVect(NGROW,NGROW,0))); // mass flux for v component
 
    vec_Akv[lev].reset                (new MultiFab(ba  ,dm,1,IntVect(NGROW,NGROW,0))); // vertical mixing coefficient (.in)
    vec_Akt[lev].reset                (new MultiFab(ba  ,dm,NCONS,IntVect(NGROW,NGROW,0))); // vertical mixing coefficient (.in)
    vec_visc3d_r[lev].reset           (new MultiFab(ba  ,dm,1,IntVect(NGROW,NGROW,0))); // not used
 
 
    // check dimensionality
    vec_visc2_p[lev].reset(new MultiFab(ba,dm,1,IntVect(NGROW,NGROW,0))); // harmonic viscosity at psi points -- difference to 3d?
    vec_visc2_r[lev].reset(new MultiFab(ba,dm,1,IntVect(NGROW,NGROW,0))); // harmonic viscosity at rho points
    vec_diff2[lev].reset(new MultiFab(ba,dm,NCONS,IntVect(NGROW,NGROW,0))); // harmonic diffusivity temperature/salt
 
    vec_ru[lev].reset(new MultiFab(convert(ba,IntVect(1,0,0)),dm,2,IntVect(NGROW,NGROW,NGROW))); // RHS u (incl horizontal and vertical advection)
    vec_rv[lev].reset(new MultiFab(convert(ba,IntVect(0,1,0)),dm,2,IntVect(NGROW,NGROW,NGROW))); // RHS v
 
    //2d, (incl advection terms and surface/bottom stresses, integral over the whole columnn, k=0)
    vec_rufrc[lev].reset(new MultiFab(convert(ba2d,IntVect(1,0,0)),dm,2,IntVect(NGROW,NGROW,0)));
    vec_rvfrc[lev].reset(new MultiFab(convert(ba2d,IntVect(0,1,0)),dm,2,IntVect(NGROW,NGROW,0))); //2d, same as above but v
 
    vec_sustr[lev].reset(new MultiFab(convert(ba2d,IntVect(1,0,0)),dm,1,IntVect(NGROW,NGROW,0))); //2d, surface stress
    vec_svstr[lev].reset(new MultiFab(convert(ba2d,IntVect(0,1,0)),dm,1,IntVect(NGROW,NGROW,0))); //2d
 
    //2d, linear drag coefficient [m/s], defined at rho, somehow related to rdrg
    vec_rdrag[lev].reset(new MultiFab(ba2d,dm,1,IntVect(NGROW,NGROW,0)));
 
    vec_bustr[lev].reset(new MultiFab(convert(ba2d,IntVect(1,0,0)),dm,1,IntVect(NGROW,NGROW,0))); //2d, bottom stress
    vec_bvstr[lev].reset(new MultiFab(convert(ba2d,IntVect(0,1,0)),dm,1,IntVect(NGROW,NGROW,0)));
 
    //all 2d -- all associated with the 2D advance
    //2d DU: sum(height[incl free surface?] * u)
    vec_DU_avg1[lev].reset(new MultiFab(convert(ba2d,IntVect(1,0,0)),dm,1,IntVect(NGROW,NGROW,0)));
 
    //2d like above, but correct(or)?
    vec_DU_avg2[lev].reset(new MultiFab(convert(ba2d,IntVect(1,0,0)),dm,1,IntVect(NGROW,NGROW,0)));
 
    vec_DV_avg1[lev].reset(new MultiFab(convert(ba2d,IntVect(0,1,0)),dm,1,IntVect(NGROW,NGROW,0)));
    vec_DV_avg2[lev].reset(new MultiFab(convert(ba2d,IntVect(0,1,0)),dm,1,IntVect(NGROW,NGROW,0)));
 
    vec_rubar[lev].reset(new MultiFab(convert(ba2d,IntVect(1,0,0)),dm,4,IntVect(NGROW,NGROW,0))); // 2d RHS ubar
    vec_rvbar[lev].reset(new MultiFab(convert(ba2d,IntVect(0,1,0)),dm,4,IntVect(NGROW,NGROW,0)));
    vec_rzeta[lev].reset(new MultiFab(ba2d,dm,4,IntVect(NGROW,NGROW,0))); // 2d RHS zeta
 
    // starts off kind of like a depth-averaged u, but exists at more points and more timesteps (b/c fast 2D update) than full u
    vec_ubar[lev].reset(new MultiFab(convert(ba2d,IntVect(1,0,0)),dm,3,IntVect(NGROW,NGROW,0)));
    vec_vbar[lev].reset(new MultiFab(convert(ba2d,IntVect(0,1,0)),dm,3,IntVect(NGROW,NGROW,0)));
    vec_zeta[lev].reset(new MultiFab(ba2d,dm,3,IntVect(NGROW+1,NGROW+1,0)));  // 2d free surface
 
    vec_pm[lev].reset(new MultiFab(ba2d,dm,1,IntVect(NGROW+1,NGROW+2,0)));
    vec_pn[lev].reset(new MultiFab(ba2d,dm,1,IntVect(NGROW+2,NGROW+1,0)));
    vec_fcor[lev].reset(new MultiFab(ba2d,dm,1,IntVect(NGROW+1,NGROW+1,0)));
 
    // tempstore, saltstore, etc
    vec_sstore[lev].reset(new MultiFab(ba,dm,NCONS,IntVect(NGROW,NGROW,0)));
 
    vec_rhoS[lev].reset(new MultiFab(ba,dm,1,IntVect(NGROW,NGROW,0)));
    vec_rhoA[lev].reset(new MultiFab(ba,dm,1,IntVect(NGROW,NGROW,0)));
 
    set_bathymetry(lev);
    stretch_transform(lev);
    set_vmix(lev);
    set_hmixcoef(lev);
    set_coriolis(lev);
    set_weights(lev);
 
    //consider tracking ru and rv indexes more specifically or more similarly to indx
    vec_ru[lev]->setVal(0.0_rt);
    vec_rv[lev]->setVal(0.0_rt);
 
    vec_DU_avg1[lev]->setVal(0.0_rt);
    vec_DU_avg2[lev]->setVal(0.0_rt);
    vec_DV_avg1[lev]->setVal(0.0_rt);
    vec_DV_avg2[lev]->setVal(0.0_rt);
    vec_rubar[lev]->setVal(0.0_rt);
    vec_rvbar[lev]->setVal(0.0_rt);
    vec_rzeta[lev]->setVal(0.0_rt);
 
    vec_ubar[lev]->setVal(0.0_rt);
    vec_vbar[lev]->setVal(0.0_rt);
    vec_zeta[lev]->setVal(0.0_rt);
 
    // Set initial linear drag coefficient
    vec_rdrag[lev]->setVal(solverChoice.rdrag);
}

InitData()

void REMORA::InitData

(

)

Initialize multilevel data

{
    // Initialize the start time for our CPU-time tracker
    startCPUTime = ParallelDescriptor::second();
 
    // Map the words in the inputs file to BC types, then translate
    //     those types into what they mean for each variable
    init_bcs();
 
    last_plot_file_step_1 = -1;
    last_plot_file_step_2 = -1;
    last_check_file_step = -1;
 
    if (restart_chkfile == "") {
        // start simulation from the beginning
 
        const Real time = 0.0;
        InitFromScratch(time);
 
        for (int lev = 0; lev <= finest_level; lev++)
            init_only(lev, time);
 
        if (solverChoice.coupling_type == CouplingType::TwoWay) {
            AverageDown();
        }
 
#ifdef REMORA_USE_PARTICLES
        particleData.init_particles((ParGDBBase*)GetParGDB(),vec_z_phys_nd);
#endif
 
    } else { // Restart from a checkpoint
 
        restart();
 
    }
 
    // Initialize flux registers (whether we start from scratch or restart)
    if (solverChoice.coupling_type == CouplingType::TwoWay) {
        advflux_reg[0] = nullptr;
        for (int lev = 1; lev <= finest_level; lev++)
        {
            advflux_reg[lev] = new YAFluxRegister(grids[lev], grids[lev-1],
                                                   dmap[lev],  dmap[lev-1],
                                                   geom[lev],  geom[lev-1],
                                              ref_ratio[lev-1], lev, NCONS);
        }
    }
 
    // Fill ghost cells/faces
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        FillPatch(lev, t_new[lev], *cons_new[lev], cons_new, BdyVars::t);
        FillPatch(lev, t_new[lev], *xvel_new[lev], xvel_new, BdyVars::u);
        FillPatch(lev, t_new[lev], *yvel_new[lev], yvel_new, BdyVars::v);
        FillPatch(lev, t_new[lev], *zvel_new[lev], zvel_new, BdyVars::null);
 
        //
        // Copy from new into old just in case
        //
        int ngs   = cons_new[lev]->nGrow();
        int ngvel = xvel_new[lev]->nGrow();
        MultiFab::Copy(*cons_old[lev],*cons_new[lev],0,0,NCONS,ngs);
        MultiFab::Copy(*xvel_old[lev],*xvel_new[lev],0,0,1,ngvel);
        MultiFab::Copy(*yvel_old[lev],*yvel_new[lev],0,0,1,ngvel);
        MultiFab::Copy(*zvel_old[lev],*zvel_new[lev],0,0,1,IntVect(ngvel,ngvel,0));
    } // lev
    if (restart_chkfile == "" && check_int > 0)
    {
        WriteCheckpointFile();
        last_check_file_step = 0;
    }
 
    if ( (restart_chkfile == "") ||
         (restart_chkfile != "" && plot_file_on_restart) )
    {
        if (plot_int_1 > 0)
        {
            if (plotfile_type == PlotfileType::amrex)
                WritePlotFile(1,plot_var_names_1);
#ifdef REMORA_USE_NETCDF
            if (plotfile_type == PlotfileType::netcdf) {
                int step0 = 0;
                WriteNCPlotFile(step0);
            }
#endif
            last_plot_file_step_1 = istep[0];
        }
        if (plot_int_2 > 0)
        {
            WritePlotFile(2,plot_var_names_2);
            last_plot_file_step_2 = istep[0];
        }
    }
 
    if (is_it_time_for_action(istep[0], t_new[0], dt[0], sum_interval, sum_per)) {
        sum_integrated_quantities(t_new[0]);
    }
 
    ComputeDt();
 
}

Referenced by main().

Here is the caller graph for this function:

initHSE()

void REMORA::initHSE ( )

private

Initialize HSE.

InitializeFromFile()

void REMORA::InitializeFromFile ( )

private

InitializeLevelFromData()

void REMORA::InitializeLevelFromData ( int  lev, const amrex::MultiFab &  initial_data  )

private

initRayleigh()

void REMORA::initRayleigh ( )

private

Initialize Rayleigh damping profiles.

is_it_time_for_action()

bool REMORA::is_it_time_for_action	(	int	nstep,
		amrex::Real	time,
		amrex::Real	dt,
		int	action_interval,
		amrex::Real	action_per
	)

{
  bool int_test = (action_interval > 0 && nstep % action_interval == 0);
 
  bool per_test = false;
  if (action_per > 0.0) {
    const int num_per_old = static_cast<int>(amrex::Math::floor((time - dtlev) / action_per));
    const int num_per_new = static_cast<int>(amrex::Math::floor((time) / action_per));
 
    if (num_per_old != num_per_new) {
      per_test = true;
    }
  }
 
  return int_test || per_test;
}

MakeNewLevelFromCoarse()

void REMORA::MakeNewLevelFromCoarse ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

overridevirtual

Make a new level using provided BoxArray and DistributionMapping and fill with interpolated coarse level data. Overrides the pure virtual function in AmrCore

{
    cons_new[lev]->define(ba, dm, NCONS, cons_new[lev-1]->nGrowVect());
    cons_old[lev]->define(ba, dm, NCONS, cons_new[lev-1]->nGrowVect());
 
    xvel_new[lev]->define(convert(ba, IntVect(1,0,0)), dm, 1, xvel_new[lev-1]->nGrowVect());
    xvel_old[lev]->define(convert(ba, IntVect(1,0,0)), dm, 1, xvel_new[lev-1]->nGrowVect());
 
    yvel_new[lev]->define(convert(ba, IntVect(0,1,0)), dm, 1, yvel_new[lev-1]->nGrowVect());
    yvel_old[lev]->define(convert(ba, IntVect(0,1,0)), dm, 1, yvel_new[lev-1]->nGrowVect());
 
    zvel_new[lev]->define(convert(ba, IntVect(0,0,1)), dm, 1, zvel_new[lev-1]->nGrowVect());
    zvel_old[lev]->define(convert(ba, IntVect(0,0,1)), dm, 1, zvel_new[lev-1]->nGrowVect());
 
    resize_stuff(lev);
      init_stuff(lev, ba, dm);
 
    t_new[lev] = time;
    t_old[lev] = time - 1.e200;
 
    FillCoarsePatch(lev, time, cons_new[lev], cons_new[lev-1]);
    FillCoarsePatch(lev, time, xvel_new[lev], xvel_new[lev-1]);
    FillCoarsePatch(lev, time, yvel_new[lev], yvel_new[lev-1]);
    FillCoarsePatch(lev, time, zvel_new[lev], zvel_new[lev-1]);
}

MakeNewLevelFromScratch()

void REMORA::MakeNewLevelFromScratch ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

overridevirtual

Make a new level from scratch using provided BoxArray and DistributionMapping. Only used during initialization. Overrides the pure virtual function in AmrCore

{
    // Set BoxArray grids and DistributionMapping dmap in AMReX_AmrMesh.H class
    SetBoxArray(lev, ba);
    SetDistributionMap(lev, dm);
 
    amrex::Print() << "GRIDS AT LEVEL " << lev << " ARE " << ba << std::endl;
 
    // The number of ghost cells for density must be 1 greater than that for velocity
    //     so that we can go back in forth between velocity and momentum on all faces
#if NGROW==2
    int ngrow_state = ComputeGhostCells(solverChoice.spatial_order)+1;
    int ngrow_vels  = ComputeGhostCells(solverChoice.spatial_order)+1;
#else
    int ngrow_state = ComputeGhostCells(solverChoice.spatial_order)+2;
    int ngrow_vels  = ComputeGhostCells(solverChoice.spatial_order)+2;
#endif
 
    cons_old[lev] = new MultiFab(ba, dm, NCONS, ngrow_state);
    cons_new[lev] = new MultiFab(ba, dm, NCONS, ngrow_state);
 
    xvel_new[lev] = new MultiFab(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
    xvel_old[lev] = new MultiFab(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
 
    yvel_new[lev] = new MultiFab(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
    yvel_old[lev] = new MultiFab(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
 
    zvel_new[lev] = new MultiFab(convert(ba, IntVect(0,0,1)), dm, 1, IntVect(ngrow_vels,ngrow_vels,0));
    zvel_old[lev] = new MultiFab(convert(ba, IntVect(0,0,1)), dm, 1, IntVect(ngrow_vels,ngrow_vels,0));
 
    resize_stuff(lev);
    init_stuff(lev, ba, dm);
}

NumDataLogs()

AMREX_FORCE_INLINE int REMORA::NumDataLogs ( )

inlineprivatenoexcept

    {
        return datalog.size();
    }

PlotFileName()

std::string REMORA::PlotFileName ( int  lev ) const

private

get plotfile name

PlotFileVarNames()

Vector< std::string > REMORA::PlotFileVarNames ( amrex::Vector< std::string >  plot_var_names ) const

private

set plotfile variables names

{
    Vector<std::string> names;
 
    names.insert(names.end(), plot_var_names.begin(), plot_var_names.end());
 
    return names;
 
}

post_timestep()

void REMORA::post_timestep	(	int	nstep,
		amrex::Real	time,
		amrex::Real	dt_lev
	)

Called after every level 0 timestep

{
    BL_PROFILE("REMORA::post_timestep()");
 
#ifdef REMORA_USE_PARTICLES
    particleData.Redistribute();
#endif
 
    if (solverChoice.coupling_type == CouplingType::TwoWay)
    {
        for (int lev = finest_level-1; lev >= 0; lev--)
        {
            // This call refluxes from the lev/lev+1 interface onto lev
            getAdvFluxReg(lev+1)->Reflux(*cons_new[lev], 0, 0, NCONS);
 
            // We need to do this before anything else because refluxing changes the
            // values of coarse cells underneath fine grids with the assumption they'll
            // be over-written by averaging down
            AverageDownTo(lev);
        }
    }
 
    if (is_it_time_for_action(nstep, time, dt_lev0, sum_interval, sum_per)) {
        sum_integrated_quantities(time);
    }
}

post_update()

void REMORA::post_update ( amrex::MultiFab &  state_mf, const amrex::Real  time, const amrex::Geometry &  geom  )

private

prestep()

void REMORA::prestep	(	int	lev,
		amrex::MultiFab &	mf_uold,
		amrex::MultiFab &	mf_vold,
		amrex::MultiFab &	mf_u,
		amrex::MultiFab &	mf_v,
		amrex::MultiFab *	mf_ru,
		amrex::MultiFab *	mf_rv,
		amrex::MultiFab &	S_old,
		amrex::MultiFab &	S_new,
		amrex::MultiFab &	mf_W,
		amrex::MultiFab &	mf_DC,
		const amrex::MultiFab *	mf_z_r,
		const amrex::MultiFab *	mf_z_w,
		const amrex::MultiFab *	mf_h,
		const amrex::MultiFab *	mf_pm,
		const amrex::MultiFab *	mf_pn,
		const amrex::MultiFab *	mf_sustr,
		const amrex::MultiFab *	mf_svstr,
		const amrex::MultiFab *	mf_bustr,
		const amrex::MultiFab *	mf_bvstr,
		const int	iic,
		const int	nfirst,
		const int	nnew,
		int	nstp,
		int	nrhs,
		int	N,
		const amrex::Real	dt_lev
	)

Wrapper function for prestep

prestep

Parameters

[in]	mf_vold
[in,out]	mf_u	(looks like reset in update_vel <- prestep_uv_3d, so maybe just out
[in,out]	mf_v	maybe just out
[in,out]	mf_ru
[in,out]	mf_rv
[in]	S_old
[in,out]	S_new
[out]	mf_W
	[none	] mf_DC (temp)
[in]	mf_z_r
[in]	mf_z_w
[in]	mf_h
[in]	mf_sustr
[in]	mf_svstr
[in]	mf_bustr
[in]	mf_bvstr
[in]	iic
[in]	ntfirst
[in]	nnew
[in]	nstp
[in]	nrhs
[in]	N
[in]	dt_lev

{
    const BoxArray&            ba = S_old.boxArray();
    const DistributionMapping& dm = S_old.DistributionMap();
 
    // Maybe not the best way to do this, but need to cache salt and temp since
    // they get rewritten by prestep_t
    MultiFab mf_saltcache(ba,dm,1,IntVect(NGROW,NGROW,0));
    MultiFab mf_tempcache(ba,dm,1,IntVect(NGROW,NGROW,0));
 
    MultiFab mf_scalarcache(ba,dm,NCONS,IntVect(NGROW,NGROW,0));
    MultiFab::Copy(mf_scalarcache,S_new,0,0,NCONS,IntVect(NGROW,NGROW,0));
 
    MultiFab::Copy(mf_saltcache,S_new,Salt_comp,0,1,IntVect(NGROW,NGROW,0));
    MultiFab::Copy(mf_tempcache,S_new,Temp_comp,0,1,IntVect(NGROW,NGROW,0));
 
    for ( MFIter mfi(S_new, TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
        Array4<Real> const& DC = mf_DC.array(mfi);
        Array4<Real> const& Akv   = vec_Akv[lev]->array(mfi);
        Array4<Real> const& Akt   = vec_Akt[lev]->array(mfi);
        Array4<Real> const& Hz    = vec_Hz[lev]->array(mfi);
        Array4<Real> const& Huon  = vec_Huon[lev]->array(mfi);
        Array4<Real> const& Hvom  = vec_Hvom[lev]->array(mfi);
        Array4<Real const> const& uold  = mf_uold.const_array(mfi);
        Array4<Real const> const& vold = mf_vold.const_array(mfi);
        Array4<Real> const& u = (mf_u).array(mfi);
        Array4<Real> const& v = (mf_v).array(mfi);
 
        Array4<Real const> const& z_r   = mf_z_r->const_array(mfi);
        Array4<Real const> const& z_w   = mf_z_w->const_array(mfi);
        Array4<Real const> const& h     = mf_h->const_array(mfi);
        Array4<Real const> const& pm    = mf_pm->const_array(mfi);
        Array4<Real const> const& pn    = mf_pn->const_array(mfi);
 
        Array4<Real      > const& ru = mf_ru->array(mfi);
        Array4<Real      > const& rv = mf_rv->array(mfi);
        Array4<Real      > const& W = (mf_W).array(mfi);
        Array4<Real const> const& sustr = mf_sustr->const_array(mfi);
        Array4<Real const> const& svstr = mf_svstr->const_array(mfi);
        Array4<Real const> const& bustr = mf_bustr->const_array(mfi);
        Array4<Real const> const& bvstr = mf_bvstr->const_array(mfi);
 
        Real lambda = 1.0_rt;
 
        Box bx = mfi.tilebox();
        Box gbx = mfi.growntilebox();
        Box gbx1 = mfi.growntilebox(IntVect(NGROW-1,NGROW-1,0));
        Box gbx2 = mfi.growntilebox(IntVect(NGROW,NGROW,0));
        //Box gbx11 = mfi.growntilebox(IntVect(NGROW-1,NGROW-1,NGROW-1));
 
        //copy the tilebox
        Box tbxp1 = bx;
        Box tbxp11 = bx;
        Box tbxp2 = bx;
 
        //TODO: adjust for tiling
        //Box gbx3uneven(IntVect(AMREX_D_DECL(bx.smallEnd(0)-3,bx.smallEnd(1)-3,bx.smallEnd(2))),
        //               IntVect(AMREX_D_DECL(bx.bigEnd(0)+2,bx.bigEnd(1)+2,bx.bigEnd(2))));
        //Box gbx2uneven(IntVect(AMREX_D_DECL(bx.smallEnd(0)-2,bx.smallEnd(1)-2,bx.smallEnd(2))),
        //               IntVect(AMREX_D_DECL(bx.bigEnd(0)+1,bx.bigEnd(1)+1,bx.bigEnd(2))));
        //make only gbx be grown to match multifabs
        tbxp2.grow(IntVect(NGROW,NGROW,0));
        tbxp1.grow(IntVect(NGROW-1,NGROW-1,0));
        tbxp11.grow(IntVect(NGROW-1,NGROW-1,NGROW-1));
 
        Box bxD = bx;
        bxD.makeSlab(2,0);
        Box gbx1D = gbx1;
        gbx1D.makeSlab(2,0);
        Box gbx2D = gbx2;
        gbx2D.makeSlab(2,0);
 
        Box tbxp1D = tbxp1;
        tbxp1D.makeSlab(2,0);
        Box tbxp2D = tbxp2;
        tbxp2D.makeSlab(2,0);
 
        FArrayBox fab_FC(tbxp2,1,amrex::The_Async_Arena()); //3D
 
        auto FC=fab_FC.array();
 
        //From ini_fields and .in file
        //fab_Akt.setVal(1e-6);
        FArrayBox fab_stflux(tbxp2,1,amrex::The_Async_Arena());
        auto stflux= fab_stflux.array();
        FArrayBox fab_btflux(tbxp2,1,amrex::The_Async_Arena());
        auto btflux= fab_btflux.array();
 
        //From ini_fields and .in file
        //fab_stflux.setVal(0.0_rt);
        //also set btflux=0 (as in ana_btflux.H)
        ParallelFor(tbxp2,
        [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            stflux(i,j,k)=0.0_rt;
            btflux(i,j,k)=0.0_rt;
        });
 
        for (int i_comp=0; i_comp < NCONS; i_comp++) {
            Array4<Real> const& sstore = (vec_sstore[lev])->array(mfi,i_comp);
            prestep_t_advection(bx, gbx, S_old.array(mfi,i_comp),
                                mf_scalarcache.array(mfi,i_comp), Hz, Huon, Hvom,
                                W, DC, FC, sstore, z_w, h, pm, pn, iic, ntfirst,
                                nrhs, N, dt_lev);
        }
 
        // Only do diffusion for salt and temperature, not other tracer(s)
        for (int i_comp=0; i_comp < NCONS; i_comp++) {
            prestep_diffusion(bx,gbx,0,0,S_new.array(mfi,i_comp), S_old.array(mfi,i_comp), ru,
                              Hz, Akt, DC, FC, stflux, btflux, z_r, pm, pn, iic, iic, nnew, nstp,
                              nrhs, N, lambda, dt_lev);
        }
 
        //
        //-----------------------------------------------------------------------
        // prestep_uv_3d
        //-----------------------------------------------------------------------
        //
        //updates u,v,ru,rv (ru and rv have multiple components)
 
        prestep_diffusion(tbxp1, gbx, 1, 0, u, uold, ru, Hz, Akv, DC, FC,
                          sustr, bustr, z_r, pm, pn, iic, ntfirst, nnew, nstp, nrhs, N, lambda, dt_lev);
 
        prestep_diffusion(tbxp1, gbx, 0, 1, v, vold, rv, Hz, Akv, DC, FC,
                          svstr, bvstr, z_r, pm, pn, iic, ntfirst, nnew, nstp, nrhs, N, lambda, dt_lev);
    }
}

prestep_diffusion()

void REMORA::prestep_diffusion	(	const amrex::Box &	bx,
		const amrex::Box &	gbx,
		const int	ioff,
		const int	joff,
		const amrex::Array4< amrex::Real > &	vel,
		const amrex::Array4< amrex::Real const > &	vel_old,
		const amrex::Array4< amrex::Real > &	rvel,
		const amrex::Array4< amrex::Real const > &	Hz,
		const amrex::Array4< amrex::Real const > &	Akv,
		const amrex::Array4< amrex::Real > &	DC,
		const amrex::Array4< amrex::Real > &	FC,
		const amrex::Array4< amrex::Real const > &	sstr,
		const amrex::Array4< amrex::Real const > &	bstr,
		const amrex::Array4< amrex::Real const > &	z_r,
		const amrex::Array4< amrex::Real const > &	pm,
		const amrex::Array4< amrex::Real const > &	pn,
		const int	iic,
		const int	ntfirst,
		const int	nnew,
		int	nstp,
		int	nrhs,
		int	N,
		const amrex::Real	lambda,
		const amrex::Real	dt_lev
	)

Update velocities or tracers with diffusion/viscosity as the last part of the prestep

{
 
    BoxArray ba_gbxvel = intersect(BoxArray(vel_bx), gbx);
    AMREX_ASSERT((ba_gbxvel.size() == 1));
    Box gbxvel = ba_gbxvel[0];
 
    //
    //  Weighting coefficient for the newest (implicit) time step derivatives
    //  using either a Crank-Nicolson implicit scheme (lambda=0.5) or a
    //  backward implicit scheme (lambda=1.0).
    //
 
    Real oml_dt = dt_lev*(1.0-lambda);
    //N is one less than ROMS
 
    //  Except the commented out part means lambda is always 1.0
    if (verbose > 1) {
        amrex::Print() << "in update_vel_3d with box " << vel_bx << std::endl;
        Print() << "vel old " << Box(vel_old) << std::endl;
        Print() << "Akv " << Box(Akv) << std::endl;
    }
    ParallelFor(vel_bx,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        if(k+1<=N&&k>=0)
        {
            Real cff = 1.0 / ( z_r(i,j,k+1)+z_r(i-ioff,j-joff,k+1)
                              -z_r(i,j,k  )-z_r(i-ioff,j-joff,k  ));
            FC(i,j,k) = oml_dt * cff * (vel_old(i,j,k+1,nstp)-vel_old(i,j,k,nstp)) *
                                           (Akv(i,j,k)     +Akv(i-ioff,j-joff,k));
        }
        else if (k==-1)
        {
            FC(i,j,-1) = dt_lev*bstr(i,j,0);
        }
        else if (k==N)
        {
            FC(i,j, N) = dt_lev*sstr(i,j,0);
        }
 
        DC(i,j,k) = 0.25 * dt_lev * (pm(i,j,0)+pm(i-ioff,j-joff,0))
                                  * (pn(i,j,0)+pn(i-ioff,j-joff,0));
    });
 
    ParallelFor(gbxvel,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real cff3=dt_lev*(1.0-lambda);
        Real cff1 = 0.0, cff2 = 0.0, cff4;
 
        int indx=0; //nrhs-3
 
        if (iic==ntfirst)
        {
            if (k+1<=N && k>=1)
            {
                cff1=vel_old(i,j,k,nstp)*0.5*(Hz(i,j,k)+Hz(i-ioff,j-joff,k));
                cff2=FC(i,j,k)-FC(i,j,k-1);
                vel(i,j,k,nnew)=cff1+cff2;
            }
            else if(k==0)
            {
                cff1=vel_old(i,j,k,nstp)*0.5*(Hz(i,j,k)+Hz(i-ioff,j-joff,k));
                cff2=FC(i,j,k)-dt_lev*bstr(i,j,0);
                vel(i,j,k,nnew)=cff1+cff2;
            }
            else if(k==N)
            {
                cff1=vel_old(i,j,k,nstp)*0.5*(Hz(i,j,k)+Hz(i-ioff,j-joff,k));
                cff2=dt_lev*sstr(i,j,0)-FC(i,j,k-1); //or: -FC(i,j,k-1)+dt_lev*sstr(i,j,0);
                vel(i,j,k,nnew)=cff1+cff2;
            }
 
        } else if(iic==ntfirst+1) {
 
            if (k<N && k>0) {
                cff1=vel_old(i,j,k,nstp)*0.5*(Hz(i,j,k)+Hz(i-ioff,j-joff,k));
                cff2=FC(i,j,k)-FC(i,j,k-1);
            }
            else if(k==0) {
                cff1=vel_old(i,j,k,nstp)*0.5*(Hz(i,j,k)+Hz(i-ioff,j-joff,k));
                cff2=FC(i,j,k)-dt_lev*bstr(i,j,0);
            }
            else if(k==N) {
                cff1=vel_old(i,j,k,nstp)*0.5*(Hz(i,j,k)+Hz(i-ioff,j-joff,k));
                cff2=dt_lev*sstr(i,j,0)-FC(i,j,k-1); //or: -FC(i,j,k-1)+dt_lev*sstr(i,j,0);
            }
 
            cff3=0.5*DC(i,j,k);
 
            if (ioff==0&&joff==0) {
                vel(i,j,k,nnew)=cff1 + cff2;
            } else {
                indx=nrhs ? 0 : 1;
                Real r_swap= rvel(i,j,k,indx);
                rvel(i,j,k,indx) = rvel(i,j,k,nrhs);
                rvel(i,j,k,nrhs) = r_swap;
                vel(i,j,k,nnew)=cff1- cff3*rvel(i,j,k,indx)+ cff2;
            }
        } else {
            cff1 =  5.0/12.0;
            cff2 = 16.0/12.0;
            if (k==0) {
                cff3=vel_old(i,j,k,nstp)*0.5*(Hz(i,j,k)+Hz(i-ioff,j-joff,k));
                cff4=FC(i,j,k)-dt_lev*bstr(i,j,0);
                //cff4=FC(i,j,k)-FC(i,j,k-1);//-bustr(i,j,0);
 
            } else if (k == N) {
                cff3=vel_old(i,j,k,nstp)*0.5*(Hz(i,j,k)+Hz(i-ioff,j-joff,k));
                cff4=dt_lev*sstr(i,j,0)-FC(i,j,k-1);
 
            } else {
                cff3=vel_old(i,j,k,nstp)*0.5*(Hz(i,j,k)+Hz(i-ioff,j-joff,k));
                cff4=FC(i,j,k)-FC(i,j,k-1);
            }
 
            if (ioff==0 && joff==0) {
                vel(i,j,k,nnew) = cff3 + cff4;
            } else {
                indx=nrhs ? 0 : 1;
                Real r_swap= rvel(i,j,k,indx);
                rvel(i,j,k,indx) = rvel(i,j,k,nrhs);
                rvel(i,j,k,nrhs) = r_swap;
 
                vel(i,j,k,nnew) = cff3 + DC(i,j,k)*(cff1*rvel(i,j,k,nrhs)-
                                                    cff2*rvel(i,j,k,indx))+
                    cff4;
                rvel(i,j,k,nrhs) = 0.0;
            }
        }
    });
}

prestep_t_advection()

void REMORA::prestep_t_advection	(	const amrex::Box &	tbx,
		const amrex::Box &	gbx,
		const amrex::Array4< amrex::Real > &	tempold,
		const amrex::Array4< amrex::Real > &	tempcache,
		const amrex::Array4< amrex::Real > &	Hz,
		const amrex::Array4< amrex::Real > &	Huon,
		const amrex::Array4< amrex::Real > &	Hvom,
		const amrex::Array4< amrex::Real > &	W,
		const amrex::Array4< amrex::Real > &	DC,
		const amrex::Array4< amrex::Real > &	FC,
		const amrex::Array4< amrex::Real > &	sstore,
		const amrex::Array4< amrex::Real const > &	z_w,
		const amrex::Array4< amrex::Real const > &	h,
		const amrex::Array4< amrex::Real const > &	pm,
		const amrex::Array4< amrex::Real const > &	pn,
		int	iic,
		int	ntfirst,
		int	nrhs,
		int	N,
		const amrex::Real	dt_lev
	)

Prestep calculations for the tracers

{
    //copy the tilebox
    Box gbx1 = tbx;
    Box gbx2 = tbx;
    Box tbxp1 = tbx;
    Box tbxp2 = tbx;
 
    tbxp1.grow(IntVect(NGROW-1,NGROW-1,0));
    tbxp2.grow(IntVect(NGROW,NGROW,0));
    FArrayBox fab_FX(tbxp2,1,amrex::The_Async_Arena()); //3D
    FArrayBox fab_FE(tbxp2,1,amrex::The_Async_Arena()); //3D
    FArrayBox fab_curv(tbxp2,1,amrex::The_Async_Arena()); //fab_curv.setVal(0.0_rt);
    FArrayBox fab_grad(tbxp2,1,amrex::The_Async_Arena()); //fab_curv.setVal(0.0_rt);
 
    auto FX=fab_FX.array();
    auto FE=fab_FE.array();
    auto curv=fab_curv.array();
    auto grad=fab_grad.array();
    ParallelFor(tbxp2,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        grad(i,j,k)=0.0_rt;
 
        curv(i,j,k)=0.0_rt;
 
        FX(i,j,k)=0.0_rt;
        FE(i,j,k)=0.0_rt;
    });
 
 
    Box ubx = surroundingNodes(tbx,0);
    Box vbx = surroundingNodes(tbx,1);
 
    Box utbxp1 = surroundingNodes(tbxp1,0);
    Box vtbxp1 = surroundingNodes(tbxp1,1);
 
    Box gbx3uneven_init(IntVect(AMREX_D_DECL(tbx.smallEnd(0)-3,tbx.smallEnd(1)-3,tbx.smallEnd(2))),
                   IntVect(AMREX_D_DECL(tbx.bigEnd(0)+2,tbx.bigEnd(1)+2,tbx.bigEnd(2))));
    BoxArray ba_gbx3uneven = intersect(BoxArray(gbx3uneven_init), gbx);
    AMREX_ASSERT((ba_gbx3uneven.size() == 1));
    Box gbx3uneven = ba_gbx3uneven[0];
 
    gbx2.grow(IntVect(NGROW,NGROW,0));
    BoxArray ba_gbx2 = intersect(BoxArray(gbx2), gbx);
    AMREX_ASSERT((ba_gbx2.size() == 1));
    gbx2 = ba_gbx2[0];
 
    gbx1.grow(IntVect(NGROW-1,NGROW-1,0));
    BoxArray ba_gbx1 = intersect(BoxArray(gbx1), gbx);
    AMREX_ASSERT((ba_gbx1.size() == 1));
    gbx1 = ba_gbx1[0];
 
    //------------------------------------------------------------------------
    //  Vertically integrate horizontal mass flux divergence.
    //------------------------------------------------------------------------
    //
    //Should really use gbx3uneven
    Box gbx3unevenD = gbx3uneven;
    gbx3unevenD.makeSlab(2,0);
    Box gbx1D = gbx1;
    gbx1D.makeSlab(2,0);
 
    ParallelFor(gbx1D,
    [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        //  Starting with zero vertical velocity at the bottom, integrate
        //  from the bottom (k=0) to the free-surface (k=N).  The w(:,:,N(ng))
        //  contains the vertical velocity at the free-surface, d(zeta)/d(t).
        //  Notice that barotropic mass flux divergence is not used directly.
        //
        //W(i,j,-1)=0.0_rt;
        int k=0;
        W(i,j,k) = -(Huon(i+1,j,k)-Huon(i,j,k)) - (Hvom(i,j+1,k)-Hvom(i,j,k));
        for(k=1;k<=N;k++) {
            W(i,j,k) = W(i,j,k-1) - (Huon(i+1,j,k)-Huon(i,j,k)) - (Hvom(i,j+1,k)-Hvom(i,j,k));
        }
    });
    ParallelFor(gbx1,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        //  Starting with zero vertical velocity at the bottom, integrate
        //  from the bottom (k=0) to the free-surface (k=N).  The w(:,:,N(ng))
        //  contains the vertical velocity at the free-surface, d(zeta)/d(t).
        //  Notice that barotropic mass flux divergence is not used directly.
        //
        Real wrk_i=W(i,j,N)/(z_w(i,j,N)+h(i,j,0,0));
 
        if(k!=N) {
            W(i,j,k) = W(i,j,k)- wrk_i*(z_w(i,j,k)+h(i,j,0,0));
        }
    });
 
    ParallelFor(gbx1,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        if (k == N) {
            W(i,j,N) = 0.0_rt;
        }
    });
 
    //From ini_fields and .in file
    //fab_Akt.setVal(1e-6);
    FArrayBox fab_stflux(tbxp2,1,amrex::The_Async_Arena());
    auto stflux= fab_stflux.array();
    FArrayBox fab_btflux(tbxp2,1,amrex::The_Async_Arena());
    auto btflux= fab_btflux.array();
 
    //From ini_fields and .in file
    //fab_stflux.setVal(0.0_rt);
    //also set btflux=0 (as in ana_btflux.H)
 
    ParallelFor(tbxp2, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        stflux(i,j,k)=0.0_rt;
        btflux(i,j,k)=0.0_rt;
    });
 
    //Use FC and DC as intermediate arrays for FX and FE
    //First pass do centered 2d terms
 
    if (solverChoice.flat_bathymetry) {
        ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            FX(i,j,k)=Box(tempold).contains(i-1,j,k) ? Huon(i,j,k)*
                        0.5_rt*(tempold(i-1,j,k)+tempold(i  ,j,k)) : 1e34;
        });
        ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            FE(i,j,k)=Box(tempold).contains(i,j-1,k) ? Hvom(i,j,k)*
                        0.5_rt*(tempold(i,j-1,k)+tempold(i,j,k)) : 1e34;
        });
 
    } else {
 
        ParallelFor(utbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            //should be t index 3
            FX(i,j,k)=tempold(i,j,k,nrhs)-tempold(i-1,j,k,nrhs);
        });
 
        ParallelFor(vtbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            //should be t index 3
            FE(i,j,k)=tempold(i,j,k,nrhs)-tempold(i,j-1,k,nrhs);
        });
 
        Real cffa=1.0_rt/6.0_rt;
        Real cffb=1.0_rt/3.0_rt;
        if (solverChoice.Hadv_scheme == AdvectionScheme::upstream3)
        {
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                //Upstream3
                curv(i,j,k)=-FX(i,j,k)+FX(i+1,j,k);
            });
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                Real max_Huon = std::max(Huon(i,j,k),0.0_rt);
                Real min_Huon = std::min(Huon(i,j,k),0.0_rt);
 
                FX(i,j,k)=Huon(i,j,k)*0.5_rt*(tempold(i,j,k)+tempold(i-1,j,k))-
                    cffa*(curv(i,j,k)*min_Huon+ curv(i-1,j,k)*max_Huon);
            });
 
        } else if (solverChoice.Hadv_scheme == AdvectionScheme::centered4) {
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                //Centered4
                grad(i,j,k)=0.5_rt*(FX(i,j,k)+FX(i+1,j,k));
            });
 
            ParallelFor(ubx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                FX(i,j,k)=Huon(i,j,k)*0.5_rt*(tempold(i,j,k)+tempold(i-1,j,k)-
                                           cffb*(grad(i,j,k)-grad(i-1,j,k)));
            });
        } else {
           Error("Not a valid horizontal advection scheme");
        }
 
        if (solverChoice.Hadv_scheme == AdvectionScheme::upstream3)
        {
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                curv(i,j,k)=-FE(i,j,k)+FE(i,j+1,k);
            });
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                Real max_Hvom = std::max(Hvom(i,j,k),0.0_rt);
                Real min_Hvom = std::min(Hvom(i,j,k),0.0_rt);
 
                FE(i,j,k)=Hvom(i,j,k)*0.5_rt*(tempold(i,j,k)+tempold(i,j-1,k))-
                    cffa*(curv(i,j,k)*min_Hvom+ curv(i,j-1,k)*max_Hvom);
            });
 
        } else if (solverChoice.Hadv_scheme == AdvectionScheme::centered4) {
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                grad(i,j,k)=0.5_rt*(FE(i,j,k)+FE(i,j+1,k));
            });
 
            ParallelFor(vbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                FE(i,j,k)=Hvom(i,j,k)*0.5_rt*(tempold(i,j,k)+tempold(i,j-1,k)-
                                           cffb*(grad(i,j,k)- grad(i,j-1,k)));
            });
        } else {
            Error("Not a valid horizontal advection scheme");
        }
    } // not flat
 
    //Intermediate tracer at 3
    //
    //  Time-step horizontal advection (m Tunits).
    //
 
    Real cff1 = 0.0_rt, cff2 = 0.0_rt, cff;
 
    Real GammaT = 1.0_rt/6.0_rt;
 
    if (iic==ntfirst)
    {
        cff=0.5_rt*dt_lev;
        cff1=1.0_rt;
        cff2=0.0_rt;
    } else {
        cff=(1.0_rt-GammaT)*dt_lev;
        cff1=0.5_rt+GammaT;
        cff2=0.5_rt-GammaT;
    }
 
    ParallelFor(tbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        tempstore(i,j,k)=Hz(i,j,k)*( cff1 * tempold(i,j,k)+
                                     cff2 * tempcache(i,j,k) )-
                                     cff  * pm(i,j,0)*pn(i,j,0) * (FX(i+1,j,k)-FX(i,j,k)+
                                                                   FE(i,j+1,k)-FE(i,j,k));
    });
 
    //
    // Time-step vertical advection of tracers (Tunits). Impose artificial
    // continuity equation.
    //
    ParallelFor(tbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
          //-----------------------------------------------------------------------
          //  Add in vertical advection.
          //-----------------------------------------------------------------------
 
          Real c1=0.5_rt;
          Real c2=7.0_rt/12.0_rt;
          Real c3=1.0_rt/12.0_rt;
 
          if (k>=1 && k<=N-2)
          {
                  FC(i,j,k)=( c2*(tempold(i  ,j,k  ,nrhs)+ tempold(i,j,k+1,nrhs))
                             -c3*(tempold(i  ,j,k-1,nrhs)+ tempold(i,j,k+2,nrhs)) )*
                                ( W(i,j,k));
          }
          else // this needs to be split up so that the following can be concurrent
          {
              FC(i,j,N)=0.0_rt;
 
              FC(i,j,N-1) = ( c2*tempold(i,j,N-1,nrhs)+ c1*tempold(i,j,N,nrhs)-c3*tempold(i,j,N-2,nrhs) )
                          * W(i,j,N-1);
 
              FC(i,j,  0) = ( c2*tempold(i,j,  1,nrhs)+ c1*tempold(i,j,0,nrhs)-c3*tempold(i,j,2,nrhs) )
                          * W(i,j,0);
          }
    });
 
    ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        if(k-1>=0) {
        DC(i,j,k)=1.0_rt/(Hz(i,j,k)-
                        cff*pm(i,j,0)*pn(i,j,0)*
                        (Huon(i+1,j,k)-Huon(i,j,k)+
                         Hvom(i,j+1,k)-Hvom(i,j,k)+
                         (W(i,j,k)-W(i,j,k-1))));
        } else {
        DC(i,j,k)=1.0_rt/(Hz(i,j,k)-
                        cff*pm(i,j,0)*pn(i,j,0)*
                        (Huon(i+1,j,k)-Huon(i,j,k)+
                         Hvom(i,j+1,k)-Hvom(i,j,k)+
                         (W(i,j,k))));
        }
    });
 
    ParallelFor(tbx,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real c1 = cff*pm(i,j,0)*pn(i,j,0);
 
        Real c4 = (k>0) ? FC(i,j,k)-FC(i,j,k-1) : FC(i,j,k);
 
        tempstore(i,j,k) = DC(i,j,k)*(tempstore(i,j,k)-c1*c4);
    });
 
}

prsgrd()

void REMORA::prsgrd	(	const amrex::Box &	bx,
		const amrex::Box &	gbx,
		const amrex::Box &	utbx,
		const amrex::Box &	vtbx,
		const amrex::Array4< amrex::Real > &	ru,
		const amrex::Array4< amrex::Real > &	rv,
		const amrex::Array4< amrex::Real const > &	pn,
		const amrex::Array4< amrex::Real const > &	pm,
		const amrex::Array4< amrex::Real const > &	rho,
		const amrex::Array4< amrex::Real > &	FC,
		const amrex::Array4< amrex::Real const > &	Hz,
		const amrex::Array4< amrex::Real const > &	z_r,
		const amrex::Array4< amrex::Real const > &	z_w,
		const int	nrhs,
		const int	N
	)

{
    auto phi_bxD=phi_bx;
    phi_bxD.makeSlab(2,0);
    auto phi_gbxD=phi_gbx & phi_bx;
    phi_gbxD.makeSlab(2,0);
    Box phi_ubx = surroundingNodes(phi_bx,0);
    Box phi_vbx = surroundingNodes(phi_bx,1);
    auto utbxD = utbx;
    auto vtbxD = vtbx;
    utbxD.makeSlab(2,0);
    vtbxD.makeSlab(2,0);
 
    const Real OneFifth = 0.2_rt;
    const Real OneTwelfth = 1.0_rt/12.0_rt;
    const Real eps = 1.0E-10_rt;
    Real GRho     = solverChoice.g/solverChoice.rho0;
    Real GRho0    = 1000.0_rt * GRho;
    Real HalfGRho = 0.5_rt    * GRho;
 
    FArrayBox fab_P(phi_bx,1,The_Async_Arena());
    FArrayBox fab_aux(Box(z_r),1,The_Async_Arena());
    FArrayBox fab_dR(phi_bx,1,The_Async_Arena());
    FArrayBox fab_dZ(phi_bx,1,The_Async_Arena());
    FArrayBox fab_dRx(phi_bx,1,The_Async_Arena());
    FArrayBox fab_dZx(phi_bx,1,The_Async_Arena());
 
    auto P=fab_P.array();
    auto aux=fab_aux.array();
    auto dR=fab_dR.array();
    auto dZ=fab_dZ.array();
    auto dRx=fab_dRx.array();
    auto dZx=fab_dZx.array();
 
    // Derivatives in the z direction
    ParallelFor(phi_bx,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        if(k>=0&&k<N) {
            dR(i,j,k)=rho(i,j,k+1)-rho(i,j,k);
            dZ(i,j,k)=z_r(i,j,k+1)-z_r(i,j,k);
        } else {
            dR(i,j,N)=rho(i,j,N)-rho(i,j,N-1);
            dZ(i,j,N)=z_r(i,j,N)-z_r(i,j,N-1);
        }
    });
 
    ParallelFor(phi_bxD,
    [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        for(int k=N;k>=0;k--) {
            Real cff= k>0 ? 2.0*dR(i,j,k)*dR(i,j,k-1) : 2.0*dR(i,j,k)*dR(i,j,k);
            if (cff>eps) {
                dR(i,j,k)= k>0 ? cff/(dR(i,j,k)+dR(i,j,k-1)) : cff/(dR(i,j,k)+dR(i,j,k));
            } else {
                dR(i,j,k)=0.0;
            }
            dZ(i,j,k)= k>0 ? 2.0_rt*dZ(i,j,k)*dZ(i,j,k-1)/(dZ(i,j,k)+dZ(i,j,k-1)) :
                             2.0_rt*dZ(i,j,k)*dZ(i,j,k)/(dZ(i,j,k)+dZ(i,j,k));
        }
    });
 
    ParallelFor(phi_bxD,
    [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        Real cff1=1.0_rt/(z_r(i,j,N)-z_r(i,j,N-1));
        Real cff2=0.5_rt*(rho(i,j,N)-rho(i,j,N-1))*(z_w(i,j,N)-z_r(i,j,N))*cff1;
 
        P(i,j,N)=GRho0*z_w(i,j,N)+GRho*(rho(i,j,N)+cff2)*(z_w(i,j,N)-z_r(i,j,N));
 
        for (int k=N-1;k>=0;k--)
        {
            Real rho_diff = rho(i,j,k+1)-rho(i,j,k) - OneTwelfth* (dR(i,j,k+1)+dR(i,j,k));
            Real   z_diff = z_r(i,j,k+1)-z_r(i,j,k) - OneTwelfth* (dZ(i,j,k+1)+dZ(i,j,k));
            Real   rz_avg = (rho(i,j,k+1)+rho(i,j,k)) * (z_r(i,j,k+1)-z_r(i,j,k));
 
            P(i,j,k) = P(i,j,k+1) + HalfGRho * ( rz_avg -
                          OneFifth* ( (dR(i,j,k+1)-dR(i,j,k)) *  z_diff -
                                      (dZ(i,j,k+1)-dZ(i,j,k)) * rho_diff ) );
         }
    });
 
    //This should be nodal
    // Derivatives in the x direction
    ParallelFor(phi_ubx,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        FC(i,j,k)=rho(i,j,k)-rho(i-1,j,k);
        aux(i,j,k)=z_r(i,j,k)-z_r(i-1,j,k);
    });
 
    //This should be nodal aux and FC need wider boxes above
    //dZx and dRx may have index mismatch issues at k=2 and k=N
    ParallelFor(phi_bxD,
    [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        for(int k=N;k>=0;k--) {
            Real cff= 2.0*aux(i,j,k)*aux(i+1,j,k);
            if (cff>eps) {
                Real cff1= 1.0_rt/(aux(i+1,j,k)+aux(i,j,k));
                dZx(i,j,k)=cff*cff1;
            } else {
                dZx(i,j,k)=0.0;
            }
            Real cff1= 2.0*FC(i,j,k)*FC(i+1,j,k);
            if (cff1>eps) {
                Real cff2= 1.0_rt/(FC(i,j,k)+FC(i+1,j,k));
                dRx(i,j,k)=cff1*cff2;
            } else {
                dRx(i,j,k)=0.0;
            }
        }
    });
 
    //This should be nodal aux and FC need wider boxes above
    ParallelFor(utbxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        for(int k=N;k>=0;k--)
        {
            Real rho_diff   = rho(i,j,k)-rho(i-1,j,k)- OneTwelfth* (dRx(i,j,k)+dRx(i-1,j,k));
            Real z_r_diff   = z_r(i,j,k)-z_r(i-1,j,k)- OneTwelfth* (dZx(i,j,k)+dZx(i-1,j,k));
            Real   Hz_avg   = 0.5_rt * (Hz(i,j,k)+Hz(i-1,j,k));
 
            Real on_u = 2.0_rt / (pn(i-1,j,0)+pn(i,j,0));
            ru(i,j,k,nrhs) = on_u * Hz_avg * (
                            P(i-1,j,k) - P(i,j,k) - HalfGRho *
                            ( (rho(i,j,k)+rho(i-1,j,k))*(z_r(i,j,k)-z_r(i-1,j,k))-
                              OneFifth * ( (dRx(i,j,k)-dRx(i-1,j,k)) * z_r_diff -
                                           (dZx(i,j,k)-dZx(i-1,j,k)) * rho_diff ) ) );
        }
    });
 
    //This should be nodal
    ParallelFor(phi_vbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        FC(i,j,k)= rho(i,j,k)-rho(i,j-1,k);
        aux(i,j,k)= z_r(i,j,k)-z_r(i,j-1,k);
    });
 
    //This should be nodal aux and FC need wider boxes above
    //dZx and dRx may have index mismatch issues at k=2 and k=N
    ParallelFor(phi_bxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        for(int k=N;k>=0;k--) {
            Real cff= 2.0*aux(i,j,k)*aux(i,j+1,k);
            if (cff>eps) {
                Real cff1= 1.0_rt/(aux(i,j+1,k)+aux(i,j,k));
                dZx(i,j,k)=cff*cff1;
            } else {
                dZx(i,j,k)=0.0;
            }
            Real cff1= 2.0*FC(i,j,k)*FC(i,j+1,k);
            if (cff1>eps) {
                Real cff2= 1.0_rt/(FC(i,j,k)+FC(i,j+1,k));
                dRx(i,j,k)=cff1*cff2;
            } else {
                dRx(i,j,k)=0.0;
            }
        }
    });
 
    //This should be nodal aux and FC need wider boxes above
    ParallelFor(vtbxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        for (int k=N;k>=0;k--)
        {
            Real rho_diff   = rho(i,j,k)-rho(i,j-1,k)- OneTwelfth* (dRx(i,j,k)+dRx(i,j-1,k));
            Real z_r_diff   = z_r(i,j,k)-z_r(i,j-1,k)- OneTwelfth* (dZx(i,j,k)+dZx(i,j-1,k));
            Real   Hz_avg   = 0.5_rt * (Hz(i,j,k)+Hz(i,j-1,k));
 
            Real om_v = 2.0_rt / (pm(i,j-1,0)+pm(i,j,0));
            rv(i,j,k,nrhs) = om_v * Hz_avg * (
                            P(i,j-1,k) - P(i,j,k) - HalfGRho *
                            ( (rho(i,j,k)+rho(i,j-1,k))*(z_r(i,j,k)-z_r(i,j-1,k))-
                              OneFifth * ( (dRx(i,j,k)-dRx(i,j-1,k)) * z_r_diff -
                                           (dZx(i,j,k)-dZx(i,j-1,k)) * rho_diff ) ) );
        } // k
    });
}

ReadCheckpointFile()

void REMORA::ReadCheckpointFile ( )

private

{
    amrex::Print() << "Restart from checkpoint " << restart_chkfile << "\n";
 
    // Header
    std::string File(restart_chkfile + "/Header");
 
    VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
 
    Vector<char> fileCharPtr;
    ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
    std::string fileCharPtrString(fileCharPtr.dataPtr());
    std::istringstream is(fileCharPtrString, std::istringstream::in);
 
    std::string line, word;
 
    int chk_ncomp;
 
    // read in title line
    std::getline(is, line);
 
    // read in finest_level
    is >> finest_level;
    GotoNextLine(is);
 
    // read the number of components
    // for each variable we store
 
    // conservative, cell-centered vars
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == NCONS);
 
    // x-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // y-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    // z-velocity on faces
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 1);
 
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 2);
 
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 2);
 
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 3);
 
    is >> chk_ncomp;
    GotoNextLine(is);
    AMREX_ASSERT(chk_ncomp == 3);
 
    // read in array of istep
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            istep[i++] = std::stoi(word);
        }
    }
 
    // read in array of dt
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            dt[i++] = std::stod(word);
        }
    }
 
    // read in array of t_new
    std::getline(is, line);
    {
        std::istringstream lis(line);
        int i = 0;
        while (lis >> word) {
            t_new[i++] = std::stod(word);
        }
    }
 
    for (int lev = 0; lev <= finest_level; ++lev) {
 
        // read in level 'lev' BoxArray from Header
        BoxArray ba;
        ba.readFrom(is);
        GotoNextLine(is);
 
        // create a distribution mapping
        DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
 
        MakeNewLevelFromScratch (lev, t_new[lev], ba, dm);
    }
 
    // read in the MultiFab data
    for (int lev = 0; lev <= finest_level; ++lev)
    {
        BoxList bl2d = grids[lev].boxList();
        for (auto& b : bl2d) {
            b.setRange(2,0);
        }
        BoxArray ba2d(std::move(bl2d));
 
        MultiFab cons(grids[lev],dmap[lev],NCONS,0);
        VisMF::Read(cons, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
        MultiFab::Copy(*cons_new[lev],cons,0,0,NCONS,0);
 
        MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
        VisMF::Read(xvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
        MultiFab::Copy(*xvel_new[lev],xvel,0,0,1,0);
 
        MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
        VisMF::Read(yvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
        MultiFab::Copy(*yvel_new[lev],yvel,0,0,1,0);
 
        MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
        VisMF::Read(zvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
        MultiFab::Copy(*zvel_new[lev],zvel,0,0,1,0);
 
       MultiFab mf_ru(grids[lev],dmap[lev],2,NGROW);
       VisMF::Read(mf_ru, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XRHS"));
       MultiFab::Copy(*(vec_ru[lev]),mf_ru,0,0,2,(vec_ru[lev])->nGrowVect());
 
       MultiFab mf_rv(grids[lev],dmap[lev],2,NGROW);
       VisMF::Read(mf_rv, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YRHS"));
       MultiFab::Copy(*(vec_rv[lev]),mf_rv,0,0,2,(vec_rv[lev])->nGrowVect());
 
       MultiFab mf_ubar(ba2d,dmap[lev],3,NGROW);
       VisMF::Read(mf_ubar, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XBar"));
       MultiFab::Copy(*(vec_ubar[lev]),mf_ubar,0,0,3,(vec_ubar[lev])->nGrowVect());
 
       MultiFab mf_vbar(ba2d,dmap[lev],3,NGROW);
       VisMF::Read(mf_vbar, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YBar"));
       MultiFab::Copy(*(vec_vbar[lev]),mf_vbar,0,0,3,(vec_vbar[lev])->nGrowVect());
 
       VisMF::Read(*(vec_rufrc[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "rufrc"));
       VisMF::Read(*(vec_rvfrc[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "rvfrc"));
 
       VisMF::Read(*(vec_sustr[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "sustr"));
       VisMF::Read(*(vec_svstr[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "svstr"));
 
       VisMF::Read(*(vec_rdrag[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "rdrag"));
 
       VisMF::Read(*(vec_bustr[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "bustr"));
       VisMF::Read(*(vec_bvstr[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "bvstr"));
 
       VisMF::Read(*(vec_DU_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "DU_avg1"));
       VisMF::Read(*(vec_DU_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "DU_avg2"));
       VisMF::Read(*(vec_DV_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "DV_avg1"));
       VisMF::Read(*(vec_DV_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "DV_avg2"));
 
       VisMF::Read(*(vec_zeta[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "zeta"));
       VisMF::Read(*(vec_Zt_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Zt_avg1"));
    }
 
#ifdef REMORA_USE_PARTICLES
   particleData.Restart((amrex::ParGDBBase*)GetParGDB(),restart_chkfile);
#endif
 
#ifdef REMORA_USE_NETCDF
    // Read bdy_data files
    if ( solverChoice.ic_bc_type == IC_BC_Type::Real)
    {
        int ioproc = ParallelDescriptor::IOProcessorNumber();  // I/O rank
        int num_time;
        int num_var;
        Vector<Box> bx_v;
        if (ParallelDescriptor::IOProcessor()) {
            // Open header file and read from it
            std::ifstream bdy_h_file(amrex::MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_H"));
            bdy_h_file >> num_time;
            bdy_h_file >> num_var;
            bdy_h_file >> start_bdy_time;
            bdy_h_file >> bdy_time_interval;
            bdy_h_file >> bdy_width;
            bx_v.resize(4*num_var);
            for (int ivar(0); ivar<num_var; ++ivar) {
                bdy_h_file >> bx_v[4*ivar  ];
                bdy_h_file >> bx_v[4*ivar+1];
                bdy_h_file >> bx_v[4*ivar+2];
                bdy_h_file >> bx_v[4*ivar+3];
            }
 
            // IO size the FABs
            bdy_data_xlo.resize(num_time);
            bdy_data_xhi.resize(num_time);
            bdy_data_ylo.resize(num_time);
            bdy_data_yhi.resize(num_time);
            for (int itime(0); itime<num_time; ++itime) {
                bdy_data_xlo[itime].resize(num_var);
                bdy_data_xhi[itime].resize(num_var);
                bdy_data_ylo[itime].resize(num_var);
                bdy_data_yhi[itime].resize(num_var);
                for (int ivar(0); ivar<num_var; ++ivar) {
                    bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
                    bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
                    bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
                    bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
                }
            }
 
            // Open data file and read from it
            std::ifstream bdy_d_file(amrex::MultiFabFileFullPrefix(0, restart_chkfile, "Level_", "bdy_D"));
            for (int itime(0); itime<num_time; ++itime) {
                for (int ivar(0); ivar<num_var; ++ivar) {
                    bdy_data_xlo[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_xhi[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_ylo[itime][ivar].readFrom(bdy_d_file);
                    bdy_data_yhi[itime][ivar].readFrom(bdy_d_file);
                }
            }
        } // IO
 
        // Broadcast the data
        ParallelDescriptor::Barrier();
        ParallelDescriptor::Bcast(&start_bdy_time,1,ioproc);
        ParallelDescriptor::Bcast(&bdy_time_interval,1,ioproc);
        ParallelDescriptor::Bcast(&bdy_width,1,ioproc);
        ParallelDescriptor::Bcast(&num_time,1,ioproc);
        ParallelDescriptor::Bcast(&num_var,1,ioproc);
 
        // Everyone size their boxes
        bx_v.resize(4*num_var);
 
        ParallelDescriptor::Bcast(bx_v.dataPtr(),bx_v.size(),ioproc);
 
        // Everyone but IO size their FABs
        if (!ParallelDescriptor::IOProcessor()) {
          bdy_data_xlo.resize(num_time);
          bdy_data_xhi.resize(num_time);
          bdy_data_ylo.resize(num_time);
          bdy_data_yhi.resize(num_time);
          for (int itime(0); itime<num_time; ++itime) {
            bdy_data_xlo[itime].resize(num_var);
            bdy_data_xhi[itime].resize(num_var);
            bdy_data_ylo[itime].resize(num_var);
            bdy_data_yhi[itime].resize(num_var);
            for (int ivar(0); ivar<num_var; ++ivar) {
              bdy_data_xlo[itime][ivar].resize(bx_v[4*ivar  ]);
              bdy_data_xhi[itime][ivar].resize(bx_v[4*ivar+1]);
              bdy_data_ylo[itime][ivar].resize(bx_v[4*ivar+2]);
              bdy_data_yhi[itime][ivar].resize(bx_v[4*ivar+3]);
            }
          }
        }
 
        for (int itime(0); itime<num_time; ++itime) {
            for (int ivar(0); ivar<num_var; ++ivar) {
                ParallelDescriptor::Bcast(bdy_data_xlo[itime][ivar].dataPtr(),bdy_data_xlo[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_xhi[itime][ivar].dataPtr(),bdy_data_xhi[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_ylo[itime][ivar].dataPtr(),bdy_data_ylo[itime][ivar].box().numPts(),ioproc);
                ParallelDescriptor::Bcast(bdy_data_yhi[itime][ivar].dataPtr(),bdy_data_yhi[itime][ivar].box().numPts(),ioproc);
            }
        }
    } // init real
#endif
}

ReadParameters()

void REMORA::ReadParameters ( )

private

read in some parameters from inputs file

{
    {
        ParmParse pp;  // Traditionally, max_step and stop_time do not have prefix.
        pp.query("max_step", max_step);
        pp.query("stop_time", stop_time);
    }
 
    ParmParse pp(pp_prefix);
    {
        pp.query("regrid_int", regrid_int);
        pp.query("check_file", check_file);
        pp.query("check_int", check_int);
 
        pp.query("restart", restart_chkfile);
 
        if (pp.contains("data_log"))
        {
            int num_datalogs = pp.countval("data_log");
            datalog.resize(num_datalogs);
            datalogname.resize(num_datalogs);
            pp.queryarr("data_log",datalogname,0,num_datalogs);
            for (int i = 0; i < num_datalogs; i++)
                setRecordDataInfo(i,datalogname[i]);
        }
 
        // Verbosity
        pp.query("v", verbose);
 
        // Frequency of diagnostic output
        pp.query("sum_interval", sum_interval);
        pp.query("sum_period"  , sum_per);
 
        // Time step controls
        pp.query("cfl", cfl);
        pp.query("init_shrink", init_shrink);
        pp.query("change_max", change_max);
 
        pp.query("fixed_dt", fixed_dt);
        pp.query("fixed_fast_dt", fixed_fast_dt);
 
        pp.query("fixed_ndtfast_ratio", fixed_ndtfast_ratio);
 
        // If all three are specified, they must be consistent
        if (fixed_dt > 0. && fixed_fast_dt > 0. &&  fixed_ndtfast_ratio > 0)
        {
            if (fixed_dt / fixed_fast_dt != fixed_ndtfast_ratio)
            {
                amrex::Abort("Dt is over-specfied");
            }
        }
        // If two are specified, initialize fixed_ndtfast_ratio
        else if (fixed_dt > 0. && fixed_fast_dt > 0. &&  fixed_ndtfast_ratio <= 0)
        {
            fixed_ndtfast_ratio = static_cast<int>(fixed_dt / fixed_fast_dt);
        }
 
        pp.query("do_substep", do_substep);
 
        AMREX_ASSERT(cfl > 0. || fixed_dt > 0.);
 
        // We use this to keep track of how many boxes we read in from WRF initialization
        num_files_at_level.resize(max_level+1,0);
 
        // We use this to keep track of how many boxes are specified thru the refinement indicators
        num_boxes_at_level.resize(max_level+1,0);
            boxes_at_level.resize(max_level+1);
 
        // We always have exactly one file at level 0
        num_boxes_at_level[0] = 1;
        boxes_at_level[0].resize(1);
        boxes_at_level[0][0] = geom[0].Domain();
 
        // Output format
        std::string plotfile_type_str = "amrex";
        pp.query("plotfile_type", plotfile_type_str);
        if (plotfile_type_str == "amrex") {
            plotfile_type = PlotfileType::amrex;
        } else if (plotfile_type_str == "netcdf" || plotfile_type_str == "NetCDF") {
            plotfile_type = PlotfileType::netcdf;
#ifdef REMORA_USE_NETCDF
            pp.query("write_history_file",write_history_file);
#endif
        } else {
            amrex::Print() << "User selected plotfile_type = " << plotfile_type_str << std::endl;
            amrex::Abort("Dont know this plotfile_type");
        }
#ifndef REMORA_USE_NETCDF
        if (plotfile_type == PlotfileType::netcdf)
        {
            amrex::Abort("Please compile with NetCDF in order to enable NetCDF plotfiles");
        }
 
#endif
 
#ifdef REMORA_USE_NETCDF
        nc_init_file.resize(max_level+1);
        nc_grid_file.resize(max_level+1);
 
        // NetCDF initialization files -- possibly multiple files at each of multiple levels
        //        but we always have exactly one file at level 0
        for (int lev = 0; lev <= max_level; lev++)
        {
            const std::string nc_file_names = amrex::Concatenate("nc_init_file_",lev,1);
            const std::string nc_bathy_file_names = amrex::Concatenate("nc_grid_file_",lev,1);
 
            if (pp.contains(nc_file_names.c_str()))
            {
                int num_files = pp.countval(nc_file_names.c_str());
                int num_bathy_files = pp.countval(nc_bathy_file_names.c_str());
                if (num_files != num_bathy_files) {
                    amrex::Error("Must have same number of netcdf files for grid info as for solution");
                }
 
                num_files_at_level[lev] = num_files;
                nc_init_file[lev].resize(num_files);
                nc_grid_file[lev].resize(num_files);
 
                pp.queryarr(nc_file_names.c_str()      , nc_init_file[lev]     ,0,num_files);
                pp.queryarr(nc_bathy_file_names.c_str(), nc_grid_file[lev],0,num_files);
            }
        }
        // We only read boundary data at level 0
        pp.query("nc_bdry_file", nc_bdry_file);
 
        // Query the set and total widths for bdy interior ghost cells
        pp.query("bdy_width", bdy_width);
        pp.query("bdy_set_width", bdy_set_width);
        AMREX_ALWAYS_ASSERT(bdy_width >= 0);
        AMREX_ALWAYS_ASSERT(bdy_set_width >= 0);
        AMREX_ALWAYS_ASSERT(bdy_width >= bdy_set_width);
#endif
        pp.query("plot_file_1", plot_file_1);
        pp.query("plot_file_2", plot_file_2);
        pp.query("plot_int_1", plot_int_1);
        pp.query("plot_int_2", plot_int_2);
 
#ifdef REMORA_USE_PARTICLES
        particleData.init_particle_params();
#endif
    }
 
    solverChoice.init_params();
}

refinement_criteria_setup()

void REMORA::refinement_criteria_setup ( )

private

Function to define the refinement criteria based on user input

{
    if (max_level > 0)
    {
        ParmParse pp(pp_prefix);
        Vector<std::string> refinement_indicators;
        pp.queryarr("refinement_indicators",refinement_indicators,0,pp.countval("refinement_indicators"));
 
        for (int i=0; i<refinement_indicators.size(); ++i)
        {
            std::string ref_prefix = pp_prefix + "." + refinement_indicators[i];
 
            ParmParse ppr(ref_prefix);
            RealBox realbox;
            int lev_for_box;
            if (ppr.countval("in_box_lo")) {
                std::vector<Real> box_lo(3), box_hi(3);
                ppr.get("max_level",lev_for_box);
                if (lev_for_box <= max_level)
                {
                    ppr.getarr("in_box_lo",box_lo,0,2);
                    ppr.getarr("in_box_hi",box_hi,0,2);
                    box_lo[2] = geom[0].ProbLo(2);
                    box_hi[2] = geom[0].ProbHi(2);
                    realbox = RealBox(&(box_lo[0]),&(box_hi[0]));
 
                    amrex::Print() << "Reading " << realbox << " at level " << lev_for_box << std::endl;
                    num_boxes_at_level[lev_for_box] += 1;
 
                    const auto* dx  = geom[lev_for_box].CellSize();
                    const Real* plo = geom[lev_for_box].ProbLo();
                    int ilo = static_cast<int>((box_lo[0] - plo[0])/dx[0]);
                    int jlo = static_cast<int>((box_lo[1] - plo[1])/dx[1]);
                    int klo = static_cast<int>((box_lo[2] - plo[2])/dx[2]);
                    int ihi = static_cast<int>((box_hi[0] - plo[0])/dx[0]-1);
                    int jhi = static_cast<int>((box_hi[1] - plo[1])/dx[1]-1);
                    int khi = static_cast<int>((box_hi[2] - plo[2])/dx[2]-1);
                    Box bx(IntVect(ilo,jlo,klo),IntVect(ihi,jhi,khi));
                    if ( (ilo%ref_ratio[lev_for_box-1][0] != 0) || ((ihi+1)%ref_ratio[lev_for_box-1][0] != 0) ||
                         (jlo%ref_ratio[lev_for_box-1][1] != 0) || ((jhi+1)%ref_ratio[lev_for_box-1][1] != 0) )
                         amrex::Error("Fine box is not legit with this ref_ratio");
                    boxes_at_level[lev_for_box].push_back(bx);
                    amrex::Print() << "Saving in 'boxes at level' as " << bx << std::endl;
                } // lev
            }
 
            AMRErrorTagInfo info;
 
            if (realbox.ok()) {
                info.SetRealBox(realbox);
            }
            if (ppr.countval("start_time") > 0) {
                Real ref_min_time; ppr.get("start_time",ref_min_time);
                info.SetMinTime(ref_min_time);
            }
            if (ppr.countval("end_time") > 0) {
                Real ref_max_time; ppr.get("end_time",ref_max_time);
                info.SetMaxTime(ref_max_time);
            }
            if (ppr.countval("max_level") > 0) {
                int ref_max_level; ppr.get("max_level",ref_max_level);
                info.SetMaxLevel(ref_max_level);
            }
 
            if (ppr.countval("value_greater")) {
                int num_val = ppr.countval("value_greater");
                Vector<Real> value(num_val);
                ppr.getarr("value_greater",value,0,num_val);
                std::string field; ppr.get("field_name",field);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GREATER,field,info));
            }
            else if (ppr.countval("value_less")) {
                int num_val = ppr.countval("value_less");
                Vector<Real> value(num_val);
                ppr.getarr("value_less",value,0,num_val);
                std::string field; ppr.get("field_name",field);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::LESS,field,info));
            }
            else if (ppr.countval("adjacent_difference_greater")) {
                int num_val = ppr.countval("adjacent_difference_greater");
                Vector<Real> value(num_val);
                ppr.getarr("adjacent_difference_greater",value,0,num_val);
                std::string field; ppr.get("field_name",field);
                ref_tags.push_back(AMRErrorTag(value,AMRErrorTag::GRAD,field,info));
            }
            else if (realbox.ok())
            {
                ref_tags.push_back(AMRErrorTag(info));
            } else {
                Abort(std::string("Unrecognized refinement indicator for " + refinement_indicators[i]).c_str());
            }
        } // loop over criteria
    } // if max_level > 0
}

RemakeLevel()

void REMORA::RemakeLevel ( int  lev, amrex::Real  time, const amrex::BoxArray &  ba, const amrex::DistributionMapping &  dm  )

overridevirtual

Remake an existing level using provided BoxArray and DistributionMapping and fill with existing fine and coarse data. Overrides the pure virtual function in AmrCore

{
#if (NGROW==2)
    int ngrow_state = ComputeGhostCells(solverChoice.spatial_order)+1;
    int ngrow_vels  = ComputeGhostCells(solverChoice.spatial_order)+1;
#else
    int ngrow_state = ComputeGhostCells(solverChoice.spatial_order)+2;
    int ngrow_vels  = ComputeGhostCells(solverChoice.spatial_order)+2;
#endif
 
    MultiFab* tmp_cons_new; tmp_cons_new->define(ba, dm, NCONS, ngrow_state);
    MultiFab* tmp_cons_old; tmp_cons_old->define(ba, dm, NCONS, ngrow_state);
 
    MultiFab* tmp_xvel_new; tmp_xvel_new->define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
    MultiFab* tmp_xvel_old; tmp_xvel_old->define(convert(ba, IntVect(1,0,0)), dm, 1, ngrow_vels);
 
    MultiFab* tmp_yvel_new; tmp_yvel_new->define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
    MultiFab* tmp_yvel_old; tmp_yvel_old->define(convert(ba, IntVect(0,1,0)), dm, 1, ngrow_vels);
 
    MultiFab* tmp_zvel_new; tmp_zvel_new->define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect(ngrow_vels,ngrow_vels,0));
    MultiFab* tmp_zvel_old; tmp_zvel_old->define(convert(ba, IntVect(0,0,1)), dm, 1, IntVect(ngrow_vels,ngrow_vels,0));
 
    // This will fill the temporary MultiFabs with data from previous fine data as well as coarse where needed
    FillPatch(lev, time, *tmp_cons_new, cons_new, BdyVars::t);
    FillPatch(lev, time, *tmp_xvel_new, xvel_new, BdyVars::u);
    FillPatch(lev, time, *tmp_yvel_new, yvel_new, BdyVars::v);
    FillPatch(lev, time, *tmp_zvel_new, zvel_new, BdyVars::null);
 
    MultiFab::Copy(*tmp_cons_old,*tmp_cons_new,0,0,NCONS,tmp_cons_new[lev].nGrowVect());
    MultiFab::Copy(*tmp_xvel_old,*tmp_xvel_new,0,0,   1,tmp_xvel_new[lev].nGrowVect());
    MultiFab::Copy(*tmp_yvel_old,*tmp_yvel_new,0,0,   1,tmp_yvel_new[lev].nGrowVect());
    MultiFab::Copy(*tmp_zvel_old,*tmp_zvel_new,0,0,   1,tmp_zvel_new[lev].nGrowVect());
 
    std::swap(tmp_cons_new, cons_new[lev]);
    std::swap(tmp_cons_old, cons_old[lev]);
    std::swap(tmp_xvel_new, xvel_new[lev]);
    std::swap(tmp_xvel_old, xvel_old[lev]);
    std::swap(tmp_yvel_new, yvel_new[lev]);
    std::swap(tmp_yvel_old, yvel_old[lev]);
    std::swap(tmp_zvel_new, zvel_new[lev]);
    std::swap(tmp_zvel_old, zvel_old[lev]);
 
    t_new[lev] = time;
    t_old[lev] = time - 1.e200;
 
    init_stuff(lev, ba, dm);
}

remora_advance()

void REMORA::remora_advance	(	int	level,
		amrex::MultiFab &	cons_old,
		amrex::MultiFab &	cons_new,
		amrex::MultiFab &	xvel_old,
		amrex::MultiFab &	yvel_old,
		amrex::MultiFab &	zvel_old,
		amrex::MultiFab &	xvel_new,
		amrex::MultiFab &	yvel_new,
		amrex::MultiFab &	zvel_new,
		amrex::MultiFab &	source,
		const amrex::Geometry	fine_geom,
		const amrex::Real	dt,
		const amrex::Real	time
	)

Interface for advancing the data at one level by one "slow" timestep

resize_stuff()

void REMORA::resize_stuff ( int  lev )

private

{
    vec_z_phys_nd.resize(lev+1);
 
    vec_hOfTheConfusingName.resize(lev+1);
    vec_Zt_avg1.resize(lev+1);
    vec_s_r.resize(lev+1);
    vec_z_w.resize(lev+1);
    vec_z_r.resize(lev+1);
    vec_y_r.resize(lev+1);
    vec_x_r.resize(lev+1);
    vec_Hz.resize(lev+1);
    vec_Huon.resize(lev+1);
    vec_Hvom.resize(lev+1);
    vec_Akv.resize(lev+1);
    vec_Akt.resize(lev+1);
    vec_visc3d_r.resize(lev+1);
    vec_visc2_p.resize(lev+1);
    vec_visc2_r.resize(lev+1);
    vec_diff2.resize(lev+1);
    vec_ru.resize(lev+1);
    vec_rv.resize(lev+1);
    vec_rufrc.resize(lev+1);
    vec_rvfrc.resize(lev+1);
    vec_sustr.resize(lev+1);
    vec_svstr.resize(lev+1);
    vec_rdrag.resize(lev+1);
    vec_bustr.resize(lev+1);
    vec_bvstr.resize(lev+1);
 
    vec_DU_avg1.resize(lev+1);
    vec_DU_avg2.resize(lev+1);
    vec_DV_avg1.resize(lev+1);
    vec_DV_avg2.resize(lev+1);
    vec_rubar.resize(lev+1);
    vec_rvbar.resize(lev+1);
    vec_rzeta.resize(lev+1);
    vec_ubar.resize(lev+1);
    vec_vbar.resize(lev+1);
    vec_zeta.resize(lev+1);
    vec_sstore.resize(lev+1);
 
    vec_pm.resize(lev+1);
    vec_pn.resize(lev+1);
    vec_fcor.resize(lev+1);
 
    vec_rhoS.resize(lev+1);
    vec_rhoA.resize(lev+1);
 
    mapfac_m.resize(lev+1);
    mapfac_u.resize(lev+1);
    mapfac_v.resize(lev+1);
}

restart()

void REMORA::restart

(

)

Restart

{
    ReadCheckpointFile();
 
    // We set this here so that we don't over-write the checkpoint file we just started from
    last_check_file_step = istep[0];
}

rho_eos()

void REMORA::rho_eos	(	const amrex::Box &	bx,
		const amrex::Array4< amrex::Real const > &	state,
		const amrex::Array4< amrex::Real > &	rho,
		const amrex::Array4< amrex::Real > &	rhoA,
		const amrex::Array4< amrex::Real > &	rhoS,
		const amrex::Array4< amrex::Real const > &	Hz,
		const amrex::Array4< amrex::Real const > &	z_w,
		const amrex::Array4< amrex::Real const > &	h,
		const int	N
	)

rho_eos

Parameters

[in]	bx	box for calculation
[in]	state	state holds temp, salt
[out]	rho	density
[out]	rhoA	vertically-averaged density
[out]	rhoS	density perturbation
[in]	Hz
[in]	z_w
[in]	h
[in]	N

{
//
    AMREX_ASSERT(bx.smallEnd(2) == 0 && bx.bigEnd(2) == N);
//
//=======================================================================
//  Linear equation of state.
//=======================================================================
//
//
//-----------------------------------------------------------------------
//  Compute "in situ" density anomaly (kg/m3 - 1000) using the linear
//  equation of state.
//-----------------------------------------------------------------------
//
    Real R0 = solverChoice.R0;
    Real S0 = solverChoice.S0;
    Real T0 = solverChoice.T0;
    Real Tcoef = solverChoice.Tcoef;
    Real Scoef = solverChoice.Scoef;
 
    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        rho(i,j,k)  = R0 - R0*Tcoef*(state(i,j,k,Temp_comp)-T0)
                         + R0*Scoef*(state(i,j,k,Salt_comp)-S0)
                         - 1000.0_rt;
    });
 
//
//-----------------------------------------------------------------------
//  Compute vertical averaged density (rhoA) and density perturbation
//  used (rhoS) in barotropic pressure gradient.
//-----------------------------------------------------------------------
//
    Real cff2 =1.0_rt/solverChoice.rho0;
 
    ParallelFor(makeSlab(bx,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        Real cff0 = rho(i,j,N)*Hz(i,j,N);
        rhoS(i,j,0) = 0.5_rt*cff0*Hz(i,j,N);
        rhoA(i,j,0) = cff0;
 
        for (int k = 1; k <= N; ++k) {
            Real cff1=rho(i,j,N-k)*Hz(i,j,N-k);
            rhoS(i,j,0) += Hz(i,j,N-k)*(rhoA(i,j,0)+0.5_rt*cff1);
            rhoA(i,j,0) += cff1;
        }
 
        Real cff11 =1.0_rt/(z_w(i,j,N)+h(i,j,0,0));
 
        rhoA(i,j,0) *= cff2*cff11;
 
        rhoS(i,j,0) *= 2.0_rt*cff11*cff11*cff2;
    });
}

rhs_t_3d()

void REMORA::rhs_t_3d	(	const amrex::Box &	bx,
		const amrex::Box &	gbx,
		const amrex::Array4< amrex::Real > &	t,
		const amrex::Array4< amrex::Real const > &	tempstore,
		const amrex::Array4< amrex::Real const > &	Huon,
		const amrex::Array4< amrex::Real const > &	Hvom,
		const amrex::Array4< amrex::Real const > &	Hz,
		const amrex::Array4< amrex::Real const > &	pn,
		const amrex::Array4< amrex::Real const > &	pm,
		const amrex::Array4< amrex::Real const > &	W,
		const amrex::Array4< amrex::Real > &	FC,
		int	nrhs,
		int	nnew,
		int	N,
		const amrex::Real	dt_lev
	)

rhs_t_3d

Parameters

[in]	gbx
[in,out]	t
[in]	sstore
[in]	Huon
[in]	Hvom
[in]	Hz
[in]	pn
[in]	pm
[in]	W
[in,out]	FC
[in]	nrhs
[in]	nnew
[in]	N
[in]	dt_lev

{
    //copy the tilebox
    Box tbxp1 = bx;
    Box tbxp2 = bx;
 
    //make only gbx be grown to match multifabs
    tbxp2.grow(IntVect(NGROW,NGROW,0));
    tbxp1.grow(IntVect(NGROW-1,NGROW-1,0));
 
    // Because grad, curv, FX, FE, are all local, do surroundinNodes
    Box utbxp1 = surroundingNodes(tbxp1, 0);
    Box vtbxp1 = surroundingNodes(tbxp1, 1);
    Box ubx = surroundingNodes(bx, 0);
    Box vbx = surroundingNodes(bx, 1);
 
    BoxArray ba_gbx1 = intersect(BoxArray(tbxp1),gbx);
    AMREX_ASSERT((ba_gbx1.size() == 1));
 
    //
    // Scratch space
    //
    FArrayBox fab_grad(tbxp2,1,amrex::The_Async_Arena());
    FArrayBox fab_curv(tbxp2,1,amrex::The_Async_Arena());
 
    FArrayBox fab_FX(tbxp2,1,amrex::The_Async_Arena());
    FArrayBox fab_FE(tbxp2,1,amrex::The_Async_Arena());
 
    auto curv=fab_curv.array();
    auto grad=fab_grad.array();
 
    auto FX=fab_FX.array();
    auto FE=fab_FE.array();
 
    fab_grad.template setVal<RunOn::Device>(0.);
    fab_curv.template setVal<RunOn::Device>(0.);
 
    fab_FX.template setVal<RunOn::Device>(0.);
    fab_FE.template setVal<RunOn::Device>(0.);
 
    ParallelFor(utbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        //should be t index 3
        FX(i,j,k)=sstore(i,j,k,nrhs)-sstore(i-1,j,k,nrhs);
    });
 
    Real cffa=1.0_rt/6.0_rt;
    Real cffb=1.0_rt/3.0_rt;
 
    if(solverChoice.flat_bathymetry) {
 
        if (solverChoice.Hadv_scheme == AdvectionScheme::upstream3) {
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                //Upstream3
                curv(i,j,k)=-FX(i,j,k)+FX(i+1,j,k);
            });
 
            ParallelFor(ubx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                Real max_Huon = std::max(Huon(i,j,k),0.0_rt);
                Real min_Huon = std::min(Huon(i,j,k),0.0_rt);
                FX(i,j,k)=Huon(i,j,k)*0.5*(sstore(i,j,k)+sstore(i-1,j,k))+
                    cffa*(curv(i,j,k)*min_Huon+ curv(i-1,j,k)*max_Huon);
            });
 
        } else if (solverChoice.Hadv_scheme == AdvectionScheme::centered4) {
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                //Centered4
                grad(i,j,k)=0.5*(FX(i,j,k)+FX(i+1,j,k));
            });
 
            ParallelFor(ubx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                FX(i,j,k)=Huon(i,j,k)*0.5*(sstore(i,j,k)+sstore(i-1,j,k))+
                    cffb*(grad(i,j,k)+ grad(i-1,j,k));
            });
 
        } else {
            Error("Not a valid horizontal advection scheme");
        }
 
    } else {
 
        if (solverChoice.Hadv_scheme == AdvectionScheme::upstream3) {
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                //Upstream3
                curv(i,j,k)=-FX(i,j,k)+FX(i+1,j,k);
            });
 
            //HACK to avoid using the wrong index of t (using upstream3)
            ParallelFor(ubx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                Real max_Huon = std::max(Huon(i,j,k),0.0_rt);
                Real min_Huon = std::min(Huon(i,j,k),0.0_rt);
                FX(i,j,k)=Huon(i,j,k)*0.5*(sstore(i,j,k)+sstore(i-1,j,k))-
                    cffa*(curv(i,j,k)*min_Huon+ curv(i-1,j,k)*max_Huon);
            });
 
        } else if (solverChoice.Hadv_scheme == AdvectionScheme::centered4) {
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                //Centered4
                grad(i,j,k)=0.5*(FX(i,j,k)+FX(i+1,j,k));
            });
 
            ParallelFor(ubx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                FX(i,j,k)=Huon(i,j,k)*0.5*(sstore(i,j,k)+sstore(i-1,j,k)-
                                           cffb*(grad(i,j,k)- grad(i-1,j,k)));
            });
 
        } else {
            Error("Not a valid horizontal advection scheme");
        }
    } // flat bathymetry?
 
    ParallelFor(vtbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        //should be t index 3
        FE(i,j,k)=sstore(i,j,k,nrhs)-sstore(i,j-1,k,nrhs);
    });
 
    cffa=1.0_rt/6.0_rt;
    cffb=1.0_rt/3.0_rt;
    if (solverChoice.flat_bathymetry) {
 
        if (solverChoice.Hadv_scheme == AdvectionScheme::upstream3) {
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                curv(i,j,k)=-FE(i,j,k)+FE(i,j+1,k);
            });
 
            ParallelFor(vbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                Real max_Hvom = std::max(Hvom(i,j,k),0.0_rt);
                Real min_Hvom = std::min(Hvom(i,j,k),0.0_rt);
 
                FE(i,j,k)=Hvom(i,j,k)*0.5*(sstore(i,j,k)+sstore(i,j-1,k))+
                    cffa*(curv(i,j,k)*min_Hvom+ curv(i,j-1,k)*max_Hvom);
            });
 
        } else if (solverChoice.Hadv_scheme == AdvectionScheme::centered4) {
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                grad(i,j,k)=0.5*(FE(i,j,k)+FE(i,j+1,k));
            });
 
            ParallelFor(vbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                FE(i,j,k)=Hvom(i,j,k)*0.5*(sstore(i,j,k)+sstore(i,j-1,k))+
                    cffb*(grad(i,j,k)+ grad(i,j-1,k));
            });
 
        } else {
            Error("Not a valid horizontal advection scheme");
        }
 
    } else {
 
        if (solverChoice.Hadv_scheme == AdvectionScheme::upstream3) {
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                curv(i,j,k)=-FE(i,j,k)+FE(i,j+1,k);
            });
 
 
            ParallelFor(vbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                Real max_Hvom = std::max(Hvom(i,j,k),0.0_rt);
                Real min_Hvom = std::min(Hvom(i,j,k),0.0_rt);
 
                FE(i,j,k)=Hvom(i,j,k)*0.5*(sstore(i,j,k)+sstore(i,j-1,k))-
                    cffa*(curv(i,j,k)*min_Hvom+ curv(i,j-1,k)*max_Hvom);
            });
 
        } else if (solverChoice.Hadv_scheme == AdvectionScheme::centered4) {
 
            ParallelFor(tbxp1, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                grad(i,j,k)=0.5*(FE(i,j,k)+FE(i,j+1,k));
            });
 
            ParallelFor(vbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
            {
                FE(i,j,k)=Hvom(i,j,k)*0.5*(sstore(i,j,k)+sstore(i,j-1,k)-
                                           cffb*(grad(i,j,k)- grad(i,j-1,k)));
            });
 
        } else {
            Error("Not a valid horizontal advection scheme");
        }
    }
 
    ParallelFor(bx,
    [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
          //
          //  Add in horizontal advection.
          //
          Real cff = dt_lev*pm(i,j,0)*pn(i,j,0);
          Real cff1=cff*(FX(i+1,j,k)-FX(i,j,k));
          Real cff2=cff*(FE(i,j+1,k)-FE(i,j,k));
          Real cff3=cff1+cff2;
 
          t(i,j,k,nnew) -= cff3;
    });
 
    //-----------------------------------------------------------------------
    //  Time-step vertical advection term.
    //-----------------------------------------------------------------------
    //Check which type of differences:
    //
    //  Fourth-order, central differences vertical advective flux
    //  (Tunits m3/s).
    //
 
    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
              //-----------------------------------------------------------------------
              //  Add in vertical advection.
              //-----------------------------------------------------------------------
 
              Real cff1=0.5_rt;
              Real cff2=7.0_rt/12.0_rt;
              Real cff3=1.0_rt/12.0_rt;
 
              if (k>=1 && k<=N-2)
              {
                      FC(i,j,k)=( cff2*(sstore(i  ,j,k  )+ sstore(i,j,k+1))
                                 -cff3*(sstore(i  ,j,k-1)+ sstore(i,j,k+2)) ) * ( W(i,j,k));
 
              } else {
 
                  FC(i,j,N)=0.0_rt;
 
                  FC(i,j,N-1)=( cff2*sstore(i  ,j,N-1)+ cff1*sstore(i,j,N  )
                               -cff3*sstore(i  ,j,N-2) ) * ( W(i  ,j,N-1));
 
                  FC(i,j,0)=( cff2*sstore(i  ,j,1)+ cff1*sstore(i,j,0)
                             -cff3*sstore(i  ,j,2) ) * ( W(i  ,j,0));
              }
 
    });
 
    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real cff1=dt_lev*pm(i,j,0)*pn(i,j,0);
        Real cff4;
        if(k-1>=0) {
            cff4=FC(i,j,k)-FC(i,j,k-1);
        } else {
            cff4=FC(i,j,k);
        }
 
        t(i,j,k) = (t(i,j,k)-cff1*cff4) / Hz(i,j,k);
    });
}

rhs_uv_2d()

void REMORA::rhs_uv_2d	(	const amrex::Box &	xbx,
		const amrex::Box &	ybx,
		const amrex::Array4< amrex::Real const > &	uold,
		const amrex::Array4< amrex::Real const > &	vold,
		const amrex::Array4< amrex::Real > &	ru,
		const amrex::Array4< amrex::Real > &	rv,
		const amrex::Array4< amrex::Real const > &	Duon,
		const amrex::Array4< amrex::Real const > &	Dvom,
		const int	nrhs
	)

rhs_uv_2d

Parameters

[in]	xbx	Box for operations on x-velocity
[in]	ybx	Box for operations on y-velocity
[in]	ubar
[in]	vbar
[out]	rhs_ubar
[out]	rhs_vbar
[in]	DUon
[in]	DVom
[in]	krhs

{
    //
    // Scratch space
    //
    FArrayBox fab_UFx(growLo(xbx,0,1),1,amrex::The_Async_Arena()); fab_UFx.template setVal<RunOn::Device>(0.);
    FArrayBox fab_UFe(growHi(xbx,1,1),1,amrex::The_Async_Arena()); fab_UFe.template setVal<RunOn::Device>(0.);
    FArrayBox fab_VFe(growLo(ybx,1,1),1,amrex::The_Async_Arena()); fab_VFe.template setVal<RunOn::Device>(0.);
    FArrayBox fab_VFx(growHi(ybx,0,1),1,amrex::The_Async_Arena()); fab_VFx.template setVal<RunOn::Device>(0.);
 
    auto UFx=fab_UFx.array();
    auto UFe=fab_UFe.array();
    auto VFx=fab_VFx.array();
    auto VFe=fab_VFe.array();
 
    // *************************************************************
    // UPDATING U
    // *************************************************************
 
    // Think of the cell-centered box as                              [ 0:nx-1, 0:ny-1] (0,0,0) cc
    //
    // xbx is the x-face-centered box on which we update ubar (with rhs_ubar)  [ 0:nx  , 0:ny-1] (1,0,0) x-faces
    //   to do so requires                                           UFx on    [-1:nx  , 0:ny-1] (0,0,0) cc
    //      which requires                                           ubar on   [-2:nx+2, 0:ny-1] (1,0,0) x-faces
    //       and  requires                                           DUon on   [-2:nx+2, 0:ny-1] (1,0,0) x-faces
    //   to do so requires                                           UFe on    [ 0:nx  , 0:ny  ] (1,1,0) xy-nodes
    //      which requires                                           ubar on   [ 0:nx  ,-2:ny+1] (1,0,0) x-faces
    //       and  requires                                           DVom on   [-2:nx+1, 0:ny-1] (0,1,0) y-faces
 
    //
    // Define UFx, the x-fluxes at cell centers for updating u
    // (Note that grow arguments are (bx, dir, ng)
    //
    ParallelFor(growLo(xbx,0,1),
    [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        Real uxx_i   = ubar(i-1,j,0,krhs)-2.0*ubar(i  ,j,0,krhs)+ubar(i+1,j,0,krhs);
        Real uxx_ip1 = ubar(i  ,j,0,krhs)-2.0*ubar(i+1,j,0,krhs)+ubar(i+2,j,0,krhs);
        Real uxx_avg = uxx_i + uxx_ip1;
 
        Real Huxx_i   = DUon(i-1,j,0)-2.0*DUon(i  ,j,0)+DUon(i+1,j,0);
        Real Huxx_ip1 = DUon(i  ,j,0)-2.0*DUon(i+1,j,0)+DUon(i+2,j,0);
 
        Real cff=1.0/6.0;
        Real ubar_avg = ubar(i  ,j,0,krhs)+ubar(i+1,j,0,krhs);
 
        UFx(i,j,0)=0.25*(ubar_avg-cff*uxx_avg) * (DUon(i,j,0)+ DUon(i+1,j,0)-cff*(Huxx_i+ Huxx_ip1));
    });
 
    //
    // Define UFe, the y-fluxes at nodes for updating u
    //
    ParallelFor(growHi(xbx,1,1),
    [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        //should not include grow cells
        Real uee_j   = ubar(i,j-1,0,krhs)-2.0*ubar(i,j  ,0,krhs)+ubar(i,j+1,0,krhs);
        Real uee_jm1 = ubar(i,j-2,0,krhs)-2.0*ubar(i,j-1,0,krhs)+ubar(i,j  ,0,krhs);
        Real uee_avg = uee_j + uee_jm1;
 
        Real Hvxx_i   = DVom(i-1,j,0)-2.0*DVom(i  ,j,0)+DVom(i+1,j,0);
        Real Hvxx_im1 = DVom(i-2,j,0)-2.0*DVom(i-1,j,0)+DVom(i  ,j,0);
        Real cff=1.0/6.0;
        Real cff1=ubar(i,j  ,0,krhs)+ubar(i,j-1,0,krhs);
        Real cff2=DVom(i,j,0)+DVom(i-1,j,0);
 
        UFe(i,j,0)=0.25*(cff1-uee_avg*cff)*
          (cff2-cff*(Hvxx_i+Hvxx_im1));
    });
 
    //
    //  Add in horizontal advection.
    //
    ParallelFor(makeSlab(xbx,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
    {
         Real cff1=UFx(i,j  ,0)-UFx(i-1,j,0);
         Real cff2=UFe(i,j+1,0)-UFe(i  ,j,0);
 
         rhs_ubar(i,j,0) -= (cff1 + cff2);
    });
 
    // *************************************************************
    // UPDATING V
    // *************************************************************
 
    // Think of the cell-centered box as                              [ 0:nx-1, 0:ny-1] (0,0,0) cc
    //
    // ybx is the y-face-centered box on which we update vbar (with rhs_vbar)  [ 0:nx-1, 0:ny  ] (1,0,0)  y-faces
    //   to do so requires                                  VFe on    [ 0:nx-1,-1:ny  ] (1,1,0)  xy-nodes
    //      which requires                                    vbar on [ 0:nx-1,-2:ny+2] (1,0,0)  y-faces
    //       and  requires                                    DVom on [ 0:nx-1,-2:ny+2] (0,1,0)  x-faces
    //   to do so requires                                  VFx on    [ 0:nx  , 0:ny  ] (0,0,0)  cc
    //      which requires                                    vbar on [-2:nx+1, 0:ny  ] (1,0,0)  y-faces
    //       and  requires                                    DUon on [ 0:nx-1,-2:ny+1] (0,0,0)  y-faces
 
    // Grow ybx by one in high x-direction
    ParallelFor(growHi(ybx,0,1),
    [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        Real cff=1.0/6.0;
        Real vxx_i   = vbar(i-1,j,0,krhs)-2.0*vbar(i  ,j,0,krhs)+vbar(i+1,j,0,krhs);
        Real vxx_im1 = vbar(i-2,j,0,krhs)-2.0*vbar(i-1,j,0,krhs)+vbar(i  ,j,0,krhs);
        Real vxx_avg = vxx_i + vxx_im1;
        //auto vxx_im1 = (i == gbx1.smallEnd(0)) ? vxx(i-1,j,k) :
        //    (vbar(i-2,j,0,krhs)-2.0*vbar(i-1,j,0,krhs)+vbar(i,j,0,krhs));
        //neglecting terms about periodicity since testing only periodic for now
        Real Huee_j   = DUon(i,j-1,0)-2.0*DUon(i,j  ,0)+DUon(i,j+1,0);
        Real Huee_jm1 = DUon(i,j-2,0)-2.0*DUon(i,j-1,0)+DUon(i,j  ,0);
        Real cff1=vbar(i  ,j,0,krhs)+vbar(i-1,j,0,krhs);
        Real cff2=DUon(i,j,0)+DUon(i,j-1,0);
 
        //auto Huee_jm1 = (j == gbx1.smallEnd(1)) ? Huee(i,j-1,k) :
        //    (DUon(i,j-2,k)-2.0*DUon(i,j-1,k)+DUon(i,j,k));
 
        VFx(i,j,0)=0.25*(cff1-vxx_avg*cff)* (cff2-cff*(Huee_j+ Huee_jm1));
    });
 
    ParallelFor(growLo(ybx,1,1),
    [=] AMREX_GPU_DEVICE (int i, int j, int)
    {
        Real vee_j    = vbar(i,j-1,0,krhs)-2.0*vbar(i,j  ,0,krhs)+vbar(i,j+1,0,krhs);
        Real vee_jp1  = vbar(i,j  ,0,krhs)-2.0*vbar(i,j+1,0,krhs)+vbar(i,j+2,0,krhs);
        Real vee_avg  = vee_j + vee_jp1;
 
        Real Hvee_j   = DVom(i,j-1,0)-2.0*DVom(i,j  ,0)+DVom(i,j+1,0);
        Real Hvee_jp1 = DVom(i,j  ,0)-2.0*DVom(i,j+1,0)+DVom(i,j+2,0);
 
        Real cff=1.0/6.0;
        Real cff1=vbar(i,j  ,0,krhs)+vbar(i,j+1,0,krhs);
 
        VFe(i,j,0) = 0.25 * (cff1-vee_avg*cff) * (DVom(i,j  ,0)+ DVom(i,j+1,0) -
                                           cff  * (Hvee_j+ Hvee_jp1));
    });
 
    ParallelFor(makeSlab(ybx,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
    {
        Real cff1=VFx(i+1,j,0)-VFx(i  ,j,0);
        Real cff2=VFe(i  ,j,0)-VFe(i,j-1,0);
 
        rhs_vbar(i,j,0) -= (cff1 + cff2);
    });
 
}

rhs_uv_3d()

void REMORA::rhs_uv_3d	(	const amrex::Box &	xbx,
		const amrex::Box &	ybx,
		const amrex::Array4< amrex::Real const > &	uold,
		const amrex::Array4< amrex::Real const > &	vold,
		const amrex::Array4< amrex::Real > &	ru,
		const amrex::Array4< amrex::Real > &	rv,
		const amrex::Array4< amrex::Real > &	rufrc,
		const amrex::Array4< amrex::Real > &	rvfrc,
		const amrex::Array4< amrex::Real const > &	sustr,
		const amrex::Array4< amrex::Real const > &	svstr,
		const amrex::Array4< amrex::Real const > &	bustr,
		const amrex::Array4< amrex::Real const > &	bvstr,
		const amrex::Array4< amrex::Real const > &	Huon,
		const amrex::Array4< amrex::Real const > &	Hvom,
		const amrex::Array4< amrex::Real const > &	pm,
		const amrex::Array4< amrex::Real const > &	pn,
		const amrex::Array4< amrex::Real const > &	W,
		const amrex::Array4< amrex::Real > &	FC,
		int	nrhs,
		int	N
	)

Calculation of the RHS

rhs_uv_2d

Parameters

[in]	xbx	Box for operations on x-velocity
[in]	ybx	Box for operations on y-velocity
[in]	uold
[in]	vold
[out]	ru
[out]	rv
[out]	rufrc
[out]	rvfrc
[in]	sustr
[in]	svstr
[in]	bustr
[in]	bvstr
[in]	Huon
[in]	Hvom
[in]	pm
[in]	pn
[in]	W
[in,out]	FC
[in]	nrhs
[in]	N

{
    //
    // Scratch space
    //
    FArrayBox fab_UFx(growLo(xbx,0,1),1,amrex::The_Async_Arena()); fab_UFx.template setVal<RunOn::Device>(0.);
    FArrayBox fab_UFe(growHi(xbx,1,1),1,amrex::The_Async_Arena()); fab_UFe.template setVal<RunOn::Device>(0.);
    FArrayBox fab_VFe(growLo(ybx,1,1),1,amrex::The_Async_Arena()); fab_VFe.template setVal<RunOn::Device>(0.);
    FArrayBox fab_VFx(growHi(ybx,0,1),1,amrex::The_Async_Arena()); fab_VFx.template setVal<RunOn::Device>(0.);
 
    auto UFx=fab_UFx.array();
    auto UFe=fab_UFe.array();
    auto VFx=fab_VFx.array();
    auto VFe=fab_VFe.array();
 
    //check this////////////
    const Real Gadv = -0.25;
 
    // *************************************************************
    // UPDATING U
    // *************************************************************
 
    // Think of the cell-centered box as                              [ 0:nx-1, 0:ny-1] (0,0,0) cc
    //
    // xbx is the x-face-centered box on which we update u (with ru)  [ 0:nx  , 0:ny-1] (1,0,0) x-faces
    //   to do so requires                                  UFx on    [-1:nx  , 0:ny-1] (0,0,0) cc
    //      which requires                                  uold on   [-2:nx+2, 0:ny-1] (1,0,0) x-faces
    //       and  requires                                  Huon on   [-2:nx+2, 0:ny-1] (0,0,0) x-faces
    //   to do so requires                                  UFe on    [ 0:nx  , 0:ny  ] (1,1,0) xy-nodes
    //      which requires                                  uold on   [ 0:nx  ,-2:ny+1] (1,0,0) x-faces
    //       and  requires                                  Hvom on   [-2:nx+1, 0:ny-1] (0,1,0) y-faces
 
    //
    // Define UFx, the x-fluxes at cell centers for updating u
    // (Note that grow arguments are (bx, dir, ng)
    //
    ParallelFor(growLo(xbx,0,1), [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real cff1 = uold(i,j,k,nrhs)+uold(i+1,j,k,nrhs);
 
        // Upwinding
        Real cff = (cff1 > 0.0) ? uold(i-1,j,k,nrhs)-2.0*uold(i  ,j,k,nrhs)+uold(i+1,j,k,nrhs) :
                                  uold(i  ,j,k,nrhs)-2.0*uold(i+1,j,k,nrhs)+uold(i+2,j,k,nrhs);
 
        Real Huxx_i   = Huon(i-1,j,k)-2.0*Huon(i  ,j,k)+Huon(i+1,j,k);
        Real Huxx_ip1 = Huon(i  ,j,k)-2.0*Huon(i+1,j,k)+Huon(i+2,j,k);
        Real Huxx_avg = 0.5 * (Huxx_i + Huxx_ip1);
 
        Real Huon_avg = (Huon(i,j,k) + Huon(i+1,j,k));
 
        UFx(i,j,k) = 0.25*(cff1+Gadv*cff) * ( Huon_avg + Gadv*Huxx_avg );
    });
 
    //
    // Define UFe, the y-fluxes at nodes for updating u
    //
    ParallelFor(growHi(xbx,1,1), [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real cff1 = uold(i,j,k,nrhs) + uold(i  ,j-1,k,nrhs);
        Real cff2 = Hvom(i,j,k)      + Hvom(i-1,j  ,k);
 
        // Upwinding
        Real cff = (cff2 > 0.0) ?  uold(i,j-2,k,nrhs) - 2.0*uold(i,j-1,k,nrhs) + uold(i  ,j,k,nrhs) :
                                   uold(i,j-1,k,nrhs) - 2.0*uold(i,j  ,k,nrhs) + uold(i,j+1,k,nrhs);
 
        Real Hvxx_i   = Hvom(i-1,j,k)-2.0*Hvom(i  ,j,k)+Hvom(i+1,j,k);
        Real Hvxx_im1 = Hvom(i-2,j,k)-2.0*Hvom(i-1,j,k)+Hvom(i  ,j,k);
 
        UFe(i,j,k) = 0.25 * (cff1+Gadv*cff)* (cff2+Gadv*0.5*(Hvxx_i + Hvxx_im1));
    });
 
    //
    // Define the RHS for u by differencing fluxes
    //
    ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
          ru(i,j,k,nrhs) -= ( (UFx(i,j,k)-UFx(i-1,j,k)) + (UFe(i,j+1,k)-UFe(i  ,j,k)) );
    });
 
    ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
          //-----------------------------------------------------------------------
          //  Add in vertical advection.
          //-----------------------------------------------------------------------
          Real cff1=9.0/16.0;
          Real cff2=1.0/16.0;
 
          if (k>=1 && k<=N-2)
          {
                  FC(i,j,k)=( cff1*(uold(i  ,j,k  ,nrhs)+ uold(i,j,k+1,nrhs))
                             -cff2*(uold(i  ,j,k-1,nrhs)+ uold(i,j,k+2,nrhs)) )*
                                ( cff1*(   W(i  ,j,k)+ W(i-1,j,k))
                                 -cff2*(   W(i+1,j,k)+ W(i-2,j,k)) );
          }
          else // this needs to be split up so that the following can be concurrent
          {
              FC(i,j,N)=0.0;
 
              FC(i,j,N-1)=( cff1*(uold(i  ,j,N-1,nrhs)+ uold(i,j,N  ,nrhs))
                           -cff2*(uold(i  ,j,N-2,nrhs)+ uold(i,j,N  ,nrhs)) )*
                              ( cff1*(   W(i  ,j,N-1)+ W(i-1,j,N-1))
                               -cff2*(   W(i+1,j,N-1)+ W(i-2,j,N-1)) );
 
              FC(i,j,0)=( cff1*(uold(i  ,j,0,nrhs)+ uold(i,j,1,nrhs))
                         -cff2*(uold(i  ,j,0,nrhs)+ uold(i,j,2,nrhs)) )*
                            ( cff1*(   W(i  ,j,0)+ W(i-1,j,0))
                             -cff2*(   W(i+1,j,0)+ W(i-2,j,0)) );
 
              //              FC(i,0,-1)=0.0;
          }
    });
 
    ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real cff = (k >= 1) ? FC(i,j,k)-FC(i,j,k-1) : FC(i,j,k);
 
        ru(i,j,k,nrhs) -= cff;
    });
 
    Gpu::synchronize();
 
    AMREX_ASSERT(xbx.smallEnd(2) == 0 && xbx.bigEnd(2) == N);
    ParallelFor(makeSlab(xbx,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
    {
       for (int k = 0; k <= N; ++k)
       {
          rufrc(i,j,0) += ru(i,j,k,nrhs);
 
          Real om_u = 2.0 / (pm(i-1,j,0)+pm(i,j,0));
          Real on_u = 2.0 / (pn(i-1,j,0)+pn(i,j,0));
          Real cff  = om_u * on_u;
 
          Real cff1 = (k == N) ?  sustr(i,j,0)*cff : 0.0;
          Real cff2 = (k == 0) ? -bustr(i,j,0)*cff : 0.0;
 
          rufrc(i,j,0) += cff1+cff2;
       }
    });
 
    // *************************************************************
    // UPDATING V
    // *************************************************************
 
    // Think of the cell-centered box as                              [ 0:nx-1, 0:ny-1] (0,0,0) cc
    //
    // ybx is the y-face-centered box on which we update v (with rv)  [ 0:nx-1, 0:ny  ] (1,0,0)  y-faces
    //   to do so requires                                  VFe on    [ 0:nx-1,-1:ny  ] (1,1,0)  xy-nodes
    //      which requires                                    vold on [ 0:nx-1,-2:ny+2] (1,0,0)  y-faces
    //       and  requires                                    Hvom on [ 0:nx-1,-2:ny+2] (0,1,0)  x-faces
    //   to do so requires                                  VFx on    [ 0:nx  , 0:ny  ] (0,0,0)  cc
    //      which requires                                    vold on [-2:nx+1, -:ny  ] (1,0,0)  y-faces
    //       and  requires                                    Hvom on [ 0:nx-1,-2:ny+1] (0,0,0)  y-faces
 
    // Grow ybx by one in low y-direction
    ParallelFor(growLo(ybx,1,1), [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real cff1=vold(i,j,k,nrhs)+vold(i,j+1,k,nrhs);
 
        // Upwinding
        Real cff = (cff1 > 0.0) ? vold(i,j-1,k,nrhs)-2.0*vold(i,j,k,nrhs)+ vold(i,j+1,k,nrhs) :
                                  vold(i,j,k,nrhs)-2.0*vold(i,j+1,k,nrhs)+ vold(i,j+2,k,nrhs);
 
        Real Hvee_j   = Hvom(i,j-1,k)-2.0*Hvom(i,j  ,k)+Hvom(i,j+1,k);
        Real Hvee_jp1 = Hvom(i,j  ,k)-2.0*Hvom(i,j+1,k)+Hvom(i,j+2,k);
 
        VFe(i,j,k) = 0.25 * (cff1+Gadv*cff) * ( Hvom(i,j  ,k)+ Hvom(i,j+1,k) + 0.5 * Gadv * (Hvee_j + Hvee_jp1) );
    });
 
    // Grow ybx by one in high x-direction
    ParallelFor(growHi(ybx,0,1), [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real cff1 = vold(i,j,k,nrhs) + vold(i-1,j  ,k,nrhs);
        Real cff2 = Huon(i,j,k)      + Huon(i  ,j-1,k);
 
        // Upwinding
        Real cff = (cff2 > 0.0) ? vold(i-2,j,k,nrhs)-2.0*vold(i-1,j,k,nrhs)+vold(i  ,j,k,nrhs) :
                                  vold(i-1,j,k,nrhs)-2.0*vold(i  ,j,k,nrhs)+vold(i+1,j,k,nrhs);
 
        Real Huee_j   = Huon(i,j-1,k)-2.0*Huon(i,j  ,k)+Huon(i,j+1,k);
        Real Huee_jm1 = Huon(i,j-2,k)-2.0*Huon(i,j-1,k)+Huon(i,j  ,k);
 
        VFx(i,j,k) = 0.25*(cff1+Gadv*cff)* (cff2+Gadv*0.5*(Huee_j + Huee_jm1));
    });
 
    ParallelFor(ybx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
          rv(i,j,k,nrhs) -= ( (VFx(i+1,j,k)-VFx(i,j,k)) + (VFe(i,j,k)-VFe(i,j-1,k)) );
    });
 
    ParallelFor(ybx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
          Real cff1=9.0/16.0;
          Real cff2=1.0/16.0;
          if (k>=1 && k<=N-2)
          {
              FC(i,j,k)=( cff1*(vold(i,j,k  ,nrhs)+ vold(i,j,k+1,nrhs))
                         -cff2*(vold(i,j,k-1,nrhs)+ vold(i,j,k+2,nrhs)) )*
                            ( cff1*(W(i,j  ,k)+ W(i,j-1,k))
                             -cff2*(W(i,j+1,k)+ W(i,j-2,k)) );
          }
          else // this needs to be split up so that the following can be concurrent
          {
              FC(i,j,N)=0.0;
              FC(i,j,N-1)=( cff1*(vold(i,j,N-1,nrhs)+ vold(i,j,N  ,nrhs))
                           -cff2*(vold(i,j,N-2,nrhs)+ vold(i,j,N  ,nrhs)) )*
                              ( cff1*(W(i,j  ,N-1)+ W(i,j-1,N-1))
                               -cff2*(W(i,j+1,N-1)+ W(i,j-2,N-1)) );
              FC(i,j,0)=( cff1*(vold(i,j,0,nrhs)+ vold(i,j,1,nrhs))
                             -cff2*(vold(i,j,0,nrhs)+ vold(i,j,2,nrhs)) )*
                            ( cff1*(W(i,j  ,0)+ W(i,j-1,0))
                             -cff2*(W(i,j+1,0)+ W(i,j-2,0)) );
              //              FC(i,0,-1)=0.0;
          }
    }); Gpu::synchronize();
 
    ParallelFor(ybx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real cff = (k >= 1) ? FC(i,j,k)-FC(i,j,k-1) : FC(i,j,k);
 
        rv(i,j,k,nrhs) -= cff;
    });
 
    Gpu::synchronize();
 
    AMREX_ASSERT(ybx.smallEnd(2) == 0 && ybx.bigEnd(2) == N);
    ParallelFor(makeSlab(ybx,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int)
    {
       for (int k = 0; k <= N; ++k)
       {
          rvfrc(i,j,0) += rv(i,j,k,nrhs);
 
          Real om_v = 2.0_rt / (pm(i,j-1,0)+pm(i,j,0));
          Real on_v = 2.0_rt / (pn(i,j-1,0)+pn(i,j,0));
          Real cff = om_v * on_v;
 
          Real cff1 = (k == N) ?  svstr(i,j,0)*cff : 0.0_rt;
          Real cff2 = (k == 0) ? -bvstr(i,j,0)*cff : 0.0_rt;
 
          rvfrc(i,j,0) += cff1+cff2;
       }
    });
}

set_2darrays()

void REMORA::set_2darrays

(

int

lev

)

{
    std::unique_ptr<MultiFab>& mf_x_r = vec_x_r[lev];
    std::unique_ptr<MultiFab>& mf_y_r = vec_y_r[lev];
    auto N = Geom(lev).Domain().size()[2]-1; // Number of vertical "levs" aka, NZ
 
    for ( MFIter mfi(*(mf_x_r), TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
 
      Array4<Real> const& x_r = (mf_x_r)->array(mfi);
      Array4<Real> const& y_r = (mf_y_r)->array(mfi);
      const Box& bx = mfi.growntilebox();
      const auto & geomdata = Geom(lev).data();
      Gpu::synchronize();
      amrex::ParallelFor(amrex::makeSlab(bx,2,0),
      [=] AMREX_GPU_DEVICE (int i, int j, int  )
      {
        const auto prob_lo         = geomdata.ProbLo();
        const auto dx              = geomdata.CellSize();
 
        x_r(i,j,0) = prob_lo[0] + (i + 0.5) * dx[0];
        y_r(i,j,0) = prob_lo[1] + (j + 0.5) * dx[1];
        //        const Real z = prob_lo[2] + (k + 0.5) * dx[2];
 
      });
    }
 
    MultiFab* U_old = xvel_new[lev];
    MultiFab* V_old = yvel_new[lev];
    std::unique_ptr<MultiFab>& mf_ubar = vec_ubar[lev];
    std::unique_ptr<MultiFab>& mf_vbar = vec_vbar[lev];
    std::unique_ptr<MultiFab>& mf_Hz  = vec_Hz[lev];
    int nstp = 0;
    int kstp = 0;
    int knew = 0;
 
    for ( MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
        Array4<Real> const& ubar = (mf_ubar)->array(mfi);
        Array4<Real> const& vbar = (mf_vbar)->array(mfi);
 
        Array4<const Real> const& Hz       = mf_Hz->const_array(mfi);
        Array4<const Real> const& u        = U_old->const_array(mfi);
        Array4<const Real> const& v        = V_old->const_array(mfi);
 
        Box ubx2 = mfi.nodaltilebox(0); ubx2.grow(IntVect(NGROW  ,NGROW  ,0)); // x-face-centered, grown by 2
        Box vbx2 = mfi.nodaltilebox(1); vbx2.grow(IntVect(NGROW  ,NGROW  ,0)); // y-face-centered, grown by 2
 
        amrex::ParallelFor(makeSlab(ubx2,2,0),
        [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                Real CF = 0.;
                Real sum_of_hz = 0.;
 
                for (int k=0; k<=N; k++) {
                    Real avg_hz = 0.5*(Hz(i,j,k)+Hz(i-1,j,k));
                    sum_of_hz += avg_hz;
                    CF += avg_hz*u(i,j,k,nstp);
                }
                ubar(i,j,0,kstp) = CF / sum_of_hz;
                ubar(i,j,0,knew) = CF / sum_of_hz;
            });
 
        amrex::ParallelFor(makeSlab(vbx2,2,0),
        [=] AMREX_GPU_DEVICE (int i, int j, int )
            {
                Real CF = 0.;
                Real sum_of_hz = 0.;
 
                for(int k=0; k<=N; k++) {
                    Real avg_hz = 0.5*(Hz(i,j,k)+Hz(i,j-1,k));
                    sum_of_hz += avg_hz;
                    CF += avg_hz*v(i,j,k,nstp);
                }
                vbar(i,j,0,kstp) = CF / sum_of_hz;
                vbar(i,j,0,knew) = CF / sum_of_hz;
            });
    }
 
    // DEBUGGING NOTE -- DoublyPeriodic fails if these are commented out
    const Real time = 0.0;
    FillPatch(lev,time, *vec_ubar[lev], GetVecOfPtrs(vec_ubar), BdyVars::ubar);
    FillPatch(lev,time, *vec_vbar[lev], GetVecOfPtrs(vec_vbar), BdyVars::vbar);
}

set_bathymetry()

void REMORA::set_bathymetry

(

int

lev

)

{
    if (solverChoice.ic_bc_type == IC_BC_Type::Custom) {
        init_custom_bathymetry(geom[lev], *vec_hOfTheConfusingName[lev], solverChoice);
 
#ifdef REMORA_USE_NETCDF
    } else if (solverChoice.ic_bc_type == IC_BC_Type::Real) {
        amrex::Print() << "Calling init_bathymetry_from_netcdf " << std::endl;
        init_bathymetry_from_netcdf(lev);
        amrex::Print() << "Bathymetry loaded from netcdf file \n " << std::endl;
#endif
    } else {
        Abort("Don't know this ic_bc_type!");
    }
 
    // HACK -- SHOULD WE ALWAYS DO THIS??
    vec_Zt_avg1[lev]->setVal(0.0);
 
    Real time = 0.0;
    FillPatch(lev, time, *vec_Zt_avg1[lev], GetVecOfPtrs(vec_Zt_avg1));
    FillPatch(lev, time, *vec_hOfTheConfusingName[lev], GetVecOfPtrs(vec_hOfTheConfusingName));
}

Here is the call graph for this function:

set_coriolis()

void REMORA::set_coriolis

(

int

lev

)

                            {
    if (solverChoice.use_coriolis) {
        if (solverChoice.coriolis_type == Cor_Type::Custom) {
            init_custom_coriolis(geom[lev], *vec_fcor[lev], solverChoice);
        } else if (solverChoice.coriolis_type == Cor_Type::Beta_Plane) {
            init_beta_plane_coriolis(lev);
#ifdef REMORA_USE_NETCDF
        } else if (solverChoice.coriolis_type == Cor_Type::Real) {
            amrex::Print() << "Calling init_coriolis_from_netcdf " << std::endl;
            init_coriolis_from_netcdf(lev);
            amrex::Print() << "Coriolis loaded from netcdf file \n" << std::endl;
#endif
        } else {
            Abort("Don't know this coriolis_type!");
        }
 
        Real time = 0.0;
        FillPatch(lev, time, *vec_fcor[lev], GetVecOfPtrs(vec_fcor));
    }
}

Here is the call graph for this function:

set_drag()

void REMORA::set_drag

(

int

lev

)

set_hmixcoef()

void REMORA::set_hmixcoef

(

int

lev

)

{
    init_custom_hmix(geom[lev], *vec_visc2_p[lev], *vec_visc2_r[lev], *vec_diff2[lev], solverChoice);
 
    Real time = 0.0;
    FillPatch(lev, time, *vec_visc2_p[lev], GetVecOfPtrs(vec_visc2_p));
    FillPatch(lev, time, *vec_visc2_r[lev], GetVecOfPtrs(vec_visc2_r));
    FillPatch(lev, time, *vec_diff2[lev]  , GetVecOfPtrs(vec_diff2));
}

Here is the call graph for this function:

set_smflux()

void REMORA::set_smflux	(	int	lev,
		amrex::Real	time
	)

{
    init_custom_smflux(geom[lev], time, *vec_sustr[lev], *vec_svstr[lev], solverChoice);
 
    // FillPatch(lev, time, *vec_sustr[lev], GetVecOfPtrs(vec_sustr));
    // FillPatch(lev, time, *vec_svstr[lev], GetVecOfPtrs(vec_svstr));
}

Here is the call graph for this function:

set_vmix()

void REMORA::set_vmix

(

int

lev

)

                        {
    init_custom_vmix(geom[lev], *vec_Akv[lev], *vec_Akt[lev], *vec_z_w[lev], solverChoice);
 
    Real time = 0.0;
    FillPatch(lev, time, *vec_Akv[lev], GetVecOfPtrs(vec_Akv));
    FillPatch(lev, time, *vec_Akt[lev], GetVecOfPtrs(vec_Akt));
}

Here is the call graph for this function:

set_weights()

void REMORA::set_weights

(

int

lev

)

                               {
 
    Real gamma, scale;
    Real wsum, shift, cff;
 
    //HACK should possibly store fixed_ndtfast elsewhere
    int ndtfast=fixed_ndtfast_ratio>0 ? fixed_ndtfast_ratio : static_cast<int>(fixed_fast_dt / fixed_dt);
 
    //From mod_scalars
    Real Falpha = 2.0_rt;
    Real Fbeta = 4.0_rt;
    Real Fgamma = 0.284_rt;
 
    vec_weight1.resize(2*ndtfast+1);
    vec_weight2.resize(2*ndtfast+1);
 
    auto weight1 = vec_weight1.dataPtr();
    auto weight2 = vec_weight2.dataPtr();
 
//
//=======================================================================
//  Compute time-averaging filter for barotropic fields.
//=======================================================================
//
//  Initialize both sets of weights to zero.
//
    nfast=0;
    for(int i=1;i<=2*ndtfast;i++) {
        weight1[i-1]=0.0_rt;
        weight2[i-1]=0.0_rt;
    }
//
//-----------------------------------------------------------------------
//  Power-law shape filters.
//-----------------------------------------------------------------------
//
//  The power-law shape filters are given by:
//
//     F(xi)=xi^Falpha*(1-xi^Fbeta)-Fgamma*xi
//
//  where xi=scale*i/ndtfast; and scale, Falpha, Fbeta, Fgamma, and
//  normalization are chosen to yield the correct zeroth-order
//  (normalization), first-order (consistency), and second-order moments,
//  resulting in overall second-order temporal accuracy for time-averaged
//  barotropic motions resolved by baroclinic time step.
//
      scale=(Falpha+1.0_rt)*(Falpha+Fbeta+1.0_rt) /
        ((Falpha+2.0_rt)*(Falpha+Fbeta+2.0_rt)*Real(ndtfast));
    //
    //  Find center of gravity of the primary weighting shape function and
    //  iteratively adjust "scale" to place the  centroid exactly at
    //  "ndtfast".
    //
    gamma = Fgamma*max(0.0_rt, 1.0_rt-10.0_rt/Real(ndtfast));
 
    for (int iter=1;iter<=16;iter++) {
        nfast=0;
        for(int i=1;i<=2*ndtfast;i++) {
            cff=scale*Real(i);
 
            weight1[i-1]=pow(cff,Falpha)-pow(cff,(Falpha+Fbeta))-gamma*cff;
 
            if (weight1[i-1] > 0.0_rt) {
                nfast=i;
            }
 
            if ( (nfast>0) && (weight1[i-1] < 0.0_rt) ) {
                weight1[i-1] = 0.0_rt;
            }
        }
        wsum  = 0.0_rt;
        shift = 0.0_rt;
        for(int i=1;i<=nfast;i++) {
            wsum=wsum+weight1[i-1];
            shift=shift+weight1[i-1]*Real(i);
        }
        scale *= shift/(wsum*Real(ndtfast));
    }
//
//-----------------------------------------------------------------------
//  Post-processing of primary weights.
//-----------------------------------------------------------------------
//
//  Although it is assumed that the initial settings of the primary
//  weights has its center of gravity "reasonably close" to NDTFAST,
//  it may be not so according to the discrete rules of integration.
//  The following procedure is designed to put the center of gravity
//  exactly to NDTFAST by computing mismatch (NDTFAST-shift) and
//  applying basically an upstream advection of weights to eliminate
//  the mismatch iteratively. Once this procedure is complete primary
//  weights are normalized.
//
//  Find center of gravity of the primary weights and subsequently
//  calculate the mismatch to be compensated.
//
    for (int iter=1;iter<=ndtfast;iter++) {
        wsum  = 0.0_rt;
        shift = 0.0_rt;
        for(int i=1;i<=nfast;i++) {
            wsum=wsum+weight1[i-1];
            shift=shift+Real(i)*weight1[i-1];
        }
        shift=shift/wsum;
        cff=Real(ndtfast)-shift;
        //
        //  Apply advection step using either whole, or fractional shifts.
        //  Notice that none of the four loops here is reversible.
        //
        if (cff > 1.0_rt) {
            nfast=nfast+1;
            for (int i=nfast;i>=2;i--) {
                weight1[i-1]=weight1[i-1-1];
            }
            weight1[1-1] = 0.0_rt;
        } else if (cff> 0.0_rt) {
            wsum=1.0_rt-cff;
            for (int i=nfast;i>=2;i--) {
                weight1[i-1]=wsum*weight1[i-1]+cff*weight1[i-1-1];
            }
            weight1[1-1]=wsum*weight1[1-1];
        } else if (cff < -1.0_rt) {
            nfast=nfast-1;
            for (int i=1;i<=nfast;i++) {
                weight1[i-1]=weight1[i+1-1];
            }
            weight1[nfast+1-1] = 0.0_rt;
        } else if (cff < 0.0_rt) {
            wsum=1.0_rt+cff;
            for (int i=1;i<=nfast-1;i++) {
                weight1[i-1]=wsum*weight1[i-1]-cff*weight1[i+1-1];
            }
            weight1[nfast-1]=wsum*weight1[nfast-1];
        }
    }
 
    //  Set SECONDARY weights assuming that backward Euler time step is used
    //  for free surface.  Notice that array weight2[i] is assumed to
    //  have all-zero status at entry in this segment of code.
    for(int j=1;j<=nfast;j++) {
        cff=weight1[j-1];
        for(int i=1;i<=j;i++) {
            weight2[i-1]=weight2[i-1]+cff;
        }
    }
 
    //
    //  Normalize both set of weights.
    //
    wsum = 0.0_rt;
    cff  = 0.0_rt;
    for(int i=1;i<=nfast;i++) {
        wsum=wsum+weight1[i-1];
        cff=cff+weight2[i-1];
    }
 
    wsum = 1.0_rt / wsum;
    cff  = 1.0_rt / cff;
 
    for(int i=1;i<=nfast;i++) {
        weight1[i-1]=wsum*weight1[i-1];
        weight2[i-1]=cff*weight2[i-1];
    }
//
//  Report weights.
//
#if 0
    Real cff1, cff2;
    if (ParallelDescriptor::IOProcessor()) {
        Print().SetPrecision(18)<<ParallelDescriptor::NProcs()<<"  "<<ndtfast<<"  "<<nfast<<"  "<<wsum<<std::endl;
        cff=0.0_rt;
        cff1=0.0_rt;
        cff2=0.0_rt;
        wsum=0.0_rt;
        shift=0.0_rt;
        for(int i=1;i<=nfast;i++) {
          cff=cff+weight1[i-1];
          cff1=cff1+weight1[i-1]*Real(i);
          cff2=cff2+weight1[i-1]*Real(i*i);
          wsum=wsum+weight2[i-1];
          shift=shift+weight2[i-1]*(Real(i)-0.5_rt);
          Print().SetPrecision(18)<<"i="<<i<<"  "<<weight1[i-1]<<"  "<<weight2[i-1]<<"  "<<cff<<"  "<<wsum<<std::endl;
        }
        cff1=cff1/Real(ndtfast);
        cff2=cff2/(Real(ndtfast)*Real(ndtfast));
        shift=shift/Real(ndtfast);
        Print().SetPrecision(18)<<ndtfast <<"  "<< nfast<<"  "<<Real(nfast)/Real(ndtfast)<<std::endl;
        Print().SetPrecision(18)<<cff1<<"  "<<cff2<<"  "<<shift<<"  "<<cff<<"  "<<wsum<<"  "<<Fgamma<<"  "<<gamma<<std::endl;
      if (cff2<1.0001_rt) Print()<<"\n\n\n"<<std::endl;
      }
#endif
}

setPlotVariables()

void REMORA::setPlotVariables ( const std::string &  pp_plot_var_names, amrex::Vector< std::string > &  plot_var_names  )

private

set which variables and derived quantities go into plotfiles

{
    ParmParse pp(pp_prefix);
 
    if (pp.contains(pp_plot_var_names.c_str()))
    {
        std::string nm;
 
        int nPltVars = pp.countval(pp_plot_var_names.c_str());
 
        for (int i = 0; i < nPltVars; i++)
        {
            pp.get(pp_plot_var_names.c_str(), nm, i);
 
            // Add the named variable to our list of plot variables
            // if it is not already in the list
            if (!containerHasElement(plot_var_names, nm)) {
                plot_var_names.push_back(nm);
            }
        }
    } else {
        //
        // The default is to add none of the variables to the list
        //
        plot_var_names.clear();
    }
 
    // Get state variables in the same order as we define them,
    // since they may be in any order in the input list
    Vector<std::string> tmp_plot_names;
 
    for (int i = 0; i < NCONS; ++i) {
        if ( containerHasElement(plot_var_names, cons_names[i]) ) {
            tmp_plot_names.push_back(cons_names[i]);
        }
    }
    // Check for velocity since it's not in cons_names
    // If we are asked for any velocity component, we will need them all
    if (containerHasElement(plot_var_names, "x_velocity") ||
        containerHasElement(plot_var_names, "y_velocity") ||
        containerHasElement(plot_var_names, "z_velocity")) {
        tmp_plot_names.push_back("x_velocity");
        tmp_plot_names.push_back("y_velocity");
        tmp_plot_names.push_back("z_velocity");
    }
 
    // If we are asked for any location component, we will provide them all
    if (containerHasElement(plot_var_names, "x_cc") ||
        containerHasElement(plot_var_names, "y_cc") ||
        containerHasElement(plot_var_names, "z_cc")) {
        tmp_plot_names.push_back("x_cc");
        tmp_plot_names.push_back("y_cc");
        tmp_plot_names.push_back("z_cc");
    }
 
    for (int i = 0; i < derived_names.size(); ++i) {
        if ( containerHasElement(plot_var_names, derived_names[i]) ) {
#ifdef REMORA_USE_PARTICLES
            if (particleData.use_tracer_particles || (derived_names[i] != "tracer_particle_count")) {
#endif
               tmp_plot_names.push_back(derived_names[i]);
#ifdef REMORA_USE_PARTICLES
            }
#endif
        } // if
    } // i
 
    // Check to see if we found all the requested variables
    for (auto plot_name : plot_var_names) {
      if (!containerHasElement(tmp_plot_names, plot_name)) {
           Warning("\nWARNING: Requested to plot variable '" + plot_name + "' but it is not available");
      }
    }
    plot_var_names = tmp_plot_names;
}

Here is the call graph for this function:

setRecordDataInfo()

void REMORA::setRecordDataInfo ( int  i, const std::string &  filename  )

inlineprivate

    {
        if (amrex::ParallelDescriptor::IOProcessor())
        {
            datalog[i] = std::make_unique<std::fstream>();
            datalog[i]->open(filename.c_str(),std::ios::out|std::ios::app);
            if (!datalog[i]->good()) {
                amrex::FileOpenFailed(filename);
            }
        }
        amrex::ParallelDescriptor::Barrier("REMORA::setRecordDataInfo");
    }

setup_step()

void REMORA::setup_step	(	int	lev,
		amrex::Real	time,
		amrex::Real	dt_lev
	)

Set everything up for a step on a level

{
    BL_PROFILE("REMORA::setup_step()");
 
    MultiFab& S_old = *cons_old[lev];
    MultiFab& S_new = *cons_new[lev];
 
    MultiFab& U_old = *xvel_old[lev];
    MultiFab& V_old = *yvel_old[lev];
    MultiFab& W_old = *zvel_old[lev];
 
    MultiFab& U_new = *xvel_new[lev];
    MultiFab& V_new = *yvel_new[lev];
    MultiFab& W_new = *zvel_new[lev];
 
    int nvars = S_old.nComp();
 
    // Fill ghost cells/faces at old time
    FillPatch(lev, time, *cons_old[lev], cons_old, BdyVars::t);
    FillPatch(lev, time, *xvel_old[lev], xvel_old, BdyVars::u);
    FillPatch(lev, time, *yvel_old[lev], yvel_old, BdyVars::v);
    FillPatch(lev, time, *zvel_old[lev], zvel_old, BdyVars::null);
 
    //////////    //pre_step3d corrections to boundaries
 
    const BoxArray&            ba = S_old.boxArray();
    const DistributionMapping& dm = S_old.DistributionMap();
 
    const int nrhs  = 0;
    const int nstp  = 0;
 
    // Place-holder for source array -- for now just set to 0
    MultiFab source(ba,dm,nvars,1);
    source.setVal(0.0);
 
    //-----------------------------------------------------------------------
    //  Time step momentum equation
    //-----------------------------------------------------------------------
 
    //Only used locally, probably should be rearranged into FArrayBox declaration
    MultiFab mf_AK(ba,dm,1,IntVect(NGROW,NGROW,0)); //2d missing j coordinate
    MultiFab mf_DC(ba,dm,1,IntVect(NGROW,NGROW,NGROW-1)); //2d missing j coordinate
    MultiFab mf_Hzk(ba,dm,1,IntVect(NGROW,NGROW,NGROW-1)); //2d missing j coordinate
 
    MultiFab* mf_z_r = vec_z_r[lev].get();
    MultiFab* mf_z_w = vec_z_w[lev].get();
    MultiFab* mf_h   = vec_hOfTheConfusingName[lev].get();
    MultiFab* mf_pm  = vec_pm[lev].get();
    MultiFab* mf_pn  =   vec_pn[lev].get();
    MultiFab* mf_fcor  = vec_fcor[lev].get();
 
    //Consider passing these into the advance function or renaming relevant things
 
    MultiFab mf_rho(ba,dm,1,IntVect(NGROW,NGROW,0));
    std::unique_ptr<MultiFab>& mf_rhoS = vec_rhoS[lev];
    std::unique_ptr<MultiFab>& mf_rhoA = vec_rhoA[lev];
    std::unique_ptr<MultiFab>& mf_ru = vec_ru[lev];
    std::unique_ptr<MultiFab>& mf_rv = vec_rv[lev];
    std::unique_ptr<MultiFab>& mf_rufrc = vec_rufrc[lev];
    std::unique_ptr<MultiFab>& mf_rvfrc = vec_rvfrc[lev];
    std::unique_ptr<MultiFab>& mf_sustr = vec_sustr[lev];
    std::unique_ptr<MultiFab>& mf_svstr = vec_svstr[lev];
    std::unique_ptr<MultiFab>& mf_rdrag = vec_rdrag[lev];
    std::unique_ptr<MultiFab>& mf_bustr = vec_bustr[lev];
    std::unique_ptr<MultiFab>& mf_bvstr = vec_bvstr[lev];
 
    MultiFab mf_rw(ba,dm,1,IntVect(NGROW,NGROW,0));
 
    std::unique_ptr<MultiFab>& mf_visc2_p = vec_visc2_p[lev];
    std::unique_ptr<MultiFab>& mf_visc2_r = vec_visc2_r[lev];
 
    // We need to set these because otherwise in the first call to remora_advance we may
    //    read uninitialized data on ghost values in setting the bc's on the velocities
    mf_rho.setVal(0.e34,IntVect(AMREX_D_DECL(NGROW-1,NGROW-1,0)));
    mf_rhoS->setVal(0.e34,IntVect(AMREX_D_DECL(NGROW-1,NGROW-1,0)));
    mf_rhoA->setVal(0.e34,IntVect(AMREX_D_DECL(NGROW-1,NGROW-1,0)));
 
    mf_DC.setVal(0);
 
    FillPatch(lev, time, *cons_old[lev], cons_old, BdyVars::t);
    FillPatch(lev, time, *xvel_old[lev], xvel_old, BdyVars::u);
    FillPatch(lev, time, *yvel_old[lev], yvel_old, BdyVars::v);
 
    FillPatch(lev, time, *cons_new[lev], cons_new, BdyVars::t);
    FillPatch(lev, time, *xvel_new[lev], xvel_new, BdyVars::u);
    FillPatch(lev, time, *yvel_new[lev], yvel_new, BdyVars::v);
 
    mf_rw.setVal(0.0);
    mf_rufrc->setVal(0);
    mf_rvfrc->setVal(0);
 
    int iic = istep[lev];
    int ntfirst = 0;
    if(iic==ntfirst) {
        MultiFab::Copy(S_new,S_old,0,0,S_new.nComp(),S_new.nGrowVect());
        MultiFab::Copy(U_new,U_old,0,0,U_new.nComp(),U_new.nGrowVect());
        MultiFab::Copy(V_new,V_old,0,0,V_new.nComp(),V_new.nGrowVect());
        MultiFab::Copy(W_new,W_old,0,0,W_new.nComp(),W_new.nGrowVect());
    }
    set_smflux(lev,t_old[lev]);
 
    auto N = Geom(lev).Domain().size()[2]-1; // Number of vertical "levs" aka, NZ
 
    for ( MFIter mfi(S_new, TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
        Array4<Real const> const& h     = vec_hOfTheConfusingName[lev]->const_array(mfi);
        Array4<Real const> const& Hz    = vec_Hz[lev]->const_array(mfi);
        Array4<Real      > const& Huon  = vec_Huon[lev]->array(mfi);
        Array4<Real      > const& Hvom  = vec_Hvom[lev]->array(mfi);
 
        Array4<Real const> const& z_w   = mf_z_w->const_array(mfi);
        Array4<Real const> const& uold  = U_old.const_array(mfi);
        Array4<Real const> const& vold  = V_old.const_array(mfi);
        Array4<Real      > const& rho   = mf_rho.array(mfi);
        Array4<Real      > const& rhoA  = mf_rhoA->array(mfi);
        Array4<Real      > const& rhoS  = mf_rhoS->array(mfi);
        Array4<Real const> const& rdrag = mf_rdrag->const_array(mfi);
        Array4<Real      > const& bustr = mf_bustr->array(mfi);
        Array4<Real      > const& bvstr = mf_bvstr->array(mfi);
 
        Array4<Real const> const& pm = mf_pm->const_array(mfi);
        Array4<Real const> const& pn = mf_pn->const_array(mfi);
 
        Box  bx = mfi.tilebox();
        Box gbx1 = mfi.growntilebox(IntVect(NGROW-1,NGROW-1,0));
        Box gbx2 = mfi.growntilebox(IntVect(NGROW,NGROW,0));
 
        Box bxD = bx;
        bxD.makeSlab(2,0);
        Box gbx1D = gbx1;
        gbx1D.makeSlab(2,0);
        Box gbx2D = gbx2;
        gbx2D.makeSlab(2,0);
 
        FArrayBox fab_FC(gbx2,1,amrex::The_Async_Arena()); //3D
        FArrayBox fab_FX(gbx2,1,amrex::The_Async_Arena()); //3D
        FArrayBox fab_FE(gbx2,1,amrex::The_Async_Arena()); //3D
        FArrayBox fab_BC(gbx2,1,amrex::The_Async_Arena());
        FArrayBox fab_CF(gbx2,1,amrex::The_Async_Arena());
 
        // Set bottom stress as defined in set_vbx.F
        ParallelFor(gbx1D, [=] AMREX_GPU_DEVICE (int i, int j, int )
        {
            bustr(i,j,0) = 0.5 * (rdrag(i-1,j,0)+rdrag(i,j,0))*(uold(i,j,0));
            bvstr(i,j,0) = 0.5 * (rdrag(i,j-1,0)+rdrag(i,j,0))*(vold(i,j,0));
        });
 
        //
        //-----------------------------------------------------------------------
        //  Compute horizontal mass fluxes, Hz*u/n and Hz*v/m (set_massflux_3d)
        //-----------------------------------------------------------------------
        //
        ParallelFor(Box(Huon), [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            Real on_u = 2.0 / (pn(i-1,j,0)+pn(i,j,0));
            Huon(i,j,k)=0.5*(Hz(i,j,k)+Hz(i-1,j,k))*uold(i,j,k)* on_u;
        });
 
        ParallelFor(Box(Hvom), [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            Real om_v= 2.0 / (pm(i,j-1,0)+pm(i,j,0));
            Hvom(i,j,k)=0.5*(Hz(i,j,k)+Hz(i,j-1,k))*vold(i,j,k)* om_v;
        });
 
        Array4<Real const> const& state_old = S_old.const_array(mfi);
        rho_eos(gbx2,state_old,rho,rhoA,rhoS,Hz,z_w,h,N);
    }
 
    MultiFab mf_W(ba,dm,1,IntVect(NGROW+1,NGROW+1,0));
    mf_W.setVal(0.0);
 
    if (solverChoice.use_prestep) {
        const int nnew  = 0;
        prestep(lev, U_old, V_old, U_new, V_new,
                mf_ru.get(), mf_rv.get(),
                S_old, S_new, mf_W,
                mf_DC, mf_z_r, mf_z_w, mf_h, mf_pm, mf_pn,
                mf_sustr.get(), mf_svstr.get(), mf_bustr.get(), mf_bvstr.get(),
                iic, ntfirst, nnew, nstp, nrhs, N, dt_lev);
    }
 
    // We use FillBoundary not FillPatch here since mf_W is single-level scratch space
    mf_W.FillBoundary(geom[lev].periodicity());
 
    for ( MFIter mfi(S_old, TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
        Array4<Real const> const& Hz    = vec_Hz[lev]->const_array(mfi);
        Array4<Real> const& Huon  = vec_Huon[lev]->array(mfi);
        Array4<Real> const& Hvom  = vec_Hvom[lev]->array(mfi);
        Array4<Real> const& z_r   = (mf_z_r)->array(mfi);
        Array4<Real> const& z_w   = (mf_z_w)->array(mfi);
        Array4<Real const> const& uold  = U_old.const_array(mfi);
        Array4<Real const> const& vold  = V_old.const_array(mfi);
        Array4<Real> const& u    = U_new.array(mfi);
        Array4<Real> const& v    = V_new.array(mfi);
        Array4<Real> const& rho = (mf_rho).array(mfi);
        Array4<Real> const& ru = (mf_ru)->array(mfi);
        Array4<Real> const& rv = (mf_rv)->array(mfi);
        Array4<Real> const& rufrc = (mf_rufrc)->array(mfi);
        Array4<Real> const& rvfrc = (mf_rvfrc)->array(mfi);
        Array4<Real> const& W = (mf_W).array(mfi);
        Array4<Real> const& sustr = (mf_sustr)->array(mfi);
        Array4<Real> const& svstr = (mf_svstr)->array(mfi);
        Array4<Real> const& bustr = (mf_bustr)->array(mfi);
        Array4<Real> const& bvstr = (mf_bvstr)->array(mfi);
        Array4<Real> const& visc2_p = (mf_visc2_p)->array(mfi);
        Array4<Real> const& visc2_r = (mf_visc2_r)->array(mfi);
 
        Array4<Real> const& zeta = (vec_zeta[lev])->array(mfi);
        Array4<Real> const& Zt_avg1 = (vec_Zt_avg1[lev])->array(mfi);
 
        Array4<Real const> const& pm = mf_pm->const_array(mfi);
        Array4<Real const> const& pn = mf_pn->const_array(mfi);
        Array4<Real const> const& fcor = mf_fcor->const_array(mfi);
 
        Box bx = mfi.tilebox();
 
        Box tbxp1 = bx;
        Box tbxp2 = bx;
        Box xbx = mfi.nodaltilebox(0);
        Box ybx = mfi.nodaltilebox(1);
        Box gbx1 = mfi.growntilebox(IntVect(NGROW-1,NGROW-1,0));
        Box gbx2 = mfi.growntilebox(IntVect(NGROW,NGROW,0));
 
        Box utbx = mfi.nodaltilebox(0);
        Box vtbx = mfi.nodaltilebox(1);
 
        tbxp1.grow(IntVect(NGROW-1,NGROW-1,0));
        tbxp2.grow(IntVect(NGROW,NGROW,0));
 
        Box bxD = bx;
        bxD.makeSlab(2,0);
        Box gbx1D = gbx1;
        gbx1D.makeSlab(2,0);
        Box gbx2D = gbx2;
        gbx2D.makeSlab(2,0);
 
        Box tbxp1D = tbxp1;
        tbxp1D.makeSlab(2,0);
        Box tbxp2D = tbxp2;
        tbxp2D.makeSlab(2,0);
 
        FArrayBox fab_FC(tbxp2,1,amrex::The_Async_Arena()); //3D
        FArrayBox fab_FX(gbx2,1,amrex::The_Async_Arena()); //3D
        FArrayBox fab_FE(gbx2,1,amrex::The_Async_Arena()); //3D
        FArrayBox fab_BC(gbx2,1,amrex::The_Async_Arena());
        FArrayBox fab_CF(gbx2,1,amrex::The_Async_Arena());
 
        FArrayBox fab_fomn(tbxp2D,1,amrex::The_Async_Arena());
 
        auto FC=fab_FC.array();
 
        auto fomn=fab_fomn.array();
 
        if (solverChoice.use_coriolis) {
            ParallelFor(tbxp2D, [=] AMREX_GPU_DEVICE (int i, int j, int  )
            {
                fomn(i,j,0) = fcor(i,j,0)*(1.0/(pm(i,j,0)*pn(i,j,0)));
            });
        }
 
        ParallelFor(gbx2, [=] AMREX_GPU_DEVICE (int i, int j, int k)
        {
            FC(i,j,k)=0.0;
        });
 
        prsgrd(tbxp1,gbx1,utbx,vtbx,ru,rv,pn,pm,rho,FC,Hz,z_r,z_w,nrhs,N);
 
        // Apply mixing to temperature and, if use_salt, salt
        int ncomp = solverChoice.use_salt ? 2 : 1;
        Array4<Real> const&     s_arr = S_old.array(mfi);
        Array4<Real> const& diff2_arr = vec_diff2[lev]->array(mfi);
 
        t3dmix(bx, s_arr, diff2_arr, Hz, pm, pn, dt_lev, ncomp);
 
        Array4<Real> const& diff2_arr_scalar = vec_diff2[lev]->array(mfi,Scalar_comp);
        t3dmix(bx, S_old.array(mfi,Scalar_comp), diff2_arr_scalar, Hz, pm, pn, dt_lev, 1);
 
        if (solverChoice.use_coriolis) {
            //-----------------------------------------------------------------------
            // coriolis
            //-----------------------------------------------------------------------
            //
            // ru, rv updated
            // In ROMS, coriolis is the first (un-ifdefed) thing to happen in rhs3d_tile, which gets called after t3dmix
            coriolis(xbx, ybx, uold, vold, ru, rv, Hz, fomn, nrhs, nrhs);
        }
 
        //
        //-----------------------------------------------------------------------
        //
 
        ////rufrc from 3d is set to ru, then the wind stress (and bottom stress) is added, then the mixing is added
        //rufrc=ru+sustr*om_u*on_u
 
        rhs_uv_3d(xbx, ybx, uold, vold, ru, rv, rufrc, rvfrc,
                  sustr, svstr, bustr, bvstr, Huon, Hvom,
                  pm, pn, W, FC, nrhs, N);
 
        if(solverChoice.use_uv3dmix) {
            const int nnew = 0;
            uv3dmix(xbx, ybx, u, v, uold, vold, rufrc, rvfrc, visc2_p, visc2_r, Hz, pm, pn, nrhs, nnew, dt_lev);
        }
 
        // Set first two components of zeta to time-averaged values before barotropic update
        ParallelFor(gbx2D, [=] AMREX_GPU_DEVICE (int i, int j, int)
        {
            zeta(i,j,0,0) = Zt_avg1(i,j,0);
            zeta(i,j,0,1) = Zt_avg1(i,j,0);
        });
    } // MFIter
 
    // Update Akv with new depth. NOTE: this happens before set_zeta in ROMS
    set_vmix(lev);
 
    FillPatch(lev, time, *cons_old[lev], cons_old, BdyVars::t);
    FillPatch(lev, time, *cons_new[lev], cons_new, BdyVars::t);
 
    FillPatch(lev, time, *vec_sstore[lev], GetVecOfPtrs(vec_sstore), BdyVars::t);
 
    FillPatch(lev, time, *vec_Huon[lev], GetVecOfPtrs(vec_Huon));
    FillPatch(lev, time, *vec_Hvom[lev], GetVecOfPtrs(vec_Hvom));
}

stretch_transform()

void REMORA::stretch_transform

(

int

lev

)

{
    std::unique_ptr<MultiFab>& mf_z_w = vec_z_w[lev];
    std::unique_ptr<MultiFab>& mf_z_r = vec_z_r[lev];
    std::unique_ptr<MultiFab>& mf_s_r = vec_s_r[lev];
    std::unique_ptr<MultiFab>& mf_Hz  = vec_Hz[lev];
    std::unique_ptr<MultiFab>& mf_h  = vec_hOfTheConfusingName[lev];
    std::unique_ptr<MultiFab>& mf_Zt_avg1  = vec_Zt_avg1[lev];
    std::unique_ptr<MultiFab>& mf_z_phys_nd  = vec_z_phys_nd[lev];
    auto N_loop = Geom(lev).Domain().size()[2]-1; // Number of vertical "levs" aka, NZ
 
    for ( MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
      Array4<Real> const& z_w = (mf_z_w)->array(mfi);
      Array4<Real> const& z_r = (mf_z_r)->array(mfi);
      Array4<Real> const& s_r = (mf_s_r)->array(mfi);
      Array4<Real> const& Hz  = (mf_Hz)->array(mfi);
      Array4<Real> const& h  = (mf_h)->array(mfi);
      Array4<Real> const& Zt_avg1  = (mf_Zt_avg1)->array(mfi);
      Box bx = mfi.tilebox();
      Box gbx2 = bx;
      gbx2.grow(IntVect(NGROW,NGROW,0));
      Box gbx3 = bx;
      gbx3.grow(IntVect(NGROW+1,NGROW+1,0));
      Box gbx2D = gbx2;
      gbx2D.makeSlab(2,0);
      Box gbx3D = gbx3;
      gbx3D.makeSlab(2,0);
      Box wgbx3 = gbx3.surroundingNodes(2);
 
      const auto & geomdata = Geom(lev).data();
 
      int nz = geom[lev].Domain().length(2);
 
      auto N = nz; // Number of vertical "levels" aka, NZ
      //forcing tcline to be the same as probhi for now, one in DataStruct.H other in inputs
      Real hc=-min(geomdata.ProbHi(2),-solverChoice.tcline); // Do we need to enforce min here?
      const auto local_theta_s = solverChoice.theta_s;
      const auto local_theta_b = solverChoice.theta_b;
 
      amrex::ParallelFor(wgbx3, [=] AMREX_GPU_DEVICE (int i, int j, int k)
      {
          z_w(i,j,k) = h(i,j,0);
      });
 
      // ROMS Transform 2
      Gpu::streamSynchronize();
 
      amrex::ParallelFor(gbx3D, [=] AMREX_GPU_DEVICE (int i, int j, int )
      {
          for (int k=-1; k<=N_loop; k++) {
              //        const Real z = prob_lo[2] + (k + 0.5) * dx[2];
              // const auto prob_lo         = geomdata.ProbLo();
              // const auto dx              = geomdata.CellSize();
              // This is the z for the bottom of the cell this k corresponds to
              //   if we weren't stretching and transforming
              //        const Real z = prob_lo[2] + (k) * dx[2];
              //        h(i,j,0) = -prob_lo[2]; // conceptually
 
              ////////////////////////////////////////////////////////////////////
              //ROMS Stretching 4
              // Move this block to it's own function for maintainability if needed
              // Information about the problem dimension would need to be added
              // This file would need a k dependent function to return the
              // stretching scalars, or access to 4 vectors of length prob_length(2)
              /////////////////////////////////////////////////////////////////////
              Real ds = 1.0_rt / Real(N);
 
              Real cff_r, cff_w, cff1_r, cff1_w, cff2_r, cff2_w, Csur, Cbot;
              Real sc_r,sc_w,Cs_r,Cs_w;
 
              if (k==N) // end of array // pretend we're storing 0?
              {
                  sc_w=0.0; //sc_w / hc
                  Cs_w=0.0; //Cs_w
              }
              else if (k==0) // beginning of array
              {
                  sc_w=-1.0; //sc_w / hc
                  Cs_w=-1.0; //Cs_w
              }
              else
              {
                sc_w=ds*(k-N);
 
                if (local_theta_s > 0.0_rt) {
                  Csur=(1.0_rt-std::cosh(local_theta_s*sc_w))/
                    (std::cosh(local_theta_s)-1.0_rt);
                } else {
                  Csur=-sc_w*sc_w;
                }
 
                if (local_theta_b > 0.0_rt) {
                  Cbot=(std::exp(local_theta_b*Csur)-1.0_rt)/
                    (1.0_rt-std::exp(-local_theta_b));
                  Cs_w=Cbot;
                } else {
                  Cs_w=Csur;
                }
              } // k test
 
              cff_w=hc*sc_w;
              cff1_w=Cs_w;
 
              //cff_r => sc_r *hc
              //cff1_r => Cs_r
              //Don't do anything special for first/last index
              {
                sc_r=ds*(k-N+0.5_rt);
 
                if (local_theta_s > 0.0_rt) {
                  Csur=(1.0_rt-std::cosh(local_theta_s*sc_r))/
                    (std::cosh(local_theta_s)-1.0_rt);
                } else {
                  Csur=-sc_r*sc_r;
                }
 
                if (local_theta_b > 0.0_rt) {
                  Cbot=(std::exp(local_theta_b*Csur)-1.0_rt)/
                    (1.0_rt-std::exp(-local_theta_b));
                  Cs_r=Cbot;
                } else {
                  Cs_r=Csur;
                }
              }
 
              if (i==0&&j==0&&k<N&&k>=0) {
                  s_r(0,0,k) = sc_r;
              }
 
              cff_r=hc*sc_r;
              cff1_r=Cs_r;
 
              ////////////////////////////////////////////////////////////////////
              Real hwater=h(i,j,0);
              //
              // if (k==0)  //extra guess added (maybe not actually defined in ROMS)
              // {
              //     Real hinv=1.0_rt/(hc+hwater);
              //     cff2_r=(cff_r+cff1_r*hwater)*hinv;
              //     //         z_w(i,j,k-2) = hwater;
              //     //         z_w(i,j,k-1)= -hwater;
 
              //     z_r(i,j,k) = Zt_avg1(i,j,0)+(Zt_avg1(i,j,0)+hwater)*cff2_r;
              //     Hz(i,j,k)=z_w(i,j,k)+hwater;//-z_w(i,j,k-1);
              // } else
 
              //Note, we are not supporting ICESHELF flag
 
              Real hinv=1.0_rt/(hc+hwater);
              cff2_r=(cff_r+cff1_r*hwater)*hinv;
              cff2_w=(cff_w+cff1_w*hwater)*hinv;
 
              if(k==0) {
                  // HACK: should actually be the normal expression with coeffs evaluated at k=N-1
                  z_w(i,j,N-1)=Zt_avg1(i,j,0);
 
              } else if (k==-1) {
                  h(i,j,0,1) = Zt_avg1(i,j,0)+(Zt_avg1(i,j,0)+hwater)*cff2_w;
 
              } else {
                  z_w(i,j,k-1)=Zt_avg1(i,j,0)+(Zt_avg1(i,j,0)+hwater)*cff2_w;
 
              }
 
              if(k!=-1) {
                  z_r(i,j,k)=Zt_avg1(i,j,0)+(Zt_avg1(i,j,0)+hwater)*cff2_r;
              }
          } // k
      });
 
      Gpu::streamSynchronize();
 
      amrex::ParallelFor(gbx3, [=] AMREX_GPU_DEVICE (int i, int j, int k)
      {
          if (k==0) {
              Hz(i,j,k)=z_w(i,j,k)+h(i,j,0);
          } else {
              Hz(i,j,k)=z_w(i,j,k)-z_w(i,j,k-1);
          }
      });
    } // mfi
 
    Real time = t_new[lev];
    FillPatch(lev, time, *vec_z_w[lev], GetVecOfPtrs(vec_z_w));
    FillPatch(lev, time, *vec_z_r[lev], GetVecOfPtrs(vec_z_r));
    FillPatch(lev, time, *vec_s_r[lev], GetVecOfPtrs(vec_s_r));
    FillPatch(lev, time, *vec_Hz[lev] , GetVecOfPtrs(vec_Hz));
 
    // Define nodal z as average of z on w-faces
    for ( MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
    {
      Array4<Real> const& z_w = (mf_z_w)->array(mfi);
      Array4<Real> const& z_phys_nd  = (mf_z_phys_nd)->array(mfi);
 
      Box z_w_box = Box(z_w);
      auto const lo = amrex::lbound(z_w_box);
      auto const hi = amrex::ubound(z_w_box);
 
      //
      // WARNING: z_w(i,j,k)       refers to the face on the HIGH side of cell (i,j,k)
      // WARNING: z_phys_nd(i,j,k) refers to the node on the LOW  side of cell (i,j,k)
      //
      ParallelFor(Box(z_phys_nd), [=] AMREX_GPU_DEVICE (int i, int j, int k)
      {
          // For now assume all boundaries are constant height --
          //     we will enforce periodicity below
          int kk = (k == 0) ? hi.z : k-1;
          if ( i >= lo.x && i <= hi.x-1 && j >= lo.y && j <= hi.y-1 )
          {
              z_phys_nd(i,j,k)=0.25*( z_w(i,j  ,kk) + z_w(i+1,j  ,kk) +
                                      z_w(i,j+1,kk) + z_w(i+1,j+1,kk) );
          } else {
             int ii = std::min(std::max(i, lo.x), hi.x);
             int jj = std::min(std::max(j, lo.y), hi.y);
             z_phys_nd(i,j,k) = z_w(ii,jj,kk);
          }
          if (k == 0)  z_phys_nd(i,j,k) *= -1.;
      });
    } // mf
 
    // Note that we do *not* want to do a multilevel fill here -- we have
    // already filled z_phys_nd on the grown boxes, but we enforce periodicity just in case
    vec_z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
}

sum_integrated_quantities()

void REMORA::sum_integrated_quantities

(

amrex::Real

time

)

Integrate conserved quantities for diagnostics

{
    BL_PROFILE("REMORA::sum_integrated_quantities()");
 
    if (verbose <= 0)
      return;
 
    int datwidth = 14;
    int datprecision = 6;
 
    Real scalar = 0.0;
    Real kineng = 0.0;
 
    for (int lev = 0; lev <= finest_level; lev++)
    {
        MultiFab kineng_mf(grids[lev], dmap[lev], 1, 0);
        MultiFab cc_vel_mf(grids[lev], dmap[lev], 3, 0);
        average_face_to_cellcenter(cc_vel_mf,0,
            Array<const MultiFab*,3>{xvel_new[lev],yvel_new[lev],zvel_new[lev]});
 
        for (MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
            const Box& bx = mfi.tilebox();
            const Array4<      Real> kineng_arr = kineng_mf.array(mfi);
            const Array4<const Real>    vel_arr = cc_vel_mf.const_array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept
            {
                kineng_arr(i,j,k) = 0.5 * ( vel_arr(i,j,k,0)*vel_arr(i,j,k,0) + vel_arr(i,j,k,1)*vel_arr(i,j,k,1) +
                                            vel_arr(i,j,k,2)*vel_arr(i,j,k,2) );
            });
        } // mfi
 
        scalar += volWgtSumMF(lev,*cons_new[lev],Scalar_comp,false,true);
        kineng += volWgtSumMF(lev,kineng_mf     ,             0,false,true);
    }
 
    if (verbose > 0) {
        const int nfoo = 2;
        Real foo[nfoo] = {scalar,kineng};
#ifdef AMREX_LAZY
        Lazy::QueueReduction([=]() mutable {
#endif
        ParallelDescriptor::ReduceRealSum(
            foo, nfoo, ParallelDescriptor::IOProcessorNumber());
 
          if (ParallelDescriptor::IOProcessor()) {
            int i = 0;
            scalar = foo[i++];
            kineng = foo[i++];
 
            amrex::Print() << '\n';
            amrex::Print() << "TIME= " << time << " SCALAR      = " << scalar << '\n';
            amrex::Print() << "TIME= " << time << " KIN. ENG.   = " << kineng << '\n';
 
            if (NumDataLogs() > 0) {
                std::ostream& data_log1 = DataLog(0);
                if (data_log1.good()) {
                    if (time == 0.0) {
                        data_log1 << std::setw(datwidth) << "          time";
                        data_log1 << std::setw(datwidth) << "        scalar";
                        data_log1 << std::setw(datwidth) << "        kineng";
                        data_log1 << std::endl;
                    }
 
                  // Write the quantities at this time
                  data_log1 << std::setw(datwidth) << time;
                  data_log1 << std::setw(datwidth) << std::setprecision(datprecision)
                            << scalar;
                  data_log1 << std::setw(datwidth) << std::setprecision(datprecision)
                            << kineng;
                  data_log1 << std::endl;
              }
            }
          }
#ifdef AMREX_LAZY
        });
#endif
    }
}

t3dmix()

void REMORA::t3dmix	(	const amrex::Box &	bx,
		const amrex::Array4< amrex::Real > &	state,
		const amrex::Array4< amrex::Real const > &	diff2,
		const amrex::Array4< amrex::Real const > &	Hz,
		const amrex::Array4< amrex::Real const > &	pm,
		const amrex::Array4< amrex::Real const > &	pn,
		const amrex::Real	dt_lev,
		const int	ncomp
	)

Harmonic diffusivity for tracers

{
    //-----------------------------------------------------------------------
    //  Add in harmonic diffusivity s terms.
    //-----------------------------------------------------------------------
 
    Box xbx(bx); xbx.surroundingNodes(0);
    Box ybx(bx); ybx.surroundingNodes(1);
 
    FArrayBox fab_FX(xbx,ncomp,The_Async_Arena());
    FArrayBox fab_FE(ybx,ncomp,The_Async_Arena());
 
    auto FX=fab_FX.array();
    auto FE=fab_FE.array();
 
    ParallelFor(xbx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
    {
        const Real pmon_u = (pm(i-1,j,0)+pm(i,j,0))/(pn(i-1,j,0)+pn(i,j,0));
 
        const Real cff = 0.25 * (diff2(i,j,n) + diff2(i-1,j,n)) * pmon_u;
        FX(i,j,k,n) = cff * (Hz(i,j,k) + Hz(i+1,j,k)) * (state(i,j,k,n)-state(i-1,j,k,n));
    });
 
    ParallelFor(ybx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
    {
        const Real pnom_v = (pn(i,j-1,0)+pn(i,j,0))/(pm(i,j-1,0)+pm(i,j,0));
 
        const Real cff = 0.25*(diff2(i,j,n)+diff2(i,j-1,n)) * pnom_v;
        FE(i,j,k,n) = cff * (Hz(i,j,k) + Hz(i,j-1,k)) * (state(i,j,k,n) - state(i,j-1,k,n));
    });
 
    /*
     Time-step harmonic, S-surfaces diffusion term.
    */
    ParallelFor(bx, ncomp, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n)
    {
        const Real cff = dt_lev*pm(i,j,0)*pn(i,j,0);
 
        state(i,j,k,n) += cff * ( (FX(i+1,j  ,k,n)-FX(i,j,k,n))
                                 +(FE(i  ,j+1,k,n)-FE(i,j,k,n)) );
    });
}

timeStep()

void REMORA::timeStep ( int  lev, amrex::Real  time, int  iteration  )

private

advance a level by dt, includes a recursive call for finer levels

{
    if (regrid_int > 0)  // We may need to regrid
    {
        // help keep track of whether a level was already regridded
        // from a coarser level call to regrid
        static Vector<int> last_regrid_step(max_level+1, 0);
 
        // regrid changes level "lev+1" so we don't regrid on max_level
        // also make sure we don't regrid fine levels again if
        // it was taken care of during a coarser regrid
        if (lev < max_level && istep[lev] > last_regrid_step[lev])
        {
            if (istep[lev] % regrid_int == 0)
            {
                // regrid could add newly refine levels (if finest_level < max_level)
                // so we save the previous finest level index
                int old_finest = finest_level;
                regrid(lev, time);
 
                // Mark that we have regridded this level already
                for (int k = lev; k <= finest_level; ++k) {
                    last_regrid_step[k] = istep[k];
                }
 
                // If there are newly created levels, set the time step
                for (int k = old_finest+1; k <= finest_level; ++k) {
                    dt[k] = dt[k-1] / MaxRefRatio(k-1);
                }
            }
        }
    }
    // Update what we call "old" and "new" time
    t_old[lev] = t_new[lev];
    t_new[lev] += dt[lev];
 
    if (Verbose()) {
        amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
        amrex::Print() << "ADVANCE from time = " << t_old[lev] << " to " << t_new[lev]
                       << " with dt = " << dt[lev] << std::endl;
    }
 
    // We must swap the pointers so the previous step's "new" is now this step's "old"
    std::swap(cons_old[lev], cons_new[lev]);
    std::swap(xvel_old[lev], xvel_new[lev]);
    std::swap(yvel_old[lev], yvel_new[lev]);
    std::swap(zvel_old[lev], zvel_new[lev]);
 
    // Advance a single level for a single time step
    Advance(lev, time, dt[lev], iteration, nsubsteps[lev]);
 
    ++istep[lev];
 
    if (Verbose())
    {
        amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
        amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
    }
 
    if (lev < finest_level)
    {
        // recursive call for next-finer level
        for (int i = 1; i <= nsubsteps[lev+1]; ++i)
        {
            timeStep(lev+1, time+(i-1)*dt[lev+1], i);
        }
 
        if (solverChoice.coupling_type == CouplingType::TwoWay) {
            AverageDownTo(lev); // average lev+1 down to lev
        }
    }
}

timeStepML()

void REMORA::timeStepML ( amrex::Real  time, int  iteration  )

private

advance all levels by dt, loops over finer levels

{
    // HACK HACK so lev is defined and compiler won't complain, but always say regrid_int=-1
    for (int lev=0; lev <= finest_level;lev++) {
        if (regrid_int > 0)  // We may need to regrid
        {
            // help keep track of whether a level was already regridded
            // from a coarser level call to regrid
            static Vector<int> last_regrid_step(max_level+1, 0);
 
            // regrid changes level "lev+1" so we don't regrid on max_level
            // also make sure we don't regrid fine levels again if
            // it was taken care of during a coarser regrid
            if (lev < max_level && istep[lev] > last_regrid_step[lev])
            {
                if (istep[lev] % regrid_int == 0)
                {
                    // regrid could add newly refine levels (if finest_level < max_level)
                    // so we save the previous finest level index
                    int old_finest = finest_level;
                    regrid(lev, time);
 
                    // Mark that we have regridded this level already
                    for (int k = lev; k <= finest_level; ++k) {
                        last_regrid_step[k] = istep[k];
                    }
 
                    // If there are newly created levels, set the time step
                    for (int k = old_finest+1; k <= finest_level; ++k) {
                        dt[k] = dt[k-1] / MaxRefRatio(k-1);
                    }
                }
            }
        }
    }
 
    for (int lev=0; lev <= finest_level;lev++)
    {
        // Update what we call "old" and "new" time
        t_old[lev] = t_new[lev];
        t_new[lev] += dt[lev];
 
        if (Verbose()) {
            amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] ";
            amrex::Print() << "ADVANCE from time = " << t_old[lev] << " to " << t_new[lev]
                           << " with dt = " << dt[lev] << std::endl;
        }
 
        // We must swap the pointers so the previous step's "new" is now this step's "old"
        std::swap(cons_old[lev], cons_new[lev]);
        std::swap(xvel_old[lev], xvel_new[lev]);
        std::swap(yvel_old[lev], yvel_new[lev]);
        std::swap(zvel_old[lev], zvel_new[lev]);
 
        setup_step(lev, time, dt[lev]);
    }
 
    if (solverChoice.use_barotropic)
    {
        int nfast_counter=nfast + 1;
 
        for (int lev=0; lev <= finest_level; lev++)
        {
            //Compute fast timestep from dt_lev and ratio
            Real dtfast_lev=dt[lev]/Real(fixed_ndtfast_ratio);
            for (int my_iif = 0; my_iif < nfast_counter; my_iif++) {
                advance_2d_onestep(lev, dt[lev], dtfast_lev, my_iif, nfast_counter);
            } // my_iif
        } // lev
    } // use_barotropic
 
    for (int lev=0; lev <= finest_level; lev++) {
        advance_3d_ml(lev, dt[lev]);
        ++istep[lev];
 
        if (Verbose())
        {
            amrex::Print() << "[Level " << lev << " step " << istep[lev] << "] ";
            amrex::Print() << "Advanced " << CountCells(lev) << " cells" << std::endl;
        }
    }
 
    if (solverChoice.coupling_type == CouplingType::TwoWay) {
        for (int lev=0; lev <= finest_level-1; lev++) {
            AverageDownTo(lev); // average lev+1 down to lev
        }
    }
}

update_massflux_3d()

void REMORA::update_massflux_3d	(	const amrex::Box &	bx,
		const int	ioff,
		const int	joff,
		const amrex::Array4< amrex::Real > &	phi,
		const amrex::Array4< amrex::Real > &	phibar,
		const amrex::Array4< amrex::Real > &	Hphi,
		const amrex::Array4< amrex::Real const > &	Hz,
		const amrex::Array4< amrex::Real const > &	pm_or_pn,
		const amrex::Array4< amrex::Real const > &	Dphi1,
		const amrex::Array4< amrex::Real const > &	Dphi2,
		const amrex::Array4< amrex::Real > &	DC,
		const amrex::Array4< amrex::Real > &	FC,
		const int	nnew
	)

update_massflux_3d

Parameters

[in]	bx	box on which to update
[in]	ioff	offset in x-direction
[in]	joff	offset in y-direction
[in]	phi	u or v
[out]	Hphi	H-weighted u or v
[in]	Hz	weighting
[in]	om_v_or_on_u
[in]	Dphi_avg1
[in]	Dphi_avg2
[in,out]	DC
[in,out]	FC
[in]	nnew	component of velocity

{
    auto N = Geom(0).Domain().size()[2]-1; // Number of vertical "levs" aka, NZ
 
    auto bxD=bx;
    auto bx_g1z=bx;
    bxD.makeSlab(2,0);
    bx_g1z.grow(IntVect(0,0,1));
 
    // auto geomdata = Geom(0).data();
    // bool NSPeriodic = geomdata.isPeriodic(1);
    // bool EWPeriodic = geomdata.isPeriodic(0);
 
    //Copied depth of water column calculation from DepthStretchTransform
    //Compute thicknesses of U-boxes DC(i,j,0:N-1), total depth of the water column DC(i,j,-1)
 
    //This takes advantage of Hz being an extra grow cell size
    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real om_v_or_on_u = 2.0_rt / (pm_or_pn(i,j,0) + pm_or_pn(i-ioff,j-joff,0));
 
        DC(i,j,k) = 0.5_rt * om_v_or_on_u * (Hz(i,j,k)+Hz(i-ioff,j-joff,k));
    });
 
    ParallelFor(bxD, [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        for (int k=0; k<=N; k++) {
            DC(i,j,-1) += DC(i,j,k);
        }
 
        DC(i,j,-1) = 1.0 / DC(i,j,-1);
 
        for (int k=0; k<=N; k++) {
 
//          BOUNDARY CONDITIONS
//          if (!(NSPeriodic&&EWPeriodic))
//          {
//              if ( ( ((i<0)||(i>=Mn+1)) && !EWPeriodic ) ||  ( ((j<0)||(j>=Mm+1)) && !NSPeriodic ) )
//              {
//                  phi(i,j,k) -= CF;
//              }
//          }
 
            Hphi(i,j,k) = 0.5 * (Hphi(i,j,k)+phi(i,j,k,nnew)*DC(i,j,k));
            FC(i,j,0)  += Hphi(i,j,k);
        } // k
 
        FC(i,j,0) = DC(i,j,-1)*(FC(i,j,0)-Dphi_avg2(i,j,0));
    });
 
    ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Hphi(i,j,k) -= DC(i,j,k)*FC(i,j,0);
    });
 
    ParallelFor(bxD, [=] AMREX_GPU_DEVICE(int i, int j, int )
    {
        phibar(i,j,0,0) = DC(i,j,-1) * Dphi_avg1(i,j,0);
        phibar(i,j,0,1) = phibar(i,j,0,0);
    });
 
}

uv3dmix()

void REMORA::uv3dmix	(	const amrex::Box &	xbx,
		const amrex::Box &	ybx,
		const amrex::Array4< amrex::Real > &	u,
		const amrex::Array4< amrex::Real > &	v,
		const amrex::Array4< amrex::Real const > &	uold,
		const amrex::Array4< amrex::Real const > &	vold,
		const amrex::Array4< amrex::Real > &	rufrc,
		const amrex::Array4< amrex::Real > &	rvfrc,
		const amrex::Array4< amrex::Real const > &	visc2_p,
		const amrex::Array4< amrex::Real const > &	visc2_r,
		const amrex::Array4< amrex::Real const > &	Hz,
		const amrex::Array4< amrex::Real const > &	pm,
		const amrex::Array4< amrex::Real const > &	pn,
		int	nrhs,
		int	nnew,
		const amrex::Real	dt_lev
	)

Harmonic viscosity

{
    //-----------------------------------------------------------------------
    //  Add in harmonic viscosity s terms.
    //-----------------------------------------------------------------------
 
    FArrayBox fab_UFx(growLo(xbx,0,1),1,amrex::The_Async_Arena()); fab_UFx.template setVal<RunOn::Device>(0.);
    FArrayBox fab_UFe(growHi(xbx,1,1),1,amrex::The_Async_Arena()); fab_UFe.template setVal<RunOn::Device>(0.);
    FArrayBox fab_VFe(growLo(ybx,1,1),1,amrex::The_Async_Arena()); fab_VFe.template setVal<RunOn::Device>(0.);
    FArrayBox fab_VFx(growHi(ybx,0,1),1,amrex::The_Async_Arena()); fab_VFx.template setVal<RunOn::Device>(0.);
 
    auto UFx=fab_UFx.array();
    auto UFe=fab_UFe.array();
    auto VFx=fab_VFx.array();
    auto VFe=fab_VFe.array();
 
    // Think of the cell-centered box as                              [ 0:nx-1, 0:ny-1] (0,0,0) cc
    //
    // xbx is the x-face-centered box on which we update u  [ 0:nx  , 0:ny-1] (1,0,0) x-faces
    //   to do so requires                        UFx on    [-1:nx  , 0:ny-1] (0,0,0) cc
    //      which requires                        uold on   [-1:nx+1, 0:ny-1] (1,0,0) x-faces
    //        and requires                        vold on   [-1:nx  , 0:ny  ] (0,1,0) y-faces
    //   to do so requires                        UFe on    [ 0:nx  , 0:ny  ] (1,1,0) xy-nodes
    //      which requires                        uold on   [ 0:nx  ,-1:ny  ] (1,0,0) x-faces
    //       and  requires                        vold on   [ -1:nx , 0:ny  ] (0,1,0) y-faces
 
    ParallelFor(growLo(xbx,0,1), [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        const Real cff = 0.5*Hz(i,j,k) * ( pm(i,j,0) / pn(i,j,0) *
                                    ( (pn(i  ,j,0) + pn(i+1,j,0)) * uold(i+1,j,k,nrhs)-
                                      (pn(i-1,j,0) + pn(i  ,j,0)) * uold(i  ,j,k,nrhs) )-
                                                pn(i,j,0) / pm(i,j,0) *
                                    ( (pm(i,j  ,0) + pm(i,j+1,0)) * vold(i,j+1,k,nrhs)-
                                      (pm(i,j-1,0) + pm(i,j  ,0)) * vold(i,j  ,k,nrhs) ) );
 
        Real on_r = 1.0_rt / pm(i,j,0);
        UFx(i,j,k) = on_r * on_r * visc2_r(i,j,0) * cff;
    });
 
    ParallelFor(growHi(xbx,1,1), [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        const Real cff = 0.125 * (Hz(i-1,j  ,k) + Hz(i,j ,k)+ Hz(i-1,j-1,k) + Hz(i,j-1,k))*
                    (pm(i,j,0)/pn(i,j,0)*
                     ((pn(i  ,j-1,0)+pn(i  ,j,0))*vold(i  ,j,k,nrhs)-
                      (pn(i-1,j-1,0)+pn(i-1,j,0))*vold(i-1,j,k,nrhs))+
                     pn(i,j,0)/pm(i,j,0)*
                     ((pm(i-1,j  ,0)+pm(i,j  ,0))*uold(i,j  ,k,nrhs)-
                      (pm(i-1,j-1,0)+pm(i,j-1,0))*uold(i,j-1,k,nrhs)));
 
        const Real om_p =  4.0_rt / (pm(i-1,j-1,0)+pm(i-1,j,0)+pm(i,j-1,0)+pm(i,j,0));
        UFe(i,j,k) = om_p*om_p*visc2_p(i,j,0)*cff;
    });
 
    ParallelFor(xbx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        const Real cff=dt_lev*0.25*(pm(i-1,j,0)+pm(i,j,0))*(pn(i-1,j,0)+pn(i,j,0));
        const Real cff1=0.5*(pn(i-1,j,0)+pn(i,j,0))*(UFx(i,j  ,k)-UFx(i-1,j,k));
        const Real cff2=0.5*(pm(i-1,j,0)+pm(i,j,0))*(UFe(i,j+1,k)-UFe(i  ,j,k));
        const Real cff3=cff*(cff1+cff2);
        amrex::Gpu::Atomic::Add(&(rufrc(i,j,0)), cff1+cff2);
        u(i,j,k,nnew)=u(i,j,k,nnew)+cff3;
    });
 
 
    // Think of the cell-centered box as                              [ 0:nx-1, 0:ny-1] (0,0,0) cc
    //
    // ybx is the y-face-centered box on which we update v  [ 0:nx-1, 0:ny  ] (1,0,0) x-faces
    //   to do so requires                        VFe on    [ 0:nx-1,-1:ny  ] (0,0,0) cc
    //      which requires                        uold on   [ 0:nx  ,-1:ny  ] (1,0,0) x-faces
    //        and requires                        vold on   [ 0:nx-1,-1:ny+1] (0,1,0) y-faces
    //   to do so requires                        VFx on    [ 0:nx  , 0:ny  ] (1,1,0) xy-nodes
    //      which requires                        uold on   [ 0:nx  ,-1:ny  ] (1,0,0) x-faces
    //       and  requires                        vold on   [-1:nx  , 0:ny  ] (0,1,0) y-faces
 
    ParallelFor(growLo(ybx,1,1), [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        // cff depends on k, but UFx and VFe will only be affected by the last cell?
        const Real cff = 0.5*Hz(i,j,k) * (pm(i,j,0) / pn(i,j,0) *
                ((pn(i,  j,0) + pn(i+1,j,0)) * uold(i+1,j,k,nrhs)-
                 (pn(i-1,j,0) + pn(i,  j,0)) * uold(i  ,j,k,nrhs))-
                pn(i,j,0) / pm(i,j,0) *
                ((pm(i,j  ,0)+pm(i,j+1,0))*vold(i,j+1,k,nrhs)-
                 (pm(i,j-1,0)+pm(i,j  ,0))*vold(i,j  ,k,nrhs)));
 
        Real om_r = 1.0_rt / pm(i,j,0);
        VFe(i,j,k) = om_r * om_r * visc2_r(i,j,0) * cff;
    });
 
    ParallelFor(growHi(ybx,0,1), [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        const Real cff = 0.125 * (Hz(i-1,j  ,k)+Hz(i,j ,k)+
                              Hz(i-1,j-1,k)+Hz(i,j-1,k))*
                    (pm(i,j,0)/pn(i,j,0)*
                     ((pn(i  ,j-1,0)+pn(i  ,j,0))*vold(i  ,j,k,nrhs)-
                      (pn(i-1,j-1,0)+pn(i-1,j,0))*vold(i-1,j,k,nrhs))+
                     pn(i,j,0)/pm(i,j,0)*
                     ((pm(i-1,j  ,0)+pm(i,j  ,0))*uold(i,j  ,k,nrhs)-
                      (pm(i-1,j-1,0)+pm(i,j-1,0))*uold(i,j-1,k,nrhs)));
 
        const Real on_p =  4.0_rt / (pn(i-1,j-1,0)+pn(i-1,j,0)+pn(i,j-1,0)+pn(i,j,0));
        VFx(i,j,k) = on_p*on_p*visc2_p(i,j,0)*cff;
    });
 
    ParallelFor(ybx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        const Real cff=dt_lev*0.25*(pm(i,j,0)+pm(i,j-1,0))*(pn(i,j,0)+pn(i,j-1,0));
        const Real cff1=0.5*(pn(i,j-1,0)+pn(i,j,0))*(VFx(i+1,j,k)-VFx(i,j  ,k));
        const Real cff2=0.5*(pm(i,j-1,0)+pm(i,j,0))*(VFe(i  ,j,k)-VFe(i,j-1,k));
        const Real cff3=cff*(cff1-cff2);
        amrex::Gpu::Atomic::Add(&(rvfrc(i,j,0)), cff1-cff2);
        v(i,j,k,nnew)=v(i,j,k,nnew)+cff3;
    });
 
}

vert_mean_3d()

void REMORA::vert_mean_3d	(	const amrex::Box &	bx,
		const int	ioff,
		const int	joff,
		const amrex::Array4< amrex::Real > &	phi,
		const amrex::Array4< amrex::Real const > &	Hz,
		const amrex::Array4< amrex::Real const > &	Dphi_avg1,
		const amrex::Array4< amrex::Real > &	DC,
		const amrex::Array4< amrex::Real > &	CF,
		const amrex::Array4< amrex::Real const > &	dxlen,
		const int	nnew,
		const int	N
	)

{
 
    ParallelFor(makeSlab(phi_bx,2,0),
    [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
      Real Hzk_on_face = 0.5*(Hz(i-ioff,j-joff,0)+Hz(i,j,0));
      CF(i,j,-1) =                 Hzk_on_face;
      DC(i,j,-1) = phi(i,j,0,nnew)*Hzk_on_face;
 
      for (int k=1; k<=N; k++) {
          Hzk_on_face = 0.5*(Hz(i-ioff,j-joff,k)+Hz(i,j,k));
          CF(i,j,-1) +=                 Hzk_on_face;
          DC(i,j,-1) += phi(i,j,k,nnew)*Hzk_on_face;
      }
    });
 
    ParallelFor(makeSlab(phi_bx,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        Real cff1=1.0/(CF(i,j,-1)*(1.0/dxlen(i,j,0)));
        DC(i,j,-1) = (DC(i,j,-1)*(1.0/dxlen(i,j,0)) - Dphi_avg1(i,j,0))*cff1; // recursive
    });
 
    ParallelFor(phi_bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        phi(i,j,k) -= DC(i,j,-1);
    });
}

vert_visc_3d()

void REMORA::vert_visc_3d	(	const amrex::Box &	bx,
		const int	ioff,
		const int	joff,
		const amrex::Array4< amrex::Real > &	phi,
		const amrex::Array4< amrex::Real const > &	Hz,
		const amrex::Array4< amrex::Real > &	Hzk,
		const amrex::Array4< amrex::Real > &	AK,
		const amrex::Array4< amrex::Real > &	Akv,
		const amrex::Array4< amrex::Real > &	BC,
		const amrex::Array4< amrex::Real > &	DC,
		const amrex::Array4< amrex::Real > &	FC,
		const amrex::Array4< amrex::Real > &	CF,
		const int	nnew,
		const int	N,
		const amrex::Real	dt_lev
	)

{
    //
    // Put Hzk on the x- or y-face as appropriate, or leave on cell center for tracers
    //
    if (verbose > 1)
        amrex::Print() << "updating on box in vert_visc_3d: " << phi_bx << std::endl;
 
    ParallelFor(phi_bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Hzk(i,j,k)=0.5*(Hz(i-ioff,j-joff,k)+Hz(i,j,k));
    });
 
    //
    // Put Akv on the x- or y-face as appropriate, or leave on cell center for tracers
    //
    ParallelFor(phi_bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        AK(i,j,k) = 0.5 * (Akv(i-ioff,j-joff,k)+Akv(i,j,k));
    });
 
    Gpu::streamSynchronize();
#ifdef AMREX_USE_GPU
    Gpu::synchronize();
#else
#endif
    /////////////////// This and the following loop is the first non-matching thing that affects plotfile comparison for cuda
    // NOTE: vertical viscosity term for tracers is identical except AK=Akt
    const Real sixth = 1.0_rt / 6.0_rt;
    const Real third = 1.0_rt / 3.0_rt;
 
    ParallelFor(makeSlab(phi_bx,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
        for (int k=0; k<=N; k++)
        {
            //
            //  Use conservative, parabolic spline reconstruction of vertical
            //  viscosity derivatives.  Then, time step vertical viscosity term
            //  implicitly by solving a tridiagonal system.
            //
            Real cff;
 
            const Real oHz = 1.0/ Hzk(i,j,k);
 
            FC(i,j,k) = (k >= 1) ? sixth*Hzk(i,j,k  )-dt_lev*AK(i,j,k-1)*oHz :
                                   sixth*Hzk(i,j,k);
 
            if (k < N)
            {
                const Real oHzkp1 = 1.0/ Hzk(i,j,k+1);
 
                CF(i,j,k)=sixth*Hzk(i,j,k+1)-dt_lev*AK(i,j,k+1)*oHzkp1;
 
                if (k==0)
                {
                    BC(i,j,k)=third*(Hzk(i,j,k)+Hzk(i,j,k+1)) + dt_lev*AK(i,j,k)*(oHz+oHzkp1);
 
                    cff=1.0/(BC(i,j,k)-FC(i,j,k)*0.0);
                    CF(i,j,k) *= cff;
                    DC(i,j,k) = cff*(phi(i,j,k+1,nnew)-phi(i,j,k,nnew)-FC(i,j,k)*0.0);
 
                } else {
 
                    BC(i,j,k)=third*(Hzk(i,j,k)+Hzk(i,j,k+1)) + dt_lev*AK(i,j,k)*(oHz+oHzkp1);
                    cff=1.0/(BC(i,j,k)-FC(i,j,k)*CF(i,j,k-1));
                    CF(i,j,k) *= cff;
                    DC(i,j,k) = cff*(phi(i,j,k+1,nnew)-phi(i,j,k,nnew)-FC(i,j,k)*DC(i,j,k-1));
                }
            } // k < N
        } // k
    });
#ifdef AMREX_USE_GPU
    Gpu::synchronize();
#else
#endif
    Gpu::streamSynchronize();
    ParallelFor(makeSlab(phi_bx,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int )
    {
       for(int k=0; k<=N; k++) {
           //
           //  Backward substitution.
           //
           DC(i,j,N)=0.0;
 
           if(N-k+1<=N && N>=k) //-N,1,-1 => kidx =N-k+1
           {
               if(N+1<k || N+2<k) amrex::Abort("-1 here");
               DC(i,j,N-k) -= CF(i,j,N-k)*DC(i,j,N-k+1);
           }
       }
    });
 
#ifdef AMREX_USE_GPU
    Gpu::synchronize();
#endif
 
    ParallelFor(phi_bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        DC(i,j,k) *= AK(i,j,k);
    });
 
    ParallelFor(phi_bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
    {
        Real cff;
        if(k-1>=0) {
            const Real oHz = 1.0/ Hzk(i,j,k);
            cff = dt_lev*oHz*(DC(i,j,k)-DC(i,j,k-1));
        } else {
            const Real oHz = 1.0/ Hzk(i,j,k);
            cff = dt_lev*oHz*(DC(i,j,k));
        }
        phi(i,j,k) += cff;
     });
}

volWgtSumMF()

Real REMORA::volWgtSumMF	(	int	lev,
		const amrex::MultiFab &	mf,
		int	comp,
		bool	local,
		bool	finemask
	)

{
    BL_PROFILE("REMORA::volWgtSumMF()");
 
    Real sum = 0.0;
    MultiFab tmp(grids[lev], dmap[lev], 1, 0);
    MultiFab::Copy(tmp, mf, comp, 0, 1, 0);
 
    if (lev < finest_level && finemask) {
        const MultiFab& mask = build_fine_mask(lev+1);
        MultiFab::Multiply(tmp, mask, 0, 0, 1, 0);
    }
 
    MultiFab volume(grids[lev], dmap[lev], 1, 0);
    auto const& dx = geom[lev].CellSizeArray();
    Real cell_vol = dx[0]*dx[1]*dx[2];
    volume.setVal(cell_vol);
    sum = MultiFab::Dot(tmp, 0, volume, 0, 1, 0, local);
 
    if (!local)
      ParallelDescriptor::ReduceRealSum(sum);
 
    return sum;
}

writeBuildInfo()

void REMORA::writeBuildInfo ( std::ostream &  os )

static

{
  std::string PrettyLine = std::string(78, '=') + "\n";
  std::string OtherLine = std::string(78, '-') + "\n";
  std::string SkipSpace = std::string(8, ' ');
 
  // build information
  os << PrettyLine;
  os << " REMORA Build Information\n";
  os << PrettyLine;
 
  os << "build date:    " << amrex::buildInfoGetBuildDate() << "\n";
  os << "build machine: " << amrex::buildInfoGetBuildMachine() << "\n";
  os << "build dir:     " << amrex::buildInfoGetBuildDir() << "\n";
  os << "AMReX dir:     " << amrex::buildInfoGetAMReXDir() << "\n";
 
  os << "\n";
 
  os << "COMP:          " << amrex::buildInfoGetComp() << "\n";
  os << "COMP version:  " << amrex::buildInfoGetCompVersion() << "\n";
 
  os << "C++ compiler:  " << amrex::buildInfoGetCXXName() << "\n";
  os << "C++ flags:     " << amrex::buildInfoGetCXXFlags() << "\n";
 
  os << "\n";
 
  os << "Link flags:    " << amrex::buildInfoGetLinkFlags() << "\n";
  os << "Libraries:     " << amrex::buildInfoGetLibraries() << "\n";
 
  os << "\n";
 
  for (int n = 1; n <= amrex::buildInfoGetNumModules(); n++) {
    os << amrex::buildInfoGetModuleName(n) << ": "
       << amrex::buildInfoGetModuleVal(n) << "\n";
  }
 
  os << "\n";
  const char* githash1 = amrex::buildInfoGetGitHash(1);
  const char* githash2 = amrex::buildInfoGetGitHash(2);
  if (strlen(githash1) > 0) {
    os << "REMORA       git hash: " << githash1 << "\n";
  }
  if (strlen(githash2) > 0) {
    os << "AMReX       git hash: " << githash2 << "\n";
  }
 
  const char* buildgithash = amrex::buildInfoGetBuildGitHash();
  const char* buildgitname = amrex::buildInfoGetBuildGitName();
  if (strlen(buildgithash) > 0) {
    os << buildgitname << " git hash: " << buildgithash << "\n";
  }
 
  os << "\n";
  os << " REMORA Compile time variables: \n";
 
  os << "\n";
  os << " REMORA Defines: \n";
#ifdef _OPENMP
  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "ON"
     << std::endl;
#else
  os << std::setw(35) << std::left << "_OPENMP " << std::setw(6) << "OFF"
     << std::endl;
#endif
 
#ifdef MPI_VERSION
  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
     << MPI_VERSION << std::endl;
#else
  os << std::setw(35) << std::left << "MPI_VERSION " << std::setw(6)
     << "UNDEFINED" << std::endl;
#endif
 
#ifdef MPI_SUBVERSION
  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
     << MPI_SUBVERSION << std::endl;
#else
  os << std::setw(35) << std::left << "MPI_SUBVERSION " << std::setw(6)
     << "UNDEFINED" << std::endl;
#endif
 
  os << "\n\n";
}

Referenced by main().

Here is the caller graph for this function:

WriteCheckpointFile()

void REMORA::WriteCheckpointFile ( ) const

private

{
    // chk00010            write a checkpoint file with this root directory
    // chk00010/Header     this contains information you need to save (e.g., finest_level, t_new, etc.) and also
    //                     the BoxArrays at each level
    // chk00010/Level_0/
    // chk00010/Level_1/
    // etc.                these subdirectories will hold the MultiFab data at each level of refinement
 
    // checkpoint file name, e.g., chk00010
    const std::string& checkpointname = amrex::Concatenate(check_file,istep[0],5);
 
    amrex::Print() << "Writing checkpoint " << checkpointname << "\n";
 
    const int nlevels = finest_level+1;
 
    // ---- prebuild a hierarchy of directories
    // ---- dirName is built first.  if dirName exists, it is renamed.  then build
    // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
    // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
    // ---- after all directories are built
    // ---- ParallelDescriptor::IOProcessor() creates the directories
    amrex::PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
 
    // write Header file
   if (ParallelDescriptor::IOProcessor()) {
 
       std::string HeaderFileName(checkpointname + "/Header");
       VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
       std::ofstream HeaderFile;
       HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
       HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                               std::ofstream::trunc |
                                               std::ofstream::binary);
       if( ! HeaderFile.good()) {
           amrex::FileOpenFailed(HeaderFileName);
       }
 
       HeaderFile.precision(17);
 
       // write out title line
       HeaderFile << "Checkpoint file for REMORA\n";
 
       // write out finest_level
       HeaderFile << finest_level << "\n";
 
       // write the number of components
       // for each variable we store
 
       // conservative, cell-centered vars
       HeaderFile << NCONS << "\n";
 
       // x-velocity on faces
       HeaderFile << 1 << "\n";
 
       // y-velocity on faces
       HeaderFile << 1 << "\n";
 
       // z-velocity on faces
       HeaderFile << 1 << "\n";
 
       HeaderFile << 2 << "\n";
 
       HeaderFile << 2 << "\n";
 
       HeaderFile << 3 << "\n";
 
       HeaderFile << 3 << "\n";
 
       // write out array of istep
       for (int i = 0; i < istep.size(); ++i) {
           HeaderFile << istep[i] << " ";
       }
       HeaderFile << "\n";
 
       // write out array of dt
       for (int i = 0; i < dt.size(); ++i) {
           HeaderFile << dt[i] << " ";
       }
       HeaderFile << "\n";
 
       // write out array of t_new
       for (int i = 0; i < t_new.size(); ++i) {
           HeaderFile << t_new[i] << " ";
       }
       HeaderFile << "\n";
 
       // write the BoxArray at each level
       for (int lev = 0; lev <= finest_level; ++lev) {
           boxArray(lev).writeOn(HeaderFile);
           HeaderFile << '\n';
       }
   }
 
   // write the MultiFab data to, e.g., chk00010/Level_0/
   // Here we make copies of the MultiFab with no ghost cells
   for (int lev = 0; lev <= finest_level; ++lev)
   {
       BoxList bl2d = grids[lev].boxList();
       for (auto& b : bl2d) {
           b.setRange(2,0);
       }
       BoxArray ba2d(std::move(bl2d));
 
       MultiFab cons(grids[lev],dmap[lev],NCONS,0);
       MultiFab::Copy(cons,*cons_new[lev],0,0,NCONS,0);
       VisMF::Write(cons, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
 
       MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,0);
       MultiFab::Copy(xvel,*xvel_new[lev],0,0,1,0);
       VisMF::Write(xvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
 
       MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,0);
       MultiFab::Copy(yvel,*yvel_new[lev],0,0,1,0);
       VisMF::Write(yvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
 
       MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,0);
       MultiFab::Copy(zvel,*zvel_new[lev],0,0,1,0);
       VisMF::Write(zvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
 
       MultiFab mf_ru(grids[lev],dmap[lev],2,NGROW);
       MultiFab::Copy(mf_ru,*vec_ru[lev],0,0,2,NGROW);
       VisMF::Write(mf_ru, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XRHS"));
 
       MultiFab mf_rv(grids[lev],dmap[lev],2,NGROW);
       MultiFab::Copy(mf_rv,*vec_rv[lev],0,0,2,NGROW);
       VisMF::Write(mf_rv, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YRHS"));
 
       MultiFab mf_ubar(ba2d,dmap[lev],3,NGROW);
       MultiFab::Copy(mf_ubar,*(vec_ubar[lev]),0,0,3,NGROW);
       VisMF::Write(mf_ubar, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XBar"));
 
       MultiFab mf_vbar(ba2d,dmap[lev],3,NGROW);
       MultiFab::Copy(mf_vbar,*(vec_vbar[lev]),0,0,3,NGROW);
       VisMF::Write(mf_vbar, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YBar"));
 
       VisMF::Write(*(vec_rufrc[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "rufrc"));
       VisMF::Write(*(vec_rvfrc[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "rvfrc"));
 
       VisMF::Write(*(vec_sustr[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "sustr"));
       VisMF::Write(*(vec_svstr[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "svstr"));
 
       VisMF::Write(*(vec_rdrag[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "rdrag"));
 
       VisMF::Write(*(vec_bustr[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "bustr"));
       VisMF::Write(*(vec_bvstr[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "bvstr"));
 
       VisMF::Write(*(vec_DU_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "DU_avg1"));
       VisMF::Write(*(vec_DU_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "DU_avg2"));
       VisMF::Write(*(vec_DV_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "DV_avg1"));
       VisMF::Write(*(vec_DV_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "DV_avg2"));
 
       VisMF::Write(*(vec_zeta[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "zeta"));
       VisMF::Write(*(vec_Zt_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Zt_avg1"));
   }
 
#ifdef REMORA_USE_PARTICLES
   particleData.Checkpoint(checkpointname);
#endif
 
#ifdef REMORA_USE_NETCDF
   // Write bdy_data files
   if ( ParallelDescriptor::IOProcessor() && (solverChoice.ic_bc_type == IC_BC_Type::Real) )
   {
 
     // Vector dimensions
     int num_time = bdy_data_xlo.size();
     int num_var  = bdy_data_xlo[0].size();
 
     // Open header file and write to it
     std::ofstream bdy_h_file(amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_H"));
     bdy_h_file << std::setprecision(1) << std::fixed;
     bdy_h_file << num_time << "\n";
     bdy_h_file << num_var  << "\n";
     bdy_h_file << start_bdy_time << "\n";
     bdy_h_file << bdy_time_interval << "\n";
     bdy_h_file << bdy_width << "\n";
     for (int ivar(0); ivar<num_var; ++ivar) {
       bdy_h_file << bdy_data_xlo[0][ivar].box() << "\n";
       bdy_h_file << bdy_data_xhi[0][ivar].box() << "\n";
       bdy_h_file << bdy_data_ylo[0][ivar].box() << "\n";
       bdy_h_file << bdy_data_yhi[0][ivar].box() << "\n";
     }
 
     // Open data file and write to it
     std::ofstream bdy_d_file(amrex::MultiFabFileFullPrefix(0, checkpointname, "Level_", "bdy_D"));
     for (int itime(0); itime<num_time; ++itime) {
       for (int ivar(0); ivar<num_var; ++ivar) {
         bdy_data_xlo[itime][ivar].writeOn(bdy_d_file,0,1);
         bdy_data_xhi[itime][ivar].writeOn(bdy_d_file,0,1);
         bdy_data_ylo[itime][ivar].writeOn(bdy_d_file,0,1);
         bdy_data_yhi[itime][ivar].writeOn(bdy_d_file,0,1);
       }
     }
   }
#endif
}

WriteGenericPlotfileHeaderWithBathymetry()

void REMORA::WriteGenericPlotfileHeaderWithBathymetry	(	std::ostream &	HeaderFile,
		int	nlevels,
		const amrex::Vector< amrex::BoxArray > &	bArray,
		const amrex::Vector< std::string > &	varnames,
		amrex::Real	time,
		const amrex::Vector< int > &	level_steps,
		const std::string &	versionName,
		const std::string &	levelPrefix,
		const std::string &	mfPrefix
	)		const

{
        BL_ASSERT(nlevels <= bArray.size());
        BL_ASSERT(nlevels <= ref_ratio.size()+1);
        BL_ASSERT(nlevels <= level_steps.size());
 
        HeaderFile.precision(17);
 
        // ---- this is the generic plot file type name
        HeaderFile << versionName << '\n';
 
        HeaderFile << varnames.size() << '\n';
 
        for (int ivar = 0; ivar < varnames.size(); ++ivar) {
            HeaderFile << varnames[ivar] << "\n";
        }
        HeaderFile << AMREX_SPACEDIM << '\n';
        HeaderFile << time << '\n';
        HeaderFile << finest_level << '\n';
        for (int i = 0; i < AMREX_SPACEDIM; ++i) {
            HeaderFile << geom[0].ProbLo(i) << ' ';
        }
        HeaderFile << '\n';
        for (int i = 0; i < AMREX_SPACEDIM; ++i) {
            HeaderFile << geom[0].ProbHi(i) << ' ';
        }
        HeaderFile << '\n';
        for (int i = 0; i < finest_level; ++i) {
            HeaderFile << ref_ratio[i][0] << ' ';
        }
        HeaderFile << '\n';
        for (int i = 0; i <= finest_level; ++i) {
            HeaderFile << geom[i].Domain() << ' ';
        }
        HeaderFile << '\n';
        for (int i = 0; i <= finest_level; ++i) {
            HeaderFile << level_steps[i] << ' ';
        }
        HeaderFile << '\n';
        for (int i = 0; i <= finest_level; ++i) {
            for (int k = 0; k < AMREX_SPACEDIM; ++k) {
                HeaderFile << geom[i].CellSize()[k] << ' ';
            }
            HeaderFile << '\n';
        }
        HeaderFile << (int) geom[0].Coord() << '\n';
        HeaderFile << "0\n";
 
        for (int level = 0; level <= finest_level; ++level) {
            HeaderFile << level << ' ' << bArray[level].size() << ' ' << time << '\n';
            HeaderFile << level_steps[level] << '\n';
 
            const IntVect& domain_lo = geom[level].Domain().smallEnd();
            for (int i = 0; i < bArray[level].size(); ++i)
            {
                // Need to shift because the RealBox ctor we call takes the
                // physical location of index (0,0,0).  This does not affect
                // the usual cases where the domain index starts with 0.
                const Box& b = shift(bArray[level][i], -domain_lo);
                RealBox loc = RealBox(b, geom[level].CellSize(), geom[level].ProbLo());
                for (int n = 0; n < AMREX_SPACEDIM; ++n) {
                    HeaderFile << loc.lo(n) << ' ' << loc.hi(n) << '\n';
                }
            }
 
            HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n';
        }
        HeaderFile << "1" << "\n";
        HeaderFile << "3" << "\n";
        HeaderFile << "amrexvec_nu_x" << "\n";
        HeaderFile << "amrexvec_nu_y" << "\n";
        HeaderFile << "amrexvec_nu_z" << "\n";
        std::string mf_nodal_prefix = "Nu_nd";
        for (int level = 0; level <= finest_level; ++level) {
            HeaderFile << MultiFabHeaderPath(level, levelPrefix, mf_nodal_prefix) << '\n';
        }
}

writeJobInfo()

void REMORA::writeJobInfo

(

const std::string &

dir

)

const

{
  // job_info file with details about the run
  std::ofstream jobInfoFile;
  std::string FullPathJobInfoFile = dir;
  FullPathJobInfoFile += "/job_info";
  jobInfoFile.open(FullPathJobInfoFile.c_str(), std::ios::out);
 
  std::string PrettyLine = "==================================================="
                           "============================\n";
  std::string OtherLine = "----------------------------------------------------"
                          "----------------------------\n";
  std::string SkipSpace = "        ";
 
  // job information
  jobInfoFile << PrettyLine;
  jobInfoFile << " REMORA Job Information\n";
  jobInfoFile << PrettyLine;
 
  jobInfoFile << "inputs file: " << inputs_name << "\n\n";
 
  jobInfoFile << "number of MPI processes: "
              << amrex::ParallelDescriptor::NProcs() << "\n";
#ifdef _OPENMP
  jobInfoFile << "number of threads:       " << omp_get_max_threads() << "\n";
#endif
 
  jobInfoFile << "\n";
  jobInfoFile << "CPU time used since start of simulation (CPU-hours): "
              << getCPUTime() / 3600.0;
 
  jobInfoFile << "\n\n";
 
  // plotfile information
  jobInfoFile << PrettyLine;
  jobInfoFile << " Plotfile Information\n";
  jobInfoFile << PrettyLine;
 
  time_t now = time(0);
 
  // Convert now to tm struct for local timezone
  tm* localtm = localtime(&now);
  jobInfoFile << "output data / time: " << asctime(localtm);
 
  std::string currentDir = amrex::FileSystem::CurrentPath();
  jobInfoFile << "output dir:         " << currentDir << "\n";
 
  jobInfoFile << "\n\n";
 
  // build information
  jobInfoFile << PrettyLine;
  jobInfoFile << " Build Information\n";
  jobInfoFile << PrettyLine;
 
  jobInfoFile << "build date:    " << amrex::buildInfoGetBuildDate() << "\n";
  jobInfoFile << "build machine: " << amrex::buildInfoGetBuildMachine() << "\n";
  jobInfoFile << "build dir:     " << amrex::buildInfoGetBuildDir() << "\n";
  jobInfoFile << "AMReX dir:     " << amrex::buildInfoGetAMReXDir() << "\n";
 
  jobInfoFile << "\n";
 
  jobInfoFile << "COMP:          " << amrex::buildInfoGetComp() << "\n";
  jobInfoFile << "COMP version:  " << amrex::buildInfoGetCompVersion() << "\n";
 
  jobInfoFile << "\n";
 
  for (int n = 1; n <= amrex::buildInfoGetNumModules(); n++) {
    jobInfoFile << amrex::buildInfoGetModuleName(n) << ": "
                << amrex::buildInfoGetModuleVal(n) << "\n";
  }
 
  jobInfoFile << "\n";
 
  const char* githash1 = amrex::buildInfoGetGitHash(1);
  const char* githash2 = amrex::buildInfoGetGitHash(2);
  if (strlen(githash1) > 0) {
    jobInfoFile << "REMORA       git hash: " << githash1 << "\n";
  }
  if (strlen(githash2) > 0) {
    jobInfoFile << "AMReX       git hash: " << githash2 << "\n";
  }
 
  const char* buildgithash = amrex::buildInfoGetBuildGitHash();
  const char* buildgitname = amrex::buildInfoGetBuildGitName();
  if (strlen(buildgithash) > 0) {
    jobInfoFile << buildgitname << " git hash: " << buildgithash << "\n";
  }
 
  jobInfoFile << "\n\n";
 
  // grid information
  jobInfoFile << PrettyLine;
  jobInfoFile << " Grid Information\n";
  jobInfoFile << PrettyLine;
 
  int f_lev = finest_level;
 
  for (int i = 0; i <= f_lev; i++) {
    jobInfoFile << " level: " << i << "\n";
    jobInfoFile << "   number of boxes = " << grids[i].size() << "\n";
    jobInfoFile << "   maximum zones   = ";
    for (int n = 0; n < AMREX_SPACEDIM; n++) {
      jobInfoFile << geom[i].Domain().length(n) << " ";
    }
    jobInfoFile << "\n\n";
  }
 
  jobInfoFile << " Boundary conditions\n";
 
  jobInfoFile << "   -x: " << domain_bc_type[0] << "\n";
  jobInfoFile << "   +x: " << domain_bc_type[3] << "\n";
  jobInfoFile << "   -y: " << domain_bc_type[1] << "\n";
  jobInfoFile << "   +y: " << domain_bc_type[4] << "\n";
  jobInfoFile << "   -z: " << domain_bc_type[2] << "\n";
  jobInfoFile << "   +z: " << domain_bc_type[5] << "\n";
 
  jobInfoFile << "\n\n";
 
  // runtime parameters
  jobInfoFile << PrettyLine;
  jobInfoFile << " Inputs File Parameters\n";
  jobInfoFile << PrettyLine;
 
  amrex::ParmParse::dumpTable(jobInfoFile, true);
  jobInfoFile.close();
}

Here is the call graph for this function:

WriteMultiLevelPlotfileWithBathymetry()

void REMORA::WriteMultiLevelPlotfileWithBathymetry	(	const std::string &	plotfilename,
		int	nlevels,
		const amrex::Vector< const amrex::MultiFab * > &	mf,
		const amrex::Vector< const amrex::MultiFab * > &	mf_nd,
		const amrex::Vector< std::string > &	varnames,
		amrex::Real	time,
		const amrex::Vector< int > &	level_steps,
		const std::string &	versionName = "HyperCLaw-V1.1",
		const std::string &	levelPrefix = "Level_",
		const std::string &	mfPrefix = "Cell",
		const amrex::Vector< std::string > &	extra_dirs = amrex::Vector<std::string>()
	)		const

{
    BL_PROFILE("WriteMultiLevelPlotfileWithBathymetry()");
 
    BL_ASSERT(nlevels <= mf.size());
    BL_ASSERT(nlevels <= ref_ratio.size()+1);
    BL_ASSERT(nlevels <= level_steps.size());
    BL_ASSERT(mf[0]->nComp() == varnames.size());
 
    bool callBarrier(false);
    PreBuildDirectorHierarchy(plotfilename, levelPrefix, nlevels, callBarrier);
    if (!extra_dirs.empty()) {
        for (const auto& d : extra_dirs) {
            const std::string ed = plotfilename+"/"+d;
            PreBuildDirectorHierarchy(ed, levelPrefix, nlevels, callBarrier);
        }
    }
    ParallelDescriptor::Barrier();
 
    if (ParallelDescriptor::MyProc() == ParallelDescriptor::NProcs()-1) {
        Vector<BoxArray> boxArrays(nlevels);
        for(int level(0); level < boxArrays.size(); ++level) {
            boxArrays[level] = mf[level]->boxArray();
        }
 
        auto f = [=]() {
            VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
            std::string HeaderFileName(plotfilename + "/Header");
            std::ofstream HeaderFile;
            HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
            HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out   |
                                                    std::ofstream::trunc |
                                                    std::ofstream::binary);
            if( ! HeaderFile.good()) FileOpenFailed(HeaderFileName);
            WriteGenericPlotfileHeaderWithBathymetry(HeaderFile, nlevels, boxArrays, varnames,
                                                     time, level_steps, versionName,
                                                     levelPrefix, mfPrefix);
        };
 
        if (AsyncOut::UseAsyncOut()) {
            AsyncOut::Submit(std::move(f));
        } else {
            f();
        }
    }
 
    std::string mf_nodal_prefix = "Nu_nd";
    for (int level = 0; level <= finest_level; ++level)
    {
        if (AsyncOut::UseAsyncOut()) {
            VisMF::AsyncWrite(*mf[level],
                              MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix),
                              true);
            VisMF::AsyncWrite(*mf_nd[level],
                              MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix),
                              true);
        } else {
            const MultiFab* data;
            std::unique_ptr<MultiFab> mf_tmp;
            if (mf[level]->nGrowVect() != 0) {
                mf_tmp = std::make_unique<MultiFab>(mf[level]->boxArray(),
                                                    mf[level]->DistributionMap(),
                                                    mf[level]->nComp(), 0, MFInfo(),
                                                    mf[level]->Factory());
                MultiFab::Copy(*mf_tmp, *mf[level], 0, 0, mf[level]->nComp(), 0);
                data = mf_tmp.get();
            } else {
                data = mf[level];
            }
            VisMF::Write(*data       , MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix));
            VisMF::Write(*mf_nd[level], MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mf_nodal_prefix));
        }
    }
}

WritePlotFile()

void REMORA::WritePlotFile	(	int	which,
		amrex::Vector< std::string >	plot_var_names
	)

write plotfile to disk

{
    const Vector<std::string> varnames = PlotFileVarNames(plot_var_names);
    const int ncomp_mf = varnames.size();
    const auto ngrow_vars = IntVect(NGROW-1,NGROW-1,0);
 
    // We fillpatch here because some of the derived quantities require derivatives
    //     which require ghost cells to be filled
    for (int lev = 0; lev <= finest_level; ++lev) {
        FillPatch(lev, t_new[lev], *cons_new[lev], cons_new, BdyVars::t);
        FillPatch(lev, t_new[lev], *xvel_new[lev], xvel_new, BdyVars::u);
        FillPatch(lev, t_new[lev], *yvel_new[lev], yvel_new, BdyVars::v);
        FillPatch(lev, t_new[lev], *zvel_new[lev], zvel_new, BdyVars::null);
    }
 
    if (ncomp_mf == 0)
        return;
 
    Vector<MultiFab> mf(finest_level+1);
    for (int lev = 0; lev <= finest_level; ++lev) {
        mf[lev].define(grids[lev], dmap[lev], ncomp_mf, ngrow_vars);
    }
 
    Vector<MultiFab> mf_nd(finest_level+1);
    for (int lev = 0; lev <= finest_level; ++lev) {
        BoxArray nodal_grids(grids[lev]); nodal_grids.surroundingNodes();
        mf_nd[lev].define(nodal_grids, dmap[lev], AMREX_SPACEDIM, 0);
        mf_nd[lev].setVal(0.);
    }
 
    for (int lev = 0; lev <= finest_level; ++lev) {
        int mf_comp = 0;
 
        // First, copy any of the conserved state variables into the output plotfile
        AMREX_ALWAYS_ASSERT(cons_names.size() == NCONS);
        for (int i = 0; i < NCONS; ++i) {
            if (containerHasElement(plot_var_names, cons_names[i])) {
              MultiFab::Copy(mf[lev],*cons_new[lev],i,mf_comp,1,ngrow_vars);
                mf_comp++;
            }
        }
 
        // Next, check for velocities and output none or all
        if (containerHasElement(plot_var_names, "x_velocity") ||
            containerHasElement(plot_var_names, "y_velocity") ||
            containerHasElement(plot_var_names, "z_velocity")) {
            if (plotfile_type == PlotfileType::amrex) {
                Print()<<"We now average the face-based velocity components onto cell centers for plotting "<<std::endl;
            }
            average_face_to_cellcenter(mf[lev],mf_comp,
                                       Array<const MultiFab*,3>{xvel_new[lev],yvel_new[lev],zvel_new[lev]});
            mf_comp += AMREX_SPACEDIM;
        }
 
        // Fill cell-centered location
        Real dx = Geom()[lev].CellSizeArray()[0];
        Real dy = Geom()[lev].CellSizeArray()[1];
 
        // Next, check for location names -- if we write one we write all
        if (containerHasElement(plot_var_names, "x_cc") ||
            containerHasElement(plot_var_names, "y_cc") ||
            containerHasElement(plot_var_names, "z_cc"))
        {
            MultiFab dmf(mf[lev], make_alias, mf_comp, AMREX_SPACEDIM);
            for (MFIter mfi(dmf, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                const Box& bx = mfi.tilebox();
                const Array4<Real> loc_arr = dmf.array(mfi);
                const Array4<Real const> zp_arr = vec_z_phys_nd[lev]->const_array(mfi);
 
                ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                    loc_arr(i,j,k,0) = (i+0.5) * dx;
                    loc_arr(i,j,k,1) = (j+0.5) * dy;
                    loc_arr(i,j,k,2) = 0.125_rt * (zp_arr(i,j  ,k  ) + zp_arr(i+1,j  ,k  ) +
                                                   zp_arr(i,j+1,k  ) + zp_arr(i+1,j+1,k  ) +
                                                   zp_arr(i,j  ,k+1) + zp_arr(i+1,j  ,k+1) +
                                                   zp_arr(i,j+1,k+1) + zp_arr(i+1,j+1,k+1) );
                });
            } // mfi
            mf_comp += AMREX_SPACEDIM;
        } // if containerHasElement
 
#ifdef REMORA_USE_PARTICLES
        if (containerHasElement(plot_var_names, "tracer_particle_count"))
        {
            MultiFab temp_dat(mf[lev].boxArray(), mf[lev].DistributionMap(), 1, 0);
            temp_dat.setVal(0);
            particleData.tracer_particles->Redistribute();
            particleData.tracer_particles->Increment(temp_dat, lev);
            MultiFab::Copy(mf[lev], temp_dat, 0, mf_comp, 1, 0);
            mf_comp += 1;
        }
#endif
 
        MultiFab::Copy(mf_nd[lev],*vec_z_phys_nd[lev],0,2,1,0);
        Real dz = Geom()[lev].CellSizeArray()[2];
        for (MFIter mfi(mf_nd[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi)
        {
            const Box& bx = mfi.tilebox();
            Array4<Real> mf_arr = mf_nd[lev].array(mfi);
            ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) {
                mf_arr(i,j,k,2) -= k * dz;
            });
        } // mfi
 
    } // lev
 
    std::string plotfilename;
    if (which == 1)
       plotfilename = Concatenate(plot_file_1, istep[0], 5);
    else if (which == 2)
       plotfilename = Concatenate(plot_file_2, istep[0], 5);
 
    if (finest_level == 0)
    {
        if (plotfile_type == PlotfileType::amrex) {
            amrex::Print() << "Writing plotfile " << plotfilename << "\n";
            WriteMultiLevelPlotfileWithBathymetry(plotfilename, finest_level+1,
                                                  GetVecOfConstPtrs(mf),
                                                  GetVecOfConstPtrs(mf_nd),
                                                  varnames,
                                                  t_new[0], istep);
            writeJobInfo(plotfilename);
 
#ifdef REMORA_USE_PARTICLES
            particleData.Checkpoint(plotfilename);
#endif
 
#ifdef REMORA_USE_HDF5
        } else if (plotfile_type == PlotfileType::hdf5) {
            amrex::Print() << "Writing plotfile " << plotfilename+"d01.h5" << "\n";
            WriteMultiLevelPlotfileHDF5(plotfilename, finest_level+1,
                                        GetVecOfConstPtrs(mf),
                                        varnames,
                                        Geom(), t_new[0], istep, refRatio());
#endif
#ifdef REMORA_USE_NETCDF
        } else if (plotfile_type == PlotfileType::netcdf) {
             // int lev   = 0;
             // int nc_which = 0;
             // writeNCPlotFile(lev, nc_which, plotfilename, GetVecOfConstPtrs(mf), varnames, istep, t_new[0]);
             // total_plot_file_step_1 += 1;
#endif
        } else {
            amrex::Abort("User specified unknown plot_filetype");
        }
 
    } else { // multilevel
 
        Vector<IntVect>   r2(finest_level);
        Vector<Geometry>  g2(finest_level+1);
        Vector<MultiFab> mf2(finest_level+1);
 
        mf2[0].define(grids[0], dmap[0], ncomp_mf, 0);
 
        // Copy level 0 as is
        MultiFab::Copy(mf2[0],mf[0],0,0,mf[0].nComp(),0);
 
        // Define a new multi-level array of Geometry's so that we pass the new "domain" at lev > 0
        Array<int,AMREX_SPACEDIM> periodicity =
                     {Geom()[0].isPeriodic(0),Geom()[0].isPeriodic(1),Geom()[0].isPeriodic(2)};
        g2[0].define(Geom()[0].Domain(),&(Geom()[0].ProbDomain()),0,periodicity.data());
 
        if (plotfile_type == PlotfileType::amrex) {
            r2[0] = IntVect(1,1,ref_ratio[0][0]);
            for (int lev = 1; lev <= finest_level; ++lev) {
                if (lev > 1) {
                    r2[lev-1][0] = 1;
                    r2[lev-1][1] = 1;
                    r2[lev-1][2] = r2[lev-2][2] * ref_ratio[lev-1][0];
                }
 
                mf2[lev].define(refine(grids[lev],r2[lev-1]), dmap[lev], ncomp_mf, 0);
 
                // Set the new problem domain
                Box d2(Geom()[lev].Domain());
                d2.refine(r2[lev-1]);
 
                g2[lev].define(d2,&(Geom()[lev].ProbDomain()),0,periodicity.data());
            }
 
            // Do piecewise interpolation of mf into mf2
            for (int lev = 1; lev <= finest_level; ++lev) {
                for (MFIter mfi(mf2[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi) {
                    const Box& bx = mfi.tilebox();
                    pcinterp_interp(bx,mf2[lev].array(mfi), 0, mf[lev].nComp(), mf[lev].const_array(mfi),0,r2[lev-1]);
                }
            }
 
            // Define an effective ref_ratio which is isotropic to be passed into WriteMultiLevelPlotfile
            Vector<IntVect> rr(finest_level);
            for (int lev = 0; lev < finest_level; ++lev) {
                rr[lev] = IntVect(ref_ratio[lev][0],ref_ratio[lev][1],ref_ratio[lev][0]);
            }
 
            WriteMultiLevelPlotfile(plotfilename, finest_level+1, GetVecOfConstPtrs(mf2), varnames,
                                    g2, t_new[0], istep, rr);
            writeJobInfo(plotfilename);
 
#ifdef REMORA_USE_PARTICLES
            particleData.Checkpoint(plotfilename);
#endif
        }
    } // end multi-level
}

Here is the call graph for this function:

Member Data Documentation

advflux_reg

amrex::Vector<amrex::YAFluxRegister*> REMORA::advflux_reg

private

Referenced by getAdvFluxReg().

bndry_output_planes_interval

int REMORA::bndry_output_planes_interval

staticprivate

bndry_output_planes_per

amrex::Real REMORA::bndry_output_planes_per

staticprivate

bndry_output_planes_start_time

amrex::Real REMORA::bndry_output_planes_start_time

staticprivate

boxes_at_level

amrex::Vector<amrex::Vector<amrex::Box> > REMORA::boxes_at_level

private

cfl

amrex::Real REMORA::cfl = 0.8

staticprivate

change_max

amrex::Real REMORA::change_max = 1.1

staticprivate

check_file

std::string REMORA::check_file {"chk"}

private

check_int

int REMORA::check_int = -1

private

column_file_name

std::string REMORA::column_file_name

staticprivate

column_interval

int REMORA::column_interval

staticprivate

column_loc_x

amrex::Real REMORA::column_loc_x

staticprivate

column_loc_y

amrex::Real REMORA::column_loc_y

staticprivate

column_per

amrex::Real REMORA::column_per

staticprivate

cons_names

const amrex::Vector<std::string> REMORA::cons_names {"temp", "salt", "scalar"}

private

cons_new

amrex::Vector<amrex::MultiFab*> REMORA::cons_new

cons_old

amrex::Vector<amrex::MultiFab*> REMORA::cons_old

d_havg_density

amrex::Gpu::DeviceVector<amrex::Real> REMORA::d_havg_density

private

d_havg_pressure

amrex::Gpu::DeviceVector<amrex::Real> REMORA::d_havg_pressure

private

d_havg_temperature

amrex::Gpu::DeviceVector<amrex::Real> REMORA::d_havg_temperature

private

datalog

amrex::Vector<std::unique_ptr<std::fstream> > REMORA::datalog

private

Referenced by DataLog(), NumDataLogs(), and setRecordDataInfo().

datalogname

amrex::Vector<std::string> REMORA::datalogname

private

Referenced by DataLogName().

derived_names

const amrex::Vector<std::string> REMORA::derived_names {"dpdx", "dpdy"}

private

do_substep

int REMORA::do_substep

private

domain_bc_type

amrex::Array<std::string,2*AMREX_SPACEDIM> REMORA::domain_bc_type

private

Referenced by writeJobInfo().

domain_bcs_type

amrex::Vector<amrex::BCRec> REMORA::domain_bcs_type

private

domain_bcs_type_d

amrex::Gpu::DeviceVector<amrex::BCRec> REMORA::domain_bcs_type_d

private

dt

amrex::Vector<amrex::Real> REMORA::dt

private

fine_mask

amrex::MultiFab REMORA::fine_mask

private

fixed_dt

amrex::Real REMORA::fixed_dt = -1.0

staticprivate

fixed_fast_dt

amrex::Real REMORA::fixed_fast_dt = -1.0

staticprivate

fixed_ndtfast_ratio

int REMORA::fixed_ndtfast_ratio = 0

staticprivate

h_havg_density

amrex::Vector<amrex::Real> REMORA::h_havg_density

private

h_havg_pressure

amrex::Vector<amrex::Real> REMORA::h_havg_pressure

private

h_havg_temperature

amrex::Vector<amrex::Real> REMORA::h_havg_temperature

private

init_shrink

amrex::Real REMORA::init_shrink = 1.0

staticprivate

input_bndry_planes

int REMORA::input_bndry_planes

staticprivate

input_sounding_file

std::string REMORA::input_sounding_file

staticprivate

istep

amrex::Vector<int> REMORA::istep

private

last_check_file_step

int REMORA::last_check_file_step

private

last_plot_file_step_1

int REMORA::last_plot_file_step_1

private

last_plot_file_step_2

int REMORA::last_plot_file_step_2

private

m_bc_extdir_vals

amrex::Array<amrex::Array<amrex::Real, AMREX_SPACEDIM*2>,AMREX_SPACEDIM+NCONS> REMORA::m_bc_extdir_vals

private

mapfac_m

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::mapfac_m

private

mapfac_u

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::mapfac_u

private

mapfac_v

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::mapfac_v

private

max_step

int REMORA::max_step = std::numeric_limits<int>::max()

private

nc_bdry_file

std::string REMORA::nc_bdry_file

staticprivate

nc_grid_file

amrex::Vector<amrex::Vector<std::string> > REMORA::nc_grid_file

staticprivate

nc_init_file

amrex::Vector<amrex::Vector<std::string> > REMORA::nc_init_file

staticprivate

nfast

int REMORA::nfast

private

nsubsteps

amrex::Vector<int> REMORA::nsubsteps

private

num_boxes_at_level

amrex::Vector<int> REMORA::num_boxes_at_level

private

num_files_at_level

amrex::Vector<int> REMORA::num_files_at_level

private

output_1d_column

int REMORA::output_1d_column

staticprivate

output_bndry_planes

int REMORA::output_bndry_planes

staticprivate

phys_bc_type

amrex::GpuArray<REMORA_BC, AMREX_SPACEDIM*2> REMORA::phys_bc_type

private

physbcs

amrex::Vector<std::unique_ptr<REMORAPhysBCFunct> > REMORA::physbcs

private

plot_file_1

std::string REMORA::plot_file_1 {"plt_1_"}

private

plot_file_2

std::string REMORA::plot_file_2 {"plt_2_"}

private

plot_file_on_restart

int REMORA::plot_file_on_restart = 1

private

plot_int_1

int REMORA::plot_int_1 = -1

private

plot_int_2

int REMORA::plot_int_2 = -1

private

plot_var_names_1

amrex::Vector<std::string> REMORA::plot_var_names_1

private

plot_var_names_2

amrex::Vector<std::string> REMORA::plot_var_names_2

private

plotfile_type

PlotfileType REMORA::plotfile_type = PlotfileType::amrex

staticprivate

pp_prefix

std::string REMORA::pp_prefix {"remora"}

previousCPUTimeUsed

amrex::Real REMORA::previousCPUTimeUsed = 0.0

staticprivate

Referenced by getCPUTime().

ref_tags

Vector< AMRErrorTag > REMORA::ref_tags

staticprivate

regrid_int

int REMORA::regrid_int = 2

private

restart_chkfile

std::string REMORA::restart_chkfile = ""

private

solverChoice

SolverChoice REMORA::solverChoice

staticprivate

sst

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::sst

private

startCPUTime

amrex::Real REMORA::startCPUTime = 0.0

staticprivate

Referenced by getCPUTime().

stop_time

amrex::Real REMORA::stop_time = std::numeric_limits<amrex::Real>::max()

private

sum_interval

int REMORA::sum_interval = -1

staticprivate

sum_per

amrex::Real REMORA::sum_per = -1.0

staticprivate

t_new

amrex::Vector<amrex::Real> REMORA::t_new

private

t_old

amrex::Vector<amrex::Real> REMORA::t_old

private

use_tracer_particles

bool REMORA::use_tracer_particles

staticprivate

vec_Akt

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_Akt

Vertical diffusion coefficient (3D), set in .in file

vec_Akv

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_Akv

Vertical viscosity coefficient (3D), set in .in file

vec_bustr

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_bustr

Bottom stress in the u direction

vec_bvstr

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_bvstr

Bottom stress in the v direction

vec_diff2

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_diff2

Harmonic diffusivity for temperature / salinity

vec_DU_avg1

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_DU_avg1

time average of barotropic x velocity flux (2D)

vec_DU_avg2

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_DU_avg2

correct time average of barotropic x velocity flux for coupling (2D)

vec_DV_avg1

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_DV_avg1

time average of barotropic y velocity flux

vec_DV_avg2

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_DV_avg2

correct time average of barotropic y velocity flux for coupling (2D)

vec_fcor

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_fcor

coriolis factor (2D)

vec_hOfTheConfusingName

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_hOfTheConfusingName

Bathymetry data (2D, positive valued, h in ROMS)

vec_Huon

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_Huon

u-volume flux (3D)

vec_Hvom

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_Hvom

v-volume flux (3D)

vec_Hz

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_Hz

Width of cells in the vertical (z-) direction (3D, Hz in ROMS)

vec_pm

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_pm

map factor (2D)

vec_pn

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_pn

map factor (2D)

vec_rdrag

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_rdrag

Linear drag coefficient [m/s], defined at rho points

vec_rhoA

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_rhoA

vec_rhoS

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_rhoS

vec_ru

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_ru

u velocity RHS (3D, includes horizontal and vertical advection)

vec_rubar

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_rubar

barotropic x velocity for the RHS (2D)

vec_rufrc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_rufrc

u velocity RHS, integrated, including advection and bottom/surface stresses (2D)

vec_rv

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_rv

v velocity RHS (3D, includes horizontal and vertical advection)

vec_rvbar

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_rvbar

barotropic y velocity for the RHS (2D)

vec_rvfrc

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_rvfrc

v velocity RHS, integrated, including advection and bottom/surface stresses (2D)

vec_rzeta

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_rzeta

free surface height for the RHS (2D)

vec_s_r

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_s_r

Scaled vertical coordinate (range [0,1]) that transforms to z, defined at rho points (cell centers)

vec_sstore

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_sstore

saltstore, tempstore, etc

vec_sustr

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_sustr

Surface stress in the u direction

vec_svstr

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_svstr

Surface stress in the v direction

vec_ubar

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_ubar

barotropic x velocity (2D)

vec_vbar

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_vbar

barotropic y velocity (2D)

vec_visc2_p

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_visc2_p

Harmonic viscosity devined on the psi points (corners of horizontal grid cells)

vec_visc2_r

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_visc2_r

Harmonic viscosity devined on the rho points (centers)

vec_visc3d_r

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_visc3d_r

vec_weight1

amrex::Vector<amrex::Real> REMORA::vec_weight1

vec_weight2

amrex::Vector<amrex::Real> REMORA::vec_weight2

vec_x_r

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_x_r

x coordinates at rho points (cell centers)

vec_y_r

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_y_r

y coordinates at rho points (cell centers)

vec_z_phys_nd

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_z_phys_nd

z coordinates at psi points (cell nodes)

vec_z_r

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_z_r

z coordinates at rho points (cell centers)

vec_z_w

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_z_w

z coordinates at w points (faces between z-cells)

vec_zeta

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_zeta

free surface height (2D)

vec_Zt_avg1

amrex::Vector<std::unique_ptr<amrex::MultiFab> > REMORA::vec_Zt_avg1

Average of the free surface, zeta (2D)

verbose

int REMORA::verbose = 0

staticprivate

xvel_new

amrex::Vector<amrex::MultiFab*> REMORA::xvel_new

xvel_old

amrex::Vector<amrex::MultiFab*> REMORA::xvel_old

yvel_new

amrex::Vector<amrex::MultiFab*> REMORA::yvel_new

yvel_old

amrex::Vector<amrex::MultiFab*> REMORA::yvel_old

zvel_new

amrex::Vector<amrex::MultiFab*> REMORA::zvel_new

zvel_old

amrex::Vector<amrex::MultiFab*> REMORA::zvel_old

---

The documentation for this class was generated from the following files:

• /home/runner/work/REMORA/REMORA/Source/REMORA.H
• /home/runner/work/REMORA/REMORA/Source/BoundaryConditions/REMORA_FillPatch.cpp
• /home/runner/work/REMORA/REMORA/Source/Initialization/REMORA_init.cpp
• /home/runner/work/REMORA/REMORA/Source/Initialization/REMORA_init_bcs.cpp
• /home/runner/work/REMORA/REMORA/Source/Initialization/REMORA_make_new_level.cpp
• /home/runner/work/REMORA/REMORA/Source/IO/Checkpoint.cpp
• /home/runner/work/REMORA/REMORA/Source/IO/Plotfile.cpp
• /home/runner/work/REMORA/REMORA/Source/IO/writeJobInfo.cpp
• /home/runner/work/REMORA/REMORA/Source/REMORA.cpp
• /home/runner/work/REMORA/REMORA/Source/REMORA_SumIQ.cpp
• /home/runner/work/REMORA/REMORA/Source/REMORA_Tagging.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_Advance.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_advance_2d.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_advance_2d_onestep.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_advance_3d.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_advance_3d_ml.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_ComputeTimestep.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_coriolis.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_prestep.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_prestep_diffusion.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_prestep_t_advection.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_prsgrd.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_rho_eos.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_rhs_t_3d.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_rhs_uv_2d.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_rhs_uv_3d.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_set_weights.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_setup_step.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_t3dmix.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_TimeStep.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_TimeStepML.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_update_massflux_3d.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_uv3dmix.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_vert_mean_3d.cpp
• /home/runner/work/REMORA/REMORA/Source/TimeIntegration/REMORA_vert_visc_3d.cpp
• /home/runner/work/REMORA/REMORA/Source/Utils/DepthStretchTransform.H