28 amrex::Print() <<
"Writing checkpoint " << checkpointname <<
"\n";
30 const int nlevels = finest_level+1;
38 amrex::PreBuildDirectorHierarchy(checkpointname,
"Level_", nlevels,
true);
41 if (ParallelDescriptor::IOProcessor()) {
43 std::string HeaderFileName(checkpointname +
"/Header");
44 VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
45 std::ofstream HeaderFile;
46 HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
47 HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
48 std::ofstream::trunc |
49 std::ofstream::binary);
50 if( ! HeaderFile.good()) {
51 amrex::FileOpenFailed(HeaderFileName);
54 HeaderFile.precision(17);
57 HeaderFile <<
"Checkpoint file for REMORA\n";
60 HeaderFile << finest_level <<
"\n";
66 HeaderFile <<
ncons <<
"\n";
69 HeaderFile << 1 <<
"\n";
72 HeaderFile << 1 <<
"\n";
75 HeaderFile << 1 <<
"\n";
77 HeaderFile << 2 <<
"\n";
79 HeaderFile << 2 <<
"\n";
81 HeaderFile << 3 <<
"\n";
83 HeaderFile << 3 <<
"\n";
86 for (
int i = 0; i <
istep.size(); ++i) {
87 HeaderFile <<
istep[i] <<
" ";
92 for (
int i = 0; i <
dt.size(); ++i) {
93 HeaderFile <<
dt[i] <<
" ";
98 for (
int i = 0; i <
t_new.size(); ++i) {
99 HeaderFile <<
t_new[i] <<
" ";
104 for (
int lev = 0; lev <= finest_level; ++lev) {
105 boxArray(lev).writeOn(HeaderFile);
112 for (
int lev = 0; lev <= finest_level; ++lev)
114 BoxList bl2d = grids[lev].boxList();
115 for (
auto& b : bl2d) {
118 BoxArray ba2d(std::move(bl2d));
120 MultiFab cons(grids[lev],dmap[lev],
ncons,
cons_new[lev]->nGrowVect());
122 VisMF::Write(cons, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Cell"));
125 VisMF::Write(cons, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Cell_old"));
127 MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,
xvel_new[lev]->nGrowVect());
129 VisMF::Write(xvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"XFace"));
132 VisMF::Write(xvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"XFace_old"));
134 MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,
yvel_new[lev]->nGrowVect());
136 VisMF::Write(yvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"YFace"));
139 VisMF::Write(yvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"YFace_old"));
141 MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,
zvel_new[lev]->nGrowVect());
143 VisMF::Write(zvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"ZFace"));
146 VisMF::Write(zvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"ZFace_old"));
148 MultiFab mf_ru(convert(grids[lev],IntVect(1,0,0)),dmap[lev],2,(
vec_ru[lev])->nGrowVect());
149 MultiFab::Copy(mf_ru,*
vec_ru[lev],0,0,2,(
vec_ru[lev])->nGrowVect());
150 VisMF::Write(mf_ru, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"XRHS"));
152 MultiFab mf_rv(convert(grids[lev],IntVect(0,1,0)),dmap[lev],2,(
vec_rv[lev])->nGrowVect());
153 MultiFab::Copy(mf_rv,*
vec_rv[lev],0,0,2,(
vec_rv[lev])->nGrowVect());
154 VisMF::Write(mf_rv, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"YRHS"));
156 MultiFab mf_ubar(convert(ba2d,IntVect(1,0,0)),dmap[lev],3,(
vec_ubar[lev])->nGrowVect());
157 MultiFab::Copy(mf_ubar,*(
vec_ubar[lev]),0,0,3,(
vec_ubar[lev])->nGrowVect());
158 VisMF::Write(mf_ubar, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"XBar"));
160 MultiFab mf_vbar(convert(ba2d,IntVect(0,1,0)),dmap[lev],3,(
vec_vbar[lev])->nGrowVect());
161 MultiFab::Copy(mf_vbar,*(
vec_vbar[lev]),0,0,3,(
vec_vbar[lev])->nGrowVect());
162 VisMF::Write(mf_vbar, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"YBar"));
164 MultiFab mf_ru2d(convert(ba2d,IntVect(1,0,0)),dmap[lev],2,(
vec_ru2d[lev])->nGrowVect());
165 MultiFab::Copy(mf_ru2d,*(
vec_ru2d[lev]),0,0,2,(
vec_ru2d[lev])->nGrowVect());
166 VisMF::Write(mf_ru2d, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"XRHS2d"));
168 MultiFab mf_rv2d(convert(ba2d,IntVect(0,1,0)),dmap[lev],2,(
vec_rv2d[lev])->nGrowVect());
169 MultiFab::Copy(mf_rv2d,*(
vec_rv2d[lev]),0,0,2,(
vec_rv2d[lev])->nGrowVect());
170 VisMF::Write(mf_rv2d, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"YRHS2d"));
172 MultiFab mf_mskr(ba2d,dmap[lev],1,(
vec_mskr[lev])->nGrowVect());
173 MultiFab::Copy(mf_mskr,*(
vec_mskr[lev]),0,0,1,(
vec_mskr[lev])->nGrowVect());
174 VisMF::Write(mf_mskr, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Mskr"));
176 MultiFab mf_msku(convert(ba2d,IntVect(1,0,0)),dmap[lev],1,(
vec_msku[lev])->nGrowVect());
177 MultiFab::Copy(mf_msku,*(
vec_msku[lev]),0,0,1,(
vec_msku[lev])->nGrowVect());
178 VisMF::Write(mf_msku, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Msku"));
180 MultiFab mf_mskv(convert(ba2d,IntVect(0,1,0)),dmap[lev],1,(
vec_mskv[lev])->nGrowVect());
181 MultiFab::Copy(mf_mskv,*(
vec_mskv[lev]),0,0,1,(
vec_mskv[lev])->nGrowVect());
182 VisMF::Write(mf_mskv, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Mskv"));
185 VisMF::Write(*(
vec_rdrag[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"rdrag"));
187 VisMF::Write(*(
vec_rdrag2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"rdrag2"));
192 VisMF::Write(*(
vec_ZoBot[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"ZoBot"));
195 VisMF::Write(*(
vec_DU_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"DU_avg1"));
196 VisMF::Write(*(
vec_DU_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"DU_avg2"));
197 VisMF::Write(*(
vec_DV_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"DV_avg1"));
198 VisMF::Write(*(
vec_DV_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"DV_avg2"));
200 VisMF::Write(*(
vec_zeta[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"zeta"));
201 VisMF::Write(*(
vec_Zt_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Zt_avg1"));
203 VisMF::Write(*(
vec_h[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"h"));
205 VisMF::Write(*(
vec_tke[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"tke"));
206 VisMF::Write(*(
vec_gls[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"gls"));
207 VisMF::Write(*(
vec_Lscale[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Lscale"));
208 VisMF::Write(*(
vec_Akk[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Akk"));
209 VisMF::Write(*(
vec_Akp[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Akp"));
210 VisMF::Write(*(
vec_Akv[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Akv"));
211 VisMF::Write(*(
vec_Akt[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Akt"));
214#ifdef REMORA_USE_PARTICLES
215 particleData.Checkpoint(checkpointname);
222 amrex::Print() <<
"Restart from checkpoint " <<
restart_chkfile <<
"\n";
227 VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
229 Vector<char> fileCharPtr;
230 ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
231 std::string fileCharPtrString(fileCharPtr.dataPtr());
232 std::istringstream is(fileCharPtrString, std::istringstream::in);
234 std::string line, word;
239 std::getline(is, line);
251 if (chk_ncomp !=
ncons) {
252 amrex::Abort(
"Checkpoint scalar component count does not match remora.nscalar");
258 AMREX_ASSERT(chk_ncomp == 1);
263 AMREX_ASSERT(chk_ncomp == 1);
268 AMREX_ASSERT(chk_ncomp == 1);
272 AMREX_ASSERT(chk_ncomp == 2);
276 AMREX_ASSERT(chk_ncomp == 2);
280 AMREX_ASSERT(chk_ncomp == 3);
284 AMREX_ASSERT(chk_ncomp == 3);
287 std::getline(is, line);
289 std::istringstream lis(line);
291 while (lis >> word) {
292 istep[i++] = std::stoi(word);
297 std::getline(is, line);
299 std::istringstream lis(line);
301 while (lis >> word) {
302#ifdef AMREX_USE_FLOAT
303 dt[i++] = std::stof(word);
305 dt[i++] = std::stod(word);
311 std::getline(is, line);
313 std::istringstream lis(line);
315 while (lis >> word) {
316#ifdef AMREX_USE_FLOAT
317 t_new[i++] = std::stof(word);
319 t_new[i++] = std::stod(word);
324 for (
int lev = 0; lev <= finest_level; ++lev) {
332 DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
338 for (
int lev = 0; lev <= finest_level; ++lev)
340 BoxList bl2d = grids[lev].boxList();
341 for (
auto& b : bl2d) {
344 BoxArray ba2d(std::move(bl2d));
346 MultiFab cons(grids[lev],dmap[lev],
ncons,
cons_new[lev]->nGrowVect());
347 VisMF::Read(cons, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"Cell"));
350 VisMF::Read(cons, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"Cell_old"));
353 MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,
xvel_new[lev]->nGrowVect());
354 VisMF::Read(xvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"XFace"));
357 VisMF::Read(xvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"XFace_old"));
360 MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,
yvel_new[lev]->nGrowVect());
361 VisMF::Read(yvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"YFace"));
364 VisMF::Read(yvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"YFace_old"));
367 MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,
zvel_new[lev]->nGrowVect());
368 VisMF::Read(zvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"ZFace"));
371 VisMF::Read(zvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"ZFace_old"));
374 MultiFab mf_ru(convert(grids[lev],IntVect(1,0,0)),dmap[lev],2,(
vec_ru[lev])->nGrowVect());
375 VisMF::Read(mf_ru, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"XRHS"));
376 MultiFab::Copy(*(
vec_ru[lev]),mf_ru,0,0,2,(
vec_ru[lev])->nGrowVect());
378 MultiFab mf_rv(convert(grids[lev],IntVect(0,1,0)),dmap[lev],2,(
vec_rv[lev])->nGrowVect());
379 VisMF::Read(mf_rv, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"YRHS"));
380 MultiFab::Copy(*(
vec_rv[lev]),mf_rv,0,0,2,(
vec_rv[lev])->nGrowVect());
382 MultiFab mf_ubar(convert(ba2d,IntVect(1,0,0)),dmap[lev],3,(
vec_ubar[lev])->nGrowVect());
383 VisMF::Read(mf_ubar, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"XBar"));
384 MultiFab::Copy(*(
vec_ubar[lev]),mf_ubar,0,0,3,(
vec_ubar[lev])->nGrowVect());
386 MultiFab mf_vbar(convert(ba2d,IntVect(0,1,0)),dmap[lev],3,(
vec_vbar[lev])->nGrowVect());
387 VisMF::Read(mf_vbar, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"YBar"));
388 MultiFab::Copy(*(
vec_vbar[lev]),mf_vbar,0,0,3,(
vec_vbar[lev])->nGrowVect());
390 MultiFab mf_ru2d(convert(ba2d,IntVect(1,0,0)),dmap[lev],2,(
vec_ru2d[lev])->nGrowVect());
391 VisMF::Read(mf_ru2d, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"XRHS2d"));
392 MultiFab::Copy(*(
vec_ru2d[lev]),mf_ru2d,0,0,2,(
vec_ru2d[lev])->nGrowVect());
394 MultiFab mf_rv2d(convert(ba2d,IntVect(0,1,0)),dmap[lev],2,(
vec_rv2d[lev])->nGrowVect());
395 VisMF::Read(mf_rv2d, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"YRHS2d"));
396 MultiFab::Copy(*(
vec_rv2d[lev]),mf_rv2d,0,0,2,(
vec_rv2d[lev])->nGrowVect());
398 MultiFab mf_mskr(ba2d,dmap[lev],1,(
vec_mskr[lev])->nGrowVect());
399 VisMF::Read(mf_mskr, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"Mskr"));
400 MultiFab::Copy(*(
vec_mskr[lev]),mf_mskr,0,0,1,(
vec_mskr[lev])->nGrowVect());
402 MultiFab mf_msku(convert(ba2d,IntVect(1,0,0)),dmap[lev],1,(
vec_msku[lev])->nGrowVect());
403 VisMF::Read(mf_msku, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"Msku"));
404 MultiFab::Copy(*(
vec_msku[lev]),mf_msku,0,0,1,(
vec_msku[lev])->nGrowVect());
406 MultiFab mf_mskv(convert(ba2d,IntVect(0,1,0)),dmap[lev],1,(
vec_mskv[lev])->nGrowVect());
407 VisMF::Read(mf_mskv, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"Mskv"));
408 MultiFab::Copy(*(
vec_mskv[lev]),mf_mskv,0,0,1,(
vec_mskv[lev])->nGrowVect());
443#ifdef REMORA_USE_PARTICLES