28 amrex::Print() <<
"Writing checkpoint " << checkpointname <<
"\n";
30 const int nlevels = finest_level+1;
38 amrex::PreBuildDirectorHierarchy(checkpointname,
"Level_", nlevels,
true);
41 if (ParallelDescriptor::IOProcessor()) {
43 std::string HeaderFileName(checkpointname +
"/Header");
44 VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
45 std::ofstream HeaderFile;
46 HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
47 HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
48 std::ofstream::trunc |
49 std::ofstream::binary);
50 if( ! HeaderFile.good()) {
51 amrex::FileOpenFailed(HeaderFileName);
54 HeaderFile.precision(17);
57 HeaderFile <<
"Checkpoint file for REMORA\n";
60 HeaderFile << finest_level <<
"\n";
66 HeaderFile <<
NCONS <<
"\n";
69 HeaderFile << 1 <<
"\n";
72 HeaderFile << 1 <<
"\n";
75 HeaderFile << 1 <<
"\n";
77 HeaderFile << 2 <<
"\n";
79 HeaderFile << 2 <<
"\n";
81 HeaderFile << 3 <<
"\n";
83 HeaderFile << 3 <<
"\n";
86 for (
int i = 0; i <
istep.size(); ++i) {
87 HeaderFile <<
istep[i] <<
" ";
92 for (
int i = 0; i <
dt.size(); ++i) {
93 HeaderFile <<
dt[i] <<
" ";
98 for (
int i = 0; i <
t_new.size(); ++i) {
99 HeaderFile <<
t_new[i] <<
" ";
104 for (
int lev = 0; lev <= finest_level; ++lev) {
105 boxArray(lev).writeOn(HeaderFile);
112 for (
int lev = 0; lev <= finest_level; ++lev)
114 BoxList bl2d = grids[lev].boxList();
115 for (
auto& b : bl2d) {
118 BoxArray ba2d(std::move(bl2d));
120 MultiFab cons(grids[lev],dmap[lev],
NCONS,
cons_new[lev]->nGrowVect());
122 VisMF::Write(cons, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Cell"));
125 VisMF::Write(cons, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Cell_old"));
127 MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,
xvel_new[lev]->nGrowVect());
129 VisMF::Write(xvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"XFace"));
132 VisMF::Write(xvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"XFace_old"));
134 MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,
yvel_new[lev]->nGrowVect());
136 VisMF::Write(yvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"YFace"));
139 VisMF::Write(yvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"YFace_old"));
141 MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,
zvel_new[lev]->nGrowVect());
143 VisMF::Write(zvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"ZFace"));
146 VisMF::Write(zvel, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"ZFace_old"));
148 MultiFab mf_ru(convert(grids[lev],IntVect(1,0,0)),dmap[lev],2,(
vec_ru[lev])->nGrowVect());
149 MultiFab::Copy(mf_ru,*
vec_ru[lev],0,0,2,(
vec_ru[lev])->nGrowVect());
150 VisMF::Write(mf_ru, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"XRHS"));
152 MultiFab mf_rv(convert(grids[lev],IntVect(0,1,0)),dmap[lev],2,(
vec_rv[lev])->nGrowVect());
153 MultiFab::Copy(mf_rv,*
vec_rv[lev],0,0,2,(
vec_rv[lev])->nGrowVect());
154 VisMF::Write(mf_rv, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"YRHS"));
156 MultiFab mf_ubar(convert(ba2d,IntVect(1,0,0)),dmap[lev],3,(
vec_ubar[lev])->nGrowVect());
157 MultiFab::Copy(mf_ubar,*(
vec_ubar[lev]),0,0,3,(
vec_ubar[lev])->nGrowVect());
158 VisMF::Write(mf_ubar, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"XBar"));
160 MultiFab mf_vbar(convert(ba2d,IntVect(0,1,0)),dmap[lev],3,(
vec_vbar[lev])->nGrowVect());
161 MultiFab::Copy(mf_vbar,*(
vec_vbar[lev]),0,0,3,(
vec_vbar[lev])->nGrowVect());
162 VisMF::Write(mf_vbar, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"YBar"));
164 MultiFab mf_ru2d(convert(ba2d,IntVect(1,0,0)),dmap[lev],2,(
vec_ru2d[lev])->nGrowVect());
165 MultiFab::Copy(mf_ru2d,*(
vec_ru2d[lev]),0,0,2,(
vec_ru2d[lev])->nGrowVect());
166 VisMF::Write(mf_ru2d, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"XRHS2d"));
168 MultiFab mf_rv2d(convert(ba2d,IntVect(0,1,0)),dmap[lev],2,(
vec_rv2d[lev])->nGrowVect());
169 MultiFab::Copy(mf_rv2d,*(
vec_rv2d[lev]),0,0,2,(
vec_rv2d[lev])->nGrowVect());
170 VisMF::Write(mf_rv2d, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"YRHS2d"));
172 MultiFab mf_mskr(ba2d,dmap[lev],1,(
vec_mskr[lev])->nGrowVect());
173 MultiFab::Copy(mf_mskr,*(
vec_mskr[lev]),0,0,1,(
vec_mskr[lev])->nGrowVect());
174 VisMF::Write(mf_mskr, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Mskr"));
176 MultiFab mf_msku(convert(ba2d,IntVect(1,0,0)),dmap[lev],1,(
vec_msku[lev])->nGrowVect());
177 MultiFab::Copy(mf_msku,*(
vec_msku[lev]),0,0,1,(
vec_msku[lev])->nGrowVect());
178 VisMF::Write(mf_msku, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Msku"));
180 MultiFab mf_mskv(convert(ba2d,IntVect(0,1,0)),dmap[lev],1,(
vec_mskv[lev])->nGrowVect());
181 MultiFab::Copy(mf_mskv,*(
vec_mskv[lev]),0,0,1,(
vec_mskv[lev])->nGrowVect());
182 VisMF::Write(mf_mskv, amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Mskv"));
185 VisMF::Write(*(
vec_rdrag[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"rdrag"));
187 VisMF::Write(*(
vec_rdrag2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"rdrag2"));
192 VisMF::Write(*(
vec_ZoBot[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"ZoBot"));
195 VisMF::Write(*(
vec_DU_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"DU_avg1"));
196 VisMF::Write(*(
vec_DU_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"DU_avg2"));
197 VisMF::Write(*(
vec_DV_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"DV_avg1"));
198 VisMF::Write(*(
vec_DV_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"DV_avg2"));
200 VisMF::Write(*(
vec_zeta[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"zeta"));
201 VisMF::Write(*(
vec_Zt_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Zt_avg1"));
203 VisMF::Write(*(
vec_h[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"h"));
205 VisMF::Write(*(
vec_tke[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"tke"));
206 VisMF::Write(*(
vec_gls[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"gls"));
207 VisMF::Write(*(
vec_Lscale[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Lscale"));
208 VisMF::Write(*(
vec_Akk[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Akk"));
209 VisMF::Write(*(
vec_Akp[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Akp"));
210 VisMF::Write(*(
vec_Akv[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Akv"));
211 VisMF::Write(*(
vec_Akt[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname,
"Level_",
"Akt"));
214#ifdef REMORA_USE_PARTICLES
215 particleData.Checkpoint(checkpointname);
222 amrex::Print() <<
"Restart from checkpoint " <<
restart_chkfile <<
"\n";
227 VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
229 Vector<char> fileCharPtr;
230 ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
231 std::string fileCharPtrString(fileCharPtr.dataPtr());
232 std::istringstream is(fileCharPtrString, std::istringstream::in);
234 std::string line, word;
239 std::getline(is, line);
251 AMREX_ASSERT(chk_ncomp ==
NCONS);
256 AMREX_ASSERT(chk_ncomp == 1);
261 AMREX_ASSERT(chk_ncomp == 1);
266 AMREX_ASSERT(chk_ncomp == 1);
270 AMREX_ASSERT(chk_ncomp == 2);
274 AMREX_ASSERT(chk_ncomp == 2);
278 AMREX_ASSERT(chk_ncomp == 3);
282 AMREX_ASSERT(chk_ncomp == 3);
285 std::getline(is, line);
287 std::istringstream lis(line);
289 while (lis >> word) {
290 istep[i++] = std::stoi(word);
295 std::getline(is, line);
297 std::istringstream lis(line);
299 while (lis >> word) {
300#ifdef AMREX_USE_FLOAT
301 dt[i++] = std::stof(word);
303 dt[i++] = std::stod(word);
309 std::getline(is, line);
311 std::istringstream lis(line);
313 while (lis >> word) {
314#ifdef AMREX_USE_FLOAT
315 t_new[i++] = std::stof(word);
317 t_new[i++] = std::stod(word);
322 for (
int lev = 0; lev <= finest_level; ++lev) {
330 DistributionMapping dm { ba, ParallelDescriptor::NProcs() };
336 for (
int lev = 0; lev <= finest_level; ++lev)
338 BoxList bl2d = grids[lev].boxList();
339 for (
auto& b : bl2d) {
342 BoxArray ba2d(std::move(bl2d));
344 MultiFab cons(grids[lev],dmap[lev],
NCONS,
cons_new[lev]->nGrowVect());
345 VisMF::Read(cons, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"Cell"));
348 VisMF::Read(cons, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"Cell_old"));
351 MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,
xvel_new[lev]->nGrowVect());
352 VisMF::Read(xvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"XFace"));
355 VisMF::Read(xvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"XFace_old"));
358 MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,
yvel_new[lev]->nGrowVect());
359 VisMF::Read(yvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"YFace"));
362 VisMF::Read(yvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"YFace_old"));
365 MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,
zvel_new[lev]->nGrowVect());
366 VisMF::Read(zvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"ZFace"));
369 VisMF::Read(zvel, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"ZFace_old"));
372 MultiFab mf_ru(convert(grids[lev],IntVect(1,0,0)),dmap[lev],2,(
vec_ru[lev])->nGrowVect());
373 VisMF::Read(mf_ru, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"XRHS"));
374 MultiFab::Copy(*(
vec_ru[lev]),mf_ru,0,0,2,(
vec_ru[lev])->nGrowVect());
376 MultiFab mf_rv(convert(grids[lev],IntVect(0,1,0)),dmap[lev],2,(
vec_rv[lev])->nGrowVect());
377 VisMF::Read(mf_rv, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"YRHS"));
378 MultiFab::Copy(*(
vec_rv[lev]),mf_rv,0,0,2,(
vec_rv[lev])->nGrowVect());
380 MultiFab mf_ubar(convert(ba2d,IntVect(1,0,0)),dmap[lev],3,(
vec_ubar[lev])->nGrowVect());
381 VisMF::Read(mf_ubar, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"XBar"));
382 MultiFab::Copy(*(
vec_ubar[lev]),mf_ubar,0,0,3,(
vec_ubar[lev])->nGrowVect());
384 MultiFab mf_vbar(convert(ba2d,IntVect(0,1,0)),dmap[lev],3,(
vec_vbar[lev])->nGrowVect());
385 VisMF::Read(mf_vbar, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"YBar"));
386 MultiFab::Copy(*(
vec_vbar[lev]),mf_vbar,0,0,3,(
vec_vbar[lev])->nGrowVect());
388 MultiFab mf_ru2d(convert(ba2d,IntVect(1,0,0)),dmap[lev],2,(
vec_ru2d[lev])->nGrowVect());
389 VisMF::Read(mf_ru2d, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"XRHS2d"));
390 MultiFab::Copy(*(
vec_ru2d[lev]),mf_ru2d,0,0,2,(
vec_ru2d[lev])->nGrowVect());
392 MultiFab mf_rv2d(convert(ba2d,IntVect(0,1,0)),dmap[lev],2,(
vec_rv2d[lev])->nGrowVect());
393 VisMF::Read(mf_rv2d, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"YRHS2d"));
394 MultiFab::Copy(*(
vec_rv2d[lev]),mf_rv2d,0,0,2,(
vec_rv2d[lev])->nGrowVect());
396 MultiFab mf_mskr(ba2d,dmap[lev],1,(
vec_mskr[lev])->nGrowVect());
397 VisMF::Read(mf_mskr, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"Mskr"));
398 MultiFab::Copy(*(
vec_mskr[lev]),mf_mskr,0,0,1,(
vec_mskr[lev])->nGrowVect());
400 MultiFab mf_msku(convert(ba2d,IntVect(1,0,0)),dmap[lev],1,(
vec_msku[lev])->nGrowVect());
401 VisMF::Read(mf_msku, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"Msku"));
402 MultiFab::Copy(*(
vec_msku[lev]),mf_msku,0,0,1,(
vec_msku[lev])->nGrowVect());
404 MultiFab mf_mskv(convert(ba2d,IntVect(0,1,0)),dmap[lev],1,(
vec_mskv[lev])->nGrowVect());
405 VisMF::Read(mf_mskv, amrex::MultiFabFileFullPrefix(lev,
restart_chkfile,
"Level_",
"Mskv"));
406 MultiFab::Copy(*(
vec_mskv[lev]),mf_mskv,0,0,1,(
vec_mskv[lev])->nGrowVect());
441#ifdef REMORA_USE_PARTICLES