REMORA
Regional Modeling of Oceans Refined Adaptively
Loading...
Searching...
No Matches
REMORA_Checkpoint.cpp
Go to the documentation of this file.
1#include <REMORA.H>
2#include "AMReX_PlotFileUtil.H"
3
4using namespace amrex;
5
6// utility to skip to next line in Header
7void
8REMORA::GotoNextLine (std::istream& is)
9{
10 constexpr std::streamsize bl_ignore_max { 100000 };
11 is.ignore(bl_ignore_max, '\n');
12}
13
14void
15REMORA::WriteCheckpointFile ()
16{
17
18 // chk00010 write a checkpoint file with this root directory
19 // chk00010/Header this contains information you need to save (e.g., finest_level, t_new, etc.) and also
20 // the BoxArrays at each level
21 // chk00010/Level_0/
22 // chk00010/Level_1/
23 // etc. these subdirectories will hold the MultiFab data at each level of refinement
24
25 // checkpoint file name, e.g., chk00010
26 const std::string& checkpointname = amrex::Concatenate(check_file,istep[0],file_min_digits);
27
28 amrex::Print() << "Writing checkpoint " << checkpointname << "\n";
29
30 const int nlevels = finest_level+1;
31
32 // ---- prebuild a hierarchy of directories
33 // ---- dirName is built first. if dirName exists, it is renamed. then build
34 // ---- dirName/subDirPrefix_0 .. dirName/subDirPrefix_nlevels-1
35 // ---- if callBarrier is true, call ParallelDescriptor::Barrier()
36 // ---- after all directories are built
37 // ---- ParallelDescriptor::IOProcessor() creates the directories
38 amrex::PreBuildDirectorHierarchy(checkpointname, "Level_", nlevels, true);
39
40 // write Header file
41 if (ParallelDescriptor::IOProcessor()) {
42
43 std::string HeaderFileName(checkpointname + "/Header");
44 VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size);
45 std::ofstream HeaderFile;
46 HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size());
47 HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out |
48 std::ofstream::trunc |
49 std::ofstream::binary);
50 if( ! HeaderFile.good()) {
51 amrex::FileOpenFailed(HeaderFileName);
52 }
53
54 HeaderFile.precision(17);
55
56 // write out title line
57 HeaderFile << "Checkpoint file for REMORA\n";
58
59 // write out finest_level
60 HeaderFile << finest_level << "\n";
61
62 // write the number of components
63 // for each variable we store
64
65 // conservative, cell-centered vars
66 HeaderFile << ncons << "\n";
67
68 // x-velocity on faces
69 HeaderFile << 1 << "\n";
70
71 // y-velocity on faces
72 HeaderFile << 1 << "\n";
73
74 // z-velocity on faces
75 HeaderFile << 1 << "\n";
76
77 HeaderFile << 2 << "\n";
78
79 HeaderFile << 2 << "\n";
80
81 HeaderFile << 3 << "\n";
82
83 HeaderFile << 3 << "\n";
84
85 // write out array of istep
86 for (int i = 0; i < istep.size(); ++i) {
87 HeaderFile << istep[i] << " ";
88 }
89 HeaderFile << "\n";
90
91 // write out array of dt
92 for (int i = 0; i < dt.size(); ++i) {
93 HeaderFile << dt[i] << " ";
94 }
95 HeaderFile << "\n";
96
97 // write out the time at each level, on the model clock: start_time plus the
98 // elapsed time t_new. Double, like start_time, so the offset survives a
99 // single-precision build. This is the line a reader of the header, such as
100 // the coupled ERF-REMORA driver, takes for the time, so it carries the offset
101 // as a plotfile header's time does.
102 for (int i = 0; i < t_new.size(); ++i) {
103 HeaderFile << model_time(t_new[i]) << " ";
104 }
105 HeaderFile << "\n";
106
107 // write the BoxArray at each level
108 for (int lev = 0; lev <= finest_level; ++lev) {
109 boxArray(lev).writeOn(HeaderFile);
110 HeaderFile << '\n';
111 }
112
113 // The split behind that time: start_time, the origin t_new counts from, and
114 // t_new itself. A restart that keeps start_time takes t_new from here and so
115 // continues the elapsed clock bit for bit, which the rounded sum above cannot
116 // promise. Trailing lines, so a reader that predates them stops before them.
117 HeaderFile << "time_origin " << start_time << "\n";
118 HeaderFile << "elapsed_time";
119 for (int i = 0; i < t_new.size(); ++i) {
120 HeaderFile << " " << t_new[i];
121 }
122 HeaderFile << "\n";
123
124 // History record for this step: the one just written if output came at
125 // this step, otherwise the next. A restart numbers records and chunk
126 // files on from it.
127 const int history_record = (last_plot_file_step == istep[0])
128 ? history_count - 1 : history_count;
129 HeaderFile << "history_count " << history_record << "\n";
130 }
131
132 // write the MultiFab data to, e.g., chk00010/Level_0/
133 // Here we make copies of the MultiFab with no ghost cells
134 for (int lev = 0; lev <= finest_level; ++lev)
135 {
136 BoxList bl2d = grids[lev].boxList();
137 for (auto& b : bl2d) {
138 b.setRange(2,0);
139 }
140 BoxArray ba2d(std::move(bl2d));
141
142 MultiFab cons(grids[lev],dmap[lev],ncons,cons_new[lev]->nGrowVect());
143 MultiFab::Copy(cons,*cons_new[lev],0,0,ncons,cons_new[lev]->nGrowVect());
144 VisMF::Write(cons, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell"));
145
146 MultiFab::Copy(cons,*cons_old[lev],0,0,ncons,cons_old[lev]->nGrowVect());
147 VisMF::Write(cons, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Cell_old"));
148
149 MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,xvel_new[lev]->nGrowVect());
150 MultiFab::Copy(xvel,*xvel_new[lev],0,0,1,xvel_new[lev]->nGrowVect());
151 VisMF::Write(xvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace"));
152
153 MultiFab::Copy(xvel,*xvel_old[lev],0,0,1,xvel_old[lev]->nGrowVect());
154 VisMF::Write(xvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XFace_old"));
155
156 MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,yvel_new[lev]->nGrowVect());
157 MultiFab::Copy(yvel,*yvel_new[lev],0,0,1,yvel_new[lev]->nGrowVect());
158 VisMF::Write(yvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace"));
159
160 MultiFab::Copy(yvel,*yvel_old[lev],0,0,1,yvel_old[lev]->nGrowVect());
161 VisMF::Write(yvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YFace_old"));
162
163 MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,zvel_new[lev]->nGrowVect());
164 MultiFab::Copy(zvel,*zvel_new[lev],0,0,1,zvel_new[lev]->nGrowVect());
165 VisMF::Write(zvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace"));
166
167 MultiFab::Copy(zvel,*zvel_old[lev],0,0,1,zvel_old[lev]->nGrowVect());
168 VisMF::Write(zvel, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZFace_old"));
169
170 MultiFab mf_ru(convert(grids[lev],IntVect(1,0,0)),dmap[lev],2,(vec_ru[lev])->nGrowVect());
171 MultiFab::Copy(mf_ru,*vec_ru[lev],0,0,2,(vec_ru[lev])->nGrowVect());
172 VisMF::Write(mf_ru, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XRHS"));
173
174 MultiFab mf_rv(convert(grids[lev],IntVect(0,1,0)),dmap[lev],2,(vec_rv[lev])->nGrowVect());
175 MultiFab::Copy(mf_rv,*vec_rv[lev],0,0,2,(vec_rv[lev])->nGrowVect());
176 VisMF::Write(mf_rv, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YRHS"));
177
178 MultiFab mf_ubar(convert(ba2d,IntVect(1,0,0)),dmap[lev],3,(vec_ubar[lev])->nGrowVect());
179 MultiFab::Copy(mf_ubar,*(vec_ubar[lev]),0,0,3,(vec_ubar[lev])->nGrowVect());
180 VisMF::Write(mf_ubar, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XBar"));
181
182 MultiFab mf_vbar(convert(ba2d,IntVect(0,1,0)),dmap[lev],3,(vec_vbar[lev])->nGrowVect());
183 MultiFab::Copy(mf_vbar,*(vec_vbar[lev]),0,0,3,(vec_vbar[lev])->nGrowVect());
184 VisMF::Write(mf_vbar, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YBar"));
185
186 MultiFab mf_ru2d(convert(ba2d,IntVect(1,0,0)),dmap[lev],2,(vec_ru2d[lev])->nGrowVect());
187 MultiFab::Copy(mf_ru2d,*(vec_ru2d[lev]),0,0,2,(vec_ru2d[lev])->nGrowVect());
188 VisMF::Write(mf_ru2d, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "XRHS2d"));
189
190 MultiFab mf_rv2d(convert(ba2d,IntVect(0,1,0)),dmap[lev],2,(vec_rv2d[lev])->nGrowVect());
191 MultiFab::Copy(mf_rv2d,*(vec_rv2d[lev]),0,0,2,(vec_rv2d[lev])->nGrowVect());
192 VisMF::Write(mf_rv2d, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YRHS2d"));
193
194 MultiFab mf_mskr(ba2d,dmap[lev],1,(vec_mskr[lev])->nGrowVect());
195 MultiFab::Copy(mf_mskr,*(vec_mskr[lev]),0,0,1,(vec_mskr[lev])->nGrowVect());
196 VisMF::Write(mf_mskr, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Mskr"));
197
198 MultiFab mf_msku(convert(ba2d,IntVect(1,0,0)),dmap[lev],1,(vec_msku[lev])->nGrowVect());
199 MultiFab::Copy(mf_msku,*(vec_msku[lev]),0,0,1,(vec_msku[lev])->nGrowVect());
200 VisMF::Write(mf_msku, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Msku"));
201
202 MultiFab mf_mskv(convert(ba2d,IntVect(0,1,0)),dmap[lev],1,(vec_mskv[lev])->nGrowVect());
203 MultiFab::Copy(mf_mskv,*(vec_mskv[lev]),0,0,1,(vec_mskv[lev])->nGrowVect());
204 VisMF::Write(mf_mskv, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Mskv"));
205
206 if (solverChoice.bottom_stress_type == BottomStressType::linear) {
207 VisMF::Write(*(vec_rdrag[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "rdrag"));
208 } else if (solverChoice.bottom_stress_type == BottomStressType::quadratic) {
209 VisMF::Write(*(vec_rdrag2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "rdrag2"));
210 }
211
212 if (solverChoice.bottom_stress_type == BottomStressType::logarithmic ||
213 solverChoice.vert_mixing_type == VertMixingType::GLS) {
214 VisMF::Write(*(vec_ZoBot[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "ZoBot"));
215 }
216
217 VisMF::Write(*(vec_DU_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "DU_avg1"));
218 VisMF::Write(*(vec_DU_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "DU_avg2"));
219 VisMF::Write(*(vec_DV_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "DV_avg1"));
220 VisMF::Write(*(vec_DV_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "DV_avg2"));
221
222 VisMF::Write(*(vec_zeta[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "zeta"));
223 VisMF::Write(*(vec_Zt_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Zt_avg1"));
224
225 VisMF::Write(*(vec_h[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "h"));
226
227 VisMF::Write(*(vec_tke[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "tke"));
228 VisMF::Write(*(vec_gls[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "gls"));
229 VisMF::Write(*(vec_Lscale[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Lscale"));
230 VisMF::Write(*(vec_Akk[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Akk"));
231 VisMF::Write(*(vec_Akp[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Akp"));
232 VisMF::Write(*(vec_Akv[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Akv"));
233 VisMF::Write(*(vec_Akt[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Akt"));
234 }
235
236#ifdef REMORA_USE_PARTICLES
237 particleData.Checkpoint(checkpointname);
238#endif
239}
240
241void
242REMORA::ReadCheckpointFile ()
243{
244 amrex::Print() << "Restart from checkpoint " << restart_chkfile << "\n";
245
246 // Header
247 std::string File(restart_chkfile + "/Header");
248
249 VisMF::IO_Buffer io_buffer(VisMF::GetIOBufferSize());
250
251 Vector<char> fileCharPtr;
252 ParallelDescriptor::ReadAndBcastFile(File, fileCharPtr);
253 std::string fileCharPtrString(fileCharPtr.dataPtr());
254 std::istringstream is(fileCharPtrString, std::istringstream::in);
255
256 std::string line, word;
257
258 int chk_ncomp;
259
260 // read in title line
261 std::getline(is, line);
262
263 // read in finest_level
264 is >> finest_level;
265 GotoNextLine(is);
266
267 if (finest_level > max_level) {
268 amrex::Abort("Checkpoint file has more levels than amr.max_level");
269 }
270
271 // read the number of components
272 // for each variable we store
273
274 // conservative, cell-centered vars
275 is >> chk_ncomp;
276 GotoNextLine(is);
277 if (chk_ncomp != ncons) {
278 // The tracer count comes from remora.nscalar or, when a biology model is
279 // active, from that model's tracer list, so name both rather than only the
280 // input a biology run never sets.
281 amrex::Abort("Checkpoint holds " + std::to_string(chk_ncomp) +
282 " cell-centered components but this run has " + std::to_string(ncons) +
283 ". The count is set by remora.nscalar and by remora.biology_model; "
284 "both must match the run the checkpoint came from.");
285 }
286
287 // x-velocity on faces
288 is >> chk_ncomp;
289 GotoNextLine(is);
290 AMREX_ASSERT(chk_ncomp == 1);
291
292 // y-velocity on faces
293 is >> chk_ncomp;
294 GotoNextLine(is);
295 AMREX_ASSERT(chk_ncomp == 1);
296
297 // z-velocity on faces
298 is >> chk_ncomp;
299 GotoNextLine(is);
300 AMREX_ASSERT(chk_ncomp == 1);
301
302 is >> chk_ncomp;
303 GotoNextLine(is);
304 AMREX_ASSERT(chk_ncomp == 2);
305
306 is >> chk_ncomp;
307 GotoNextLine(is);
308 AMREX_ASSERT(chk_ncomp == 2);
309
310 is >> chk_ncomp;
311 GotoNextLine(is);
312 AMREX_ASSERT(chk_ncomp == 3);
313
314 is >> chk_ncomp;
315 GotoNextLine(is);
316 AMREX_ASSERT(chk_ncomp == 3);
317
318 // read in array of istep
319 std::getline(is, line);
320 {
321 std::istringstream lis(line);
322 int i = 0;
323 while (lis >> word && i < istep.size()) {
324 istep[i++] = std::stoi(word);
325 }
326 }
327
328 // read in array of dt
329 std::getline(is, line);
330 {
331 std::istringstream lis(line);
332 int i = 0;
333 while (lis >> word && i < dt.size()) {
334#ifdef AMREX_USE_FLOAT
335 dt[i++] = std::stof(word);
336#else
337 dt[i++] = std::stod(word);
338#endif
339 }
340 }
341
342 // read in the time line. Kept in double until the trailing lines say what it
343 // holds: the model-clock time, or elapsed time since an origin given there.
344 Vector<double> chk_time(t_new.size(), 0.0);
345 std::getline(is, line);
346 {
347 std::istringstream lis(line);
348 int i = 0;
349 while (lis >> word && i < chk_time.size()) {
350 chk_time[i++] = std::stod(word);
351 }
352 }
353
354 // read in the BoxArray at each level
355 Vector<BoxArray> chk_ba(finest_level+1);
356 for (int lev = 0; lev <= finest_level; ++lev) {
357 chk_ba[lev].readFrom(is);
358 GotoNextLine(is);
359 }
360
361 // Trailing "keyword value(s)" lines, in any order. Which are present says how
362 // the time line was written:
363 // time_origin and elapsed_time: the time line is on the model clock, and
364 // these carry the start_time and t_new it was summed from;
365 // start_time alone: the time line is t_new, elapsed since that origin;
366 // neither: the time line is on the model clock.
367 bool have_time_origin = false;
368 bool have_start_time = false;
369 double chk_time_origin = 0.0;
370 double chk_start_time = 0.0;
371 Vector<double> chk_elapsed;
372 while (std::getline(is, line)) {
373 std::istringstream lis(line);
374 if (!(lis >> word)) { continue; }
375 if (word == "time_origin") {
376 lis >> chk_time_origin;
377 have_time_origin = true;
378 } else if (word == "elapsed_time") {
379 double value;
380 while (lis >> value) { chk_elapsed.push_back(value); }
381 } else if (word == "start_time") {
382 lis >> chk_start_time;
383 have_start_time = true;
384 } else if (word == "history_count") {
385 // History record for this step. Without it, history output starts over.
386 lis >> history_count;
387 }
388 }
389
390 // A checkpoint that records its origin can say whether this run moved it.
391 // Rebasing keeps the model clock right, but everything keyed to elapsed time
392 // shifts with the origin, so say so rather than let a forcing ramp restart
393 // or an indicator window move without a word.
394 const bool origin_known = have_time_origin || have_start_time;
395 const double chk_origin = have_time_origin ? chk_time_origin : chk_start_time;
396 if (origin_known && chk_origin != start_time && ParallelDescriptor::IOProcessor()) {
397 std::ostringstream msg;
398 msg.precision(17);
399 msg << "remora.start_time is " << start_time << " but the checkpoint "
400 << restart_chkfile << " was written with start_time " << chk_origin
401 << ". The model clock continues where the checkpoint left off, but"
402 << " the elapsed time the solver sees is rebased by "
403 << (chk_origin - start_time) << " s, so anything keyed to elapsed time"
404 << " moves with it: analytic forcing that ramps from the start of the run,"
405 << " refinement indicator start_time/end_time windows, and the"
406 << " plot_int_time/check_int_time cadence. NetCDF time axes and"
407 << " start_total_time/end_total_time windows stay where they were."
408 << " Keep remora.start_time unchanged to continue the run as it was.";
409 amrex::Warning(msg.str());
410 }
411
412 // Rebase onto this run's start_time. Take the elapsed time verbatim when the
413 // origin is unchanged, the usual case, so a restart reproduces the run it
414 // continues.
415 for (int lev = 0; lev < t_new.size(); ++lev) {
416 if (have_time_origin) {
417 const bool same_origin = (chk_time_origin == start_time)
418 && (lev < chk_elapsed.size());
419 t_new[lev] = same_origin ? static_cast<Real>(chk_elapsed[lev])
420 : elapsed_time(chk_time[lev]);
421 } else if (have_start_time) {
422 t_new[lev] = (chk_start_time == start_time)
423 ? static_cast<Real>(chk_time[lev])
424 : elapsed_time(chk_start_time + chk_time[lev]);
425 } else {
426 t_new[lev] = elapsed_time(chk_time[lev]);
427 }
428 }
429
430 for (int lev = 0; lev <= finest_level; ++lev) {
431 // create a distribution mapping
432 DistributionMapping dm { chk_ba[lev], ParallelDescriptor::NProcs() };
433
434 MakeNewLevelFromScratch (lev, t_new[lev], chk_ba[lev], dm);
435 }
436
437 // read in the MultiFab data
438 for (int lev = 0; lev <= finest_level; ++lev)
439 {
440 BoxList bl2d = grids[lev].boxList();
441 for (auto& b : bl2d) {
442 b.setRange(2,0);
443 }
444 BoxArray ba2d(std::move(bl2d));
445
446 MultiFab cons(grids[lev],dmap[lev],ncons,cons_new[lev]->nGrowVect());
447 VisMF::Read(cons, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell"));
448 MultiFab::Copy(*cons_new[lev],cons,0,0,ncons,cons_new[lev]->nGrowVect());
449
450 VisMF::Read(cons, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Cell_old"));
451 MultiFab::Copy(*cons_old[lev],cons,0,0,ncons,cons_old[lev]->nGrowVect());
452
453 MultiFab xvel(convert(grids[lev],IntVect(1,0,0)),dmap[lev],1,xvel_new[lev]->nGrowVect());
454 VisMF::Read(xvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace"));
455 MultiFab::Copy(*xvel_new[lev],xvel,0,0,1,xvel_new[lev]->nGrowVect());
456
457 VisMF::Read(xvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XFace_old"));
458 MultiFab::Copy(*xvel_old[lev],xvel,0,0,1,xvel_old[lev]->nGrowVect());
459
460 MultiFab yvel(convert(grids[lev],IntVect(0,1,0)),dmap[lev],1,yvel_new[lev]->nGrowVect());
461 VisMF::Read(yvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace"));
462 MultiFab::Copy(*yvel_new[lev],yvel,0,0,1,yvel_new[lev]->nGrowVect());
463
464 VisMF::Read(yvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YFace_old"));
465 MultiFab::Copy(*yvel_old[lev],yvel,0,0,1,yvel_old[lev]->nGrowVect());
466
467 MultiFab zvel(convert(grids[lev],IntVect(0,0,1)),dmap[lev],1,zvel_new[lev]->nGrowVect());
468 VisMF::Read(zvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace"));
469 MultiFab::Copy(*zvel_new[lev],zvel,0,0,1,zvel_new[lev]->nGrowVect());
470
471 VisMF::Read(zvel, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZFace_old"));
472 MultiFab::Copy(*zvel_old[lev],zvel,0,0,1,zvel_old[lev]->nGrowVect());
473
474 MultiFab mf_ru(convert(grids[lev],IntVect(1,0,0)),dmap[lev],2,(vec_ru[lev])->nGrowVect());
475 VisMF::Read(mf_ru, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XRHS"));
476 MultiFab::Copy(*(vec_ru[lev]),mf_ru,0,0,2,(vec_ru[lev])->nGrowVect());
477
478 MultiFab mf_rv(convert(grids[lev],IntVect(0,1,0)),dmap[lev],2,(vec_rv[lev])->nGrowVect());
479 VisMF::Read(mf_rv, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YRHS"));
480 MultiFab::Copy(*(vec_rv[lev]),mf_rv,0,0,2,(vec_rv[lev])->nGrowVect());
481
482 MultiFab mf_ubar(convert(ba2d,IntVect(1,0,0)),dmap[lev],3,(vec_ubar[lev])->nGrowVect());
483 VisMF::Read(mf_ubar, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XBar"));
484 MultiFab::Copy(*(vec_ubar[lev]),mf_ubar,0,0,3,(vec_ubar[lev])->nGrowVect());
485
486 MultiFab mf_vbar(convert(ba2d,IntVect(0,1,0)),dmap[lev],3,(vec_vbar[lev])->nGrowVect());
487 VisMF::Read(mf_vbar, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YBar"));
488 MultiFab::Copy(*(vec_vbar[lev]),mf_vbar,0,0,3,(vec_vbar[lev])->nGrowVect());
489
490 MultiFab mf_ru2d(convert(ba2d,IntVect(1,0,0)),dmap[lev],2,(vec_ru2d[lev])->nGrowVect());
491 VisMF::Read(mf_ru2d, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "XRHS2d"));
492 MultiFab::Copy(*(vec_ru2d[lev]),mf_ru2d,0,0,2,(vec_ru2d[lev])->nGrowVect());
493
494 MultiFab mf_rv2d(convert(ba2d,IntVect(0,1,0)),dmap[lev],2,(vec_rv2d[lev])->nGrowVect());
495 VisMF::Read(mf_rv2d, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YRHS2d"));
496 MultiFab::Copy(*(vec_rv2d[lev]),mf_rv2d,0,0,2,(vec_rv2d[lev])->nGrowVect());
497
498 MultiFab mf_mskr(ba2d,dmap[lev],1,(vec_mskr[lev])->nGrowVect());
499 VisMF::Read(mf_mskr, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Mskr"));
500 MultiFab::Copy(*(vec_mskr[lev]),mf_mskr,0,0,1,(vec_mskr[lev])->nGrowVect());
501
502 MultiFab mf_msku(convert(ba2d,IntVect(1,0,0)),dmap[lev],1,(vec_msku[lev])->nGrowVect());
503 VisMF::Read(mf_msku, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Msku"));
504 MultiFab::Copy(*(vec_msku[lev]),mf_msku,0,0,1,(vec_msku[lev])->nGrowVect());
505
506 MultiFab mf_mskv(convert(ba2d,IntVect(0,1,0)),dmap[lev],1,(vec_mskv[lev])->nGrowVect());
507 VisMF::Read(mf_mskv, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Mskv"));
508 MultiFab::Copy(*(vec_mskv[lev]),mf_mskv,0,0,1,(vec_mskv[lev])->nGrowVect());
509
510 if (solverChoice.bottom_stress_type == BottomStressType::linear) {
511 VisMF::Read(*(vec_rdrag[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "rdrag"));
512 } else if (solverChoice.bottom_stress_type == BottomStressType::quadratic) {
513 VisMF::Read(*(vec_rdrag2[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "rdrag2"));
514 }
515
516 if (solverChoice.bottom_stress_type == BottomStressType::logarithmic ||
517 solverChoice.vert_mixing_type == VertMixingType::GLS) {
518 VisMF::Read(*(vec_ZoBot[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "ZoBot"));
519 }
520
521 VisMF::Read(*(vec_DU_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "DU_avg1"));
522 VisMF::Read(*(vec_DU_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "DU_avg2"));
523 VisMF::Read(*(vec_DV_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "DV_avg1"));
524 VisMF::Read(*(vec_DV_avg2[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "DV_avg2"));
525
526 VisMF::Read(*(vec_zeta[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "zeta"));
527 VisMF::Read(*(vec_Zt_avg1[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Zt_avg1"));
528
529 VisMF::Read(*(vec_h[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "h"));
530
531 VisMF::Read(*(vec_tke[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "tke"));
532 VisMF::Read(*(vec_gls[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "gls"));
533 VisMF::Read(*(vec_Lscale[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Lscale"));
534 VisMF::Read(*(vec_Akk[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Akk"));
535 VisMF::Read(*(vec_Akp[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Akp"));
536 VisMF::Read(*(vec_Akt[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Akt"));
537 VisMF::Read(*(vec_Akv[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Akv"));
538
539 stretch_transform(lev);
540
541 }
542
543#ifdef REMORA_USE_PARTICLES
544 particleData.Restart((amrex::ParGDBBase*)GetParGDB(),restart_chkfile);
545#endif
546}
BottomStressType::logarithmic
@ logarithmic
BottomStressType::linear
@ linear
BottomStressType::quadratic
@ quadratic
VertMixingType::GLS
@ GLS
setVal
mf_h setVal(geomdata.ProbHi(2))
ParticleData::Restart
void Restart(amrex::ParGDBBase *a_gdb, const std::string &a_fname)
Definition REMORA_ParticleData.H:77
ParticleData::Checkpoint
void Checkpoint(const std::string &a_fname) const
Definition REMORA_ParticleData.H:54
REMORA::GotoNextLine
static void GotoNextLine(std::istream &is)
utility to skip to next line in Header
Definition REMORA_Checkpoint.cpp:8
REMORA::ncons
int ncons
Number of conserved scalars in the state (temperature + salt + passive scalars + biology tracers)
Definition REMORA.H:1839
REMORA::vec_rv2d
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_rv2d
v velocity RHS (2D, includes horizontal and vertical advection)
Definition REMORA.H:436
REMORA::vec_h
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_h
multilevel data container for current step's z velocities (largely unused; W stored separately)
Definition REMORA.H:413
REMORA::vec_ZoBot
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_ZoBot
Bottom roughness length [m], defined at rho points.
Definition REMORA.H:537
REMORA::vec_DU_avg2
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_DU_avg2
correct time average of barotropic x velocity flux for coupling (2D)
Definition REMORA.H:547
REMORA::cons_new
amrex::Vector< amrex::MultiFab * > cons_new
multilevel data container for current step's scalar data: temperature, salinity, passive tracer
Definition REMORA.H:393
REMORA::stretch_transform
void stretch_transform(int lev)
Calculate vertical stretched coordinates.
Definition REMORA_DepthStretchTransform.H:92
REMORA::vec_mskr
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_mskr
land/sea mask at cell centers (2D)
Definition REMORA.H:584
REMORA::history_count
int history_count
Counter for which time index we are writing to in the netcdf history file.
Definition REMORA.H:1993
REMORA::vec_tke
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_tke
Turbulent kinetic energy.
Definition REMORA.H:667
REMORA::ReadCheckpointFile
void ReadCheckpointFile()
read checkpoint file from disk
Definition REMORA_Checkpoint.cpp:242
REMORA::vec_gls
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_gls
Turbulent generic length scale.
Definition REMORA.H:669
REMORA::vec_ru2d
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_ru2d
u velocity RHS (2D, includes horizontal and vertical advection)
Definition REMORA.H:434
REMORA::zvel_new
amrex::Vector< amrex::MultiFab * > zvel_new
multilevel data container for current step's z velocities (largely unused; W stored separately)
Definition REMORA.H:399
REMORA::start_time
double start_time
Time of the start of the simulation, in seconds on the model clock.
Definition REMORA.H:1822
REMORA::vec_Lscale
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_Lscale
Vertical mixing turbulent length scale.
Definition REMORA.H:671
REMORA::vec_Akt
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_Akt
Vertical diffusion coefficient (3D)
Definition REMORA.H:444
REMORA::vec_msku
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_msku
land/sea mask at x-faces (2D)
Definition REMORA.H:586
REMORA::check_file
std::string check_file
Checkpoint file prefix.
Definition REMORA.H:1982
REMORA::xvel_old
amrex::Vector< amrex::MultiFab * > xvel_old
multilevel data container for last step's x velocities (u in ROMS)
Definition REMORA.H:386
REMORA::yvel_new
amrex::Vector< amrex::MultiFab * > yvel_new
multilevel data container for current step's y velocities (v in ROMS)
Definition REMORA.H:397
REMORA::zvel_old
amrex::Vector< amrex::MultiFab * > zvel_old
multilevel data container for last step's z velocities (largely unused; W stored separately)
Definition REMORA.H:390
REMORA::xvel_new
amrex::Vector< amrex::MultiFab * > xvel_new
multilevel data container for current step's x velocities (u in ROMS)
Definition REMORA.H:395
REMORA::vec_mskv
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_mskv
land/sea mask at y-faces (2D)
Definition REMORA.H:588
REMORA::istep
amrex::Vector< int > istep
which step?
Definition REMORA.H:1733
REMORA::WriteCheckpointFile
void WriteCheckpointFile()
write checkpoint file to disk
Definition REMORA_Checkpoint.cpp:15
REMORA::file_min_digits
static int file_min_digits
Minimum number of digits in plotfile name or chunked history file.
Definition REMORA.H:2061
REMORA::yvel_old
amrex::Vector< amrex::MultiFab * > yvel_old
multilevel data container for last step's y velocities (v in ROMS)
Definition REMORA.H:388
REMORA::vec_DV_avg1
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_DV_avg1
time average of barotropic y velocity flux
Definition REMORA.H:549
REMORA::t_new
amrex::Vector< amrex::Real > t_new
new time at each level, in seconds since start_time
Definition REMORA.H:1744
REMORA::solverChoice
static SolverChoice solverChoice
Container for algorithmic choices.
Definition REMORA.H:2017
REMORA::vec_Akk
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_Akk
Turbulent kinetic energy vertical diffusion coefficient.
Definition REMORA.H:673
REMORA::vec_rdrag2
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_rdrag2
Quadratic drag coefficient [unitless], defined at rho points.
Definition REMORA.H:535
REMORA::vec_ru
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_ru
u velocity RHS (3D, includes horizontal and vertical advection)
Definition REMORA.H:430
REMORA::vec_zeta
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_zeta
free surface height (2D)
Definition REMORA.H:578
REMORA::vec_vbar
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_vbar
barotropic y velocity (2D)
Definition REMORA.H:576
REMORA::vec_DU_avg1
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_DU_avg1
time average of barotropic x velocity flux (2D)
Definition REMORA.H:545
REMORA::vec_ubar
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_ubar
barotropic x velocity (2D)
Definition REMORA.H:574
REMORA::cons_old
amrex::Vector< amrex::MultiFab * > cons_old
multilevel data container for last step's scalar data: temperature, salinity, passive tracer
Definition REMORA.H:384
REMORA::vec_DV_avg2
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_DV_avg2
correct time average of barotropic y velocity flux for coupling (2D)
Definition REMORA.H:551
REMORA::vec_Akv
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_Akv
Vertical viscosity coefficient (3D)
Definition REMORA.H:442
REMORA::MakeNewLevelFromScratch
virtual void MakeNewLevelFromScratch(int lev, amrex::Real time, const amrex::BoxArray &ba, const amrex::DistributionMapping &dm) override
Make a new level from scratch using provided BoxArray and DistributionMapping. Only used during initi...
Definition REMORA_make_new_level.cpp:353
REMORA::elapsed_time
amrex::Real elapsed_time(double time) const noexcept
Elapsed time since start_time of a time on the model clock.
Definition REMORA.H:2216
REMORA::vec_rdrag
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_rdrag
Linear drag coefficient [m/s], defined at rho points.
Definition REMORA.H:533
REMORA::particleData
ParticleData particleData
Particle container with all particle species.
Definition REMORA.H:2030
REMORA::vec_Zt_avg1
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_Zt_avg1
Average of the free surface, zeta (2D)
Definition REMORA.H:476
REMORA::restart_chkfile
std::string restart_chkfile
If set, restart from this checkpoint file.
Definition REMORA.H:1825
REMORA::vec_rv
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_rv
v velocity RHS (3D, includes horizontal and vertical advection)
Definition REMORA.H:432
REMORA::last_plot_file_step
int last_plot_file_step
Step when we last output a plotfile.
Definition REMORA.H:1799
REMORA::model_time
double model_time(amrex::Real elapsed) const noexcept
Time on the model clock, in seconds, of an elapsed time such as t_new.
Definition REMORA.H:2210
REMORA::dt
amrex::Vector< amrex::Real > dt
time step at each level
Definition REMORA.H:1748
REMORA::vec_Akp
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_Akp
Turbulent length scale vertical diffusion coefficient.
Definition REMORA.H:675
SolverChoice::bottom_stress_type
BottomStressType bottom_stress_type
Definition REMORA_DataStruct.H:966
SolverChoice::vert_mixing_type
VertMixingType vert_mixing_type
Definition REMORA_DataStruct.H:976