REMORA
Regional Modeling of Oceans Refined Adaptively
Loading...
Searching...
No Matches
REMORA_NCPlotFile.cpp
Go to the documentation of this file.
1#include <iomanip>
2#include <iostream>
3#include <string>
4#include <ctime>
5
6#ifdef _OPENMP
7#include <omp.h>
8#endif
9
10#include <AMReX_Utility.H>
11#include <AMReX_buildInfo.H>
12#include <AMReX_ParmParse.H>
13
14#include "REMORA.H"
15#include "REMORA_NCInterface.H"
16#include "REMORA_NCPlotFile.H"
17#include "REMORA_IndexDefines.H"
18
19using namespace amrex;
20
21/**
22 * @param which_step current step for output
23 */
24void REMORA::WriteNCPlotFile(int which_step) {
25 AMREX_ASSERT(max_level == 0);
26 // For right now we assume single level -- we will generalize this later to multilevel
27 int lev = 0;
28 int which_subdomain = 0;
29 int which_step_in_chunk = -1;
30
31 // Create filename
32 std::string plt_string;
33 std::string plotfilename;
34 if (REMORA::write_history_file) {
35 plotfilename = plot_file_name + "_his";
36 } else {
37 plotfilename = Concatenate(plot_file_name, which_step, file_min_digits);
38 }
39 // If chunking, concatenate with which file we're in
40 if (REMORA::write_history_file and REMORA::chunk_history_file) {
41 int which_chunk = history_count / REMORA::steps_per_history_file;
42 plotfilename = Concatenate(plotfilename, which_chunk, file_min_digits);
43 which_step_in_chunk = history_count - which_chunk * REMORA::steps_per_history_file;
44 }
45
46 // Set the full IO path for NetCDF output
47 std::string FullPath = plotfilename;
48 if (lev == 0) {
49 const std::string &extension = amrex::Concatenate("_d", lev + 1, 2);
50 FullPath += extension + ".nc";
51 } else {
52 const std::string &extension = amrex::Concatenate("_d", lev + 1 + which_subdomain, 2);
53 FullPath += extension + ".nc";
54 }
55
56 //
57 // Check if this file/directory already exists and if so,
58 // have the IOProcessor move the existing
59 // file/directory to filename.old
60 //
61 if ((!REMORA::write_history_file) || (which_step == 0) || (which_step_in_chunk == 0)) {
62 if (amrex::ParallelDescriptor::IOProcessor()) {
63 if (amrex::FileExists(FullPath)) {
64 std::string newoldname(FullPath + ".old." + amrex::UniqueString());
65 amrex::Print() << "WriteNCPlotFile: " << FullPath << " exists. Renaming to: " << newoldname << std::endl;
66 if (std::rename(FullPath.c_str(), newoldname.c_str())) {
67 amrex::Abort("WriteNCPlotfile:: std::rename failed");
68 }
69 }
70 }
71 ParallelDescriptor::Barrier();
72 }
73
74 bool is_history;
75
76 if (REMORA::write_history_file) {
77 is_history = true;
78 bool write_header = !(amrex::FileExists(FullPath));
79
80 auto ncf =
81 write_header ?
82 ncutils::NCFile::create(FullPath, NC_CLOBBER|NC_64BIT_DATA, amrex::ParallelContext::CommunicatorSub(), MPI_INFO_NULL) :
83 ncutils::NCFile::open(FullPath, NC_WRITE, amrex::ParallelContext::CommunicatorSub(), MPI_INFO_NULL);
84
85 amrex::Print() << "Writing into level " << lev << " NetCDF history file " << FullPath << std::endl;
86
87 WriteNCPlotFile_which(lev, which_subdomain, write_header, ncf, is_history);
88
89 } else {
90
91 is_history = false;
92 bool write_header = true;
93
94 // Open new netcdf file to write data
95 auto ncf = ncutils::NCFile::create(FullPath, NC_CLOBBER|NC_64BIT_DATA, amrex::ParallelContext::CommunicatorSub(), MPI_INFO_NULL);
96 amrex::Print() << "Writing level " << lev << " NetCDF plot file " << FullPath << std::endl;
97
98 WriteNCPlotFile_which(lev, which_subdomain, write_header, ncf, is_history);
99 }
100}
101
102/**
103 * @param lev level of data to output
104 * @param which_subdomain index of subdomain if lev != 0
105 * @param write_header whether to write a header
106 * @param ncf netcdf file object
107 * @param is_history whether the file being written is a history file
108 */
109void REMORA::WriteNCPlotFile_which(int lev, int which_subdomain, bool write_header, ncutils::NCFile &ncf, bool is_history) {
110 // Number of cells in this "domain" at this level
111 std::vector<int> n_cells;
112
113 // We only do single-level writes when using NetCDF format
114 int flev = 1; //max_level;
115
116 Box subdomain;
117 if (lev == 0) {
118 subdomain = geom[lev].Domain();
119 } else {
120 subdomain = boxes_at_level[lev][which_subdomain];
121 }
122
123 int nx = subdomain.length(0);
124 int ny = subdomain.length(1);
125 int nz = subdomain.length(2);
126
127 // unsigned long int nt= NC_UNLIMITED;
128 if (is_history && max_step < 0)
129 amrex::Abort("Need to know max_step if writing history file");
130 long long int nt = is_history ? static_cast<long long int>(max_step / std::min(plot_int, max_step)) + 1 : 1;
131 if (chunk_history_file) {
132 // First index of the last history file
133 int last_file_index = REMORA::steps_per_history_file * int(nt / REMORA::steps_per_history_file);
134 if (history_count >= last_file_index) {
135 nt = nt - last_file_index;
136 } else {
137 nt = REMORA::steps_per_history_file;
138 }
139 }
140
141 n_cells.push_back(nx);
142 n_cells.push_back(ny);
143 n_cells.push_back(nz);
144
145 const std::string nt_name = "ocean_time";
146 const std::string ndim_name = "num_geo_dimensions";
147
148 const std::string flev_name = "FINEST_LEVEL";
149
150 const std::string nx_name = "NX";
151 const std::string ny_name = "NY";
152 const std::string nz_name = "NZ";
153
154 const std::string nx_r_name = "xi_rho";
155 const std::string ny_r_name = "eta_rho";
156 const std::string nz_r_name = "s_rho";
157
158 const std::string nx_u_name = "xi_u";
159 const std::string ny_u_name = "eta_u";
160
161 const std::string nx_v_name = "xi_v";
162 const std::string ny_v_name = "eta_v";
163
164 const std::string nx_p_name = "xi_psi";
165 const std::string ny_p_name = "eta_psi";
166 const std::string nz_w_name = "s_w";
167
168 const Real fill_value = 1.0e37_rt;
169
170 if (write_header) {
171 ncf.enter_def_mode();
172 ncf.put_attr("title", "REMORA data ");
173 ncf.def_dim(nt_name, nt);
174 ncf.def_dim(ndim_name, AMREX_SPACEDIM);
175
176 ncf.def_dim(nx_r_name, nx + 2);
177 ncf.def_dim(ny_r_name, ny + 2);
178 ncf.def_dim(nz_r_name, nz);
179
180 ncf.def_dim(nx_u_name, nx + 1);
181 ncf.def_dim(ny_u_name, ny + 2);
182
183 ncf.def_dim(nx_v_name, nx + 2);
184 ncf.def_dim(ny_v_name, ny + 1);
185
186 ncf.def_dim(nx_p_name, nx + 1);
187 ncf.def_dim(ny_p_name, ny + 1);
188
189 ncf.def_dim(nz_w_name, nz + 1);
190
191 ncf.def_dim(flev_name, flev);
192
193 ncf.def_dim(nx_name, n_cells[0]);
194 ncf.def_dim(ny_name, n_cells[1]);
195 ncf.def_dim(nz_name, n_cells[2]);
196
197 ncf.def_var("probLo", ncutils::NCDType::Real, { ndim_name });
198 ncf.var("probLo").put_attr("long_name","Low side of problem domain in internal AMReX grid");
199 ncf.var("probLo").put_attr("units","meter");
200 ncf.def_var("probHi", ncutils::NCDType::Real, { ndim_name });
201 ncf.var("probHi").put_attr("long_name","High side of problem domain in internal AMReX grid");
202 ncf.var("probHi").put_attr("units","meter");
203
204 ncf.def_var("Geom.smallend", NC_INT, { flev_name, ndim_name });
205 ncf.var("Geom.smallend").put_attr("long_name","Low side of problem domain in index space");
206 ncf.def_var("Geom.bigend", NC_INT, { flev_name, ndim_name });
207 ncf.var("Geom.bigend").put_attr("long_name","High side of problem domain in index space");
208 ncf.def_var("CellSize", ncutils::NCDType::Real, { flev_name, ndim_name });
209 ncf.var("CellSize").put_attr("long_name","Cell size on internal AMReX grid");
210 ncf.var("CellSize").put_attr("units","meter");
211
212 ncf.def_var("theta_s",ncutils::NCDType::Real,{});
213 ncf.var("theta_s").put_attr("long_name","S-coordinate surface control parameter");
214 ncf.def_var("theta_b",ncutils::NCDType::Real,{});
215 ncf.var("theta_b").put_attr("long_name","S-coordinate bottom control parameter");
216 ncf.def_var("hc",ncutils::NCDType::Real,{});
217 ncf.var("hc").put_attr("long_name","S-coordinate parameter, critical depth");
218 ncf.var("hc").put_attr("units","meter");
219
220 ncf.def_var("grid",NC_INT, {});
221 ncf.var("grid").put_attr("cf_role","grid_topology");
222 ncf.var("grid").put_attr("topology_dimension",std::vector({2}));
223 ncf.var("grid").put_attr("node_dimensions", "xi_psi eta_psi");
224 ncf.var("grid").put_attr("face_dimensions", "xi_rho: xi_psi (padding: both) eta_rho: eta_psi (padding: both)");
225 ncf.var("grid").put_attr("edge1_dimensions", "xi_u: xi_psi eta_u: eta_psi (padding: both)");
226 ncf.var("grid").put_attr("edge2_dimensions", "xi_v: xi_psi (padding: both) eta_v: eta_psi");
227 ncf.var("grid").put_attr("node_coordinates", "x_psi y_psi");
228 ncf.var("grid").put_attr("face_coordinates", "x_rho y_rho");
229 ncf.var("grid").put_attr("edge1_coordinates", "x_u y_u");
230 ncf.var("grid").put_attr("edge2_coordinates", "x_v y_v");
231 ncf.var("grid").put_attr("vertical_dimensions", "s_rho: s_w (padding: none)");
232
233 ncf.def_var("s_rho",ncutils::NCDType::Real, {nz_r_name});
234 ncf.var("s_rho").put_attr("long_name","S-coordinate at RHO-points");
235 ncf.var("s_rho").put_attr("field","s_rho, scalar");
236
237 ncf.def_var("s_w",ncutils::NCDType::Real, {nz_w_name});
238 ncf.var("s_w").put_attr("long_name","S-coordinate at W-points");
239 ncf.var("s_w").put_attr("field","s_w, scalar");
240
241 ncf.def_var("pm",ncutils::NCDType::Real, {ny_r_name, nx_r_name});
242 ncf.var("pm").put_attr("long_name","curvilinear coordinate metric in XI");
243 ncf.var("pm").put_attr("units","meter-1");
244 ncf.var("pm").put_attr("grid","grid");
245 ncf.var("pm").put_attr("location","face");
246 ncf.var("pm").put_attr("coordinates","x_rho y_rho");
247 ncf.var("pm").put_attr("field","pm, scalar");
248
249 ncf.def_var("pn",ncutils::NCDType::Real, {ny_r_name, nx_r_name});
250 ncf.var("pn").put_attr("long_name","curvilinear coordinate metric in ETA");
251 ncf.var("pn").put_attr("units","meter-1");
252 ncf.var("pn").put_attr("grid","grid");
253 ncf.var("pn").put_attr("location","face");
254 ncf.var("pn").put_attr("coordinates","x_rho y_rho");
255 ncf.var("pn").put_attr("field","pn, scalar");
256
257 ncf.def_var("f",ncutils::NCDType::Real, {ny_r_name, nx_r_name});
258 ncf.var("f").put_attr("long_name","Coriolis parameter at RHO-points");
259 ncf.var("f").put_attr("units","second-1");
260 ncf.var("f").put_attr("grid","grid");
261 ncf.var("f").put_attr("location","face");
262 ncf.var("f").put_attr("coordinates","x_rho y_rho");
263 ncf.var("f").put_attr("field","coriolis, scalar");
264
265 ncf.def_var("x_rho",ncutils::NCDType::Real, {ny_r_name, nx_r_name});
266 ncf.var("x_rho").put_attr("long_name","x-locations of RHO-points");
267 ncf.var("x_rho").put_attr("units","meter");
268 ncf.var("x_rho").put_attr("field","x_rho, scalar");
269
270 ncf.def_var("y_rho",ncutils::NCDType::Real, {ny_r_name, nx_r_name});
271 ncf.var("y_rho").put_attr("long_name","y-locations of RHO-points");
272 ncf.var("y_rho").put_attr("units","meter");
273 ncf.var("y_rho").put_attr("field","y_rho, scalar");
274
275 ncf.def_var("x_u",ncutils::NCDType::Real, {ny_u_name, nx_u_name});
276 ncf.var("x_u").put_attr("long_name","x-locations of U-points");
277 ncf.var("x_u").put_attr("units","meter");
278 ncf.var("x_u").put_attr("field","x_u, scalar");
279
280 ncf.def_var("y_u",ncutils::NCDType::Real, {ny_u_name, nx_u_name});
281 ncf.var("y_u").put_attr("long_name","y-locations of U-points");
282 ncf.var("y_u").put_attr("units","meter");
283 ncf.var("y_u").put_attr("field","y_u, scalar");
284
285 ncf.def_var("x_v",ncutils::NCDType::Real, {ny_v_name, nx_v_name});
286 ncf.var("x_v").put_attr("long_name","x-locations of V-points");
287 ncf.var("x_v").put_attr("units","meter");
288 ncf.var("x_v").put_attr("field","x_v, scalar");
289
290 ncf.def_var("y_v",ncutils::NCDType::Real, {ny_v_name, nx_v_name});
291 ncf.var("y_v").put_attr("long_name","y-locations of V-points");
292 ncf.var("y_v").put_attr("units","meter");
293 ncf.var("y_v").put_attr("field","y_v, scalar");
294
295 ncf.def_var("x_psi",ncutils::NCDType::Real, {ny_p_name, nx_p_name});
296 ncf.var("x_psi").put_attr("long_name","x-locations of PSI-points");
297 ncf.var("x_psi").put_attr("units","meter");
298 ncf.var("x_psi").put_attr("field","x_psi, scalar");
299
300 ncf.def_var("y_psi",ncutils::NCDType::Real, {ny_p_name, nx_p_name});
301 ncf.var("y_psi").put_attr("long_name","y-locations of PSI-points");
302 ncf.var("y_psi").put_attr("units","meter");
303 ncf.var("y_psi").put_attr("field","y_psi, scalar");
304
305 ncf.def_var("ocean_time", ncutils::NCDType::Real, { nt_name });
306 ncf.var("ocean_time").put_attr("long_name","time since initialization");
307 ncf.var("ocean_time").put_attr("units","seconds since 0001-01-01 00:00:00");
308 ncf.var("ocean_time").put_attr("field","time, scalar, series");
309
310 ncf.def_var("Cs_r", ncutils::NCDType::Real, {nz_r_name});
311 ncf.var("Cs_r").put_attr("long_name", "S-coordinate stretching curves at RHO points");
312 ncf.var("Cs_r").put_attr("valid_min",std::vector({-1.}));
313 ncf.var("Cs_r").put_attr("valid_max",std::vector({0.}));
314 ncf.var("Cs_r").put_attr("field","Cs_r, scalar");
315
316 ncf.def_var("Cs_w", ncutils::NCDType::Real, {nz_w_name});
317 ncf.var("Cs_w").put_attr("long_name", "S-coordinate stretching curves at W points");
318 ncf.var("Cs_w").put_attr("valid_min",std::vector({-1.}));
319 ncf.var("Cs_w").put_attr("valid_max",std::vector({0.}));
320 ncf.var("Cs_w").put_attr("field","Cs_w, scalar");
321
322 ncf.def_var("h", ncutils::NCDType::Real, { ny_r_name, nx_r_name });
323 ncf.var("h").put_attr("long_name","bathymetry at RHO-points");
324 ncf.var("h").put_attr("units","meter");
325 ncf.var("h").put_attr("grid","grid");
326 ncf.var("h").put_attr("location","face");
327 ncf.var("h").put_attr("coordinates","x_rho y_rho");
328 ncf.var("h").put_attr("field","bath, scalar");
329
330 ncf.def_var_fill("zeta", ncutils::NCDType::Real, { nt_name, ny_r_name, nx_r_name }, &fill_value);
331 ncf.var("zeta").put_attr("long_name","free-surface");
332 ncf.var("zeta").put_attr("units","meter");
333 ncf.var("zeta").put_attr("time","ocean_time");
334 ncf.var("zeta").put_attr("grid","grid");
335 ncf.var("zeta").put_attr("location","face");
336 ncf.var("zeta").put_attr("coordinates","x_rho y_rho ocean_time");
337 ncf.var("zeta").put_attr("field","free-surface, scalar, series");
338
339 ncf.def_var_fill("temp", ncutils::NCDType::Real, { nt_name, nz_r_name, ny_r_name, nx_r_name }, &fill_value);
340 ncf.var("temp").put_attr("long_name","potential temperature");
341 ncf.var("temp").put_attr("units","Celsius");
342 ncf.var("temp").put_attr("time","ocean_time");
343 ncf.var("temp").put_attr("grid","grid");
344 ncf.var("temp").put_attr("location","face");
345 ncf.var("temp").put_attr("coordinates","x_rho y_rho s_rho ocean_time");
346 ncf.var("temp").put_attr("field","temperature, scalar, series");
347
348 ncf.def_var_fill("salt", ncutils::NCDType::Real, { nt_name, nz_r_name, ny_r_name, nx_r_name }, &fill_value);
349 ncf.var("salt").put_attr("long_name","salinity");
350 ncf.var("salt").put_attr("time","ocean_time");
351 ncf.var("salt").put_attr("grid","grid");
352 ncf.var("salt").put_attr("location","face");
353 ncf.var("salt").put_attr("coordinates","x_rho y_rho s_rho ocean_time");
354 ncf.var("salt").put_attr("field","salinity, scalar, series");
355
356 ncf.def_var_fill("tracer", ncutils::NCDType::Real, { nt_name, nz_r_name, ny_r_name, nx_r_name }, &fill_value);
357 ncf.var("tracer").put_attr("long_name","passive tracer");
358 ncf.var("tracer").put_attr("time","ocean_time");
359 ncf.var("tracer").put_attr("grid","grid");
360 ncf.var("tracer").put_attr("location","face");
361 ncf.var("tracer").put_attr("coordinates","x_rho y_rho s_rho ocean_time");
362 ncf.var("tracer").put_attr("field","tracer, scalar, series");
363
364 ncf.def_var_fill("u", ncutils::NCDType::Real, { nt_name, nz_r_name, ny_u_name, nx_u_name }, &fill_value);
365 ncf.var("u").put_attr("long_name","u-momentum component");
366 ncf.var("u").put_attr("units","meter second-1");
367 ncf.var("u").put_attr("time","ocean_time");
368 ncf.var("u").put_attr("grid","grid");
369 ncf.var("u").put_attr("location","edge1");
370 ncf.var("u").put_attr("coordinates","x_u y_u s_rho ocean_time");
371 ncf.var("u").put_attr("field","u-velocity, scalar, series");
372
373 ncf.def_var_fill("v", ncutils::NCDType::Real, { nt_name, nz_r_name, ny_v_name, nx_v_name }, &fill_value);
374 ncf.var("v").put_attr("long_name","v-momentum component");
375 ncf.var("v").put_attr("units","meter second-1");
376 ncf.var("v").put_attr("time","ocean_time");
377 ncf.var("v").put_attr("grid","grid");
378 ncf.var("v").put_attr("location","edge2");
379 ncf.var("v").put_attr("coordinates","x_v y_v s_rho ocean_time");
380 ncf.var("v").put_attr("field","v-velocity, scalar, series");
381
382 ncf.def_var_fill("ubar", ncutils::NCDType::Real, { nt_name, ny_u_name, nx_u_name }, &fill_value);
383 ncf.var("ubar").put_attr("long_name","vertically integrated u-momentum component");
384 ncf.var("ubar").put_attr("units","meter second-1");
385 ncf.var("ubar").put_attr("time","ocean_time");
386 ncf.var("ubar").put_attr("grid","grid");
387 ncf.var("ubar").put_attr("location","edge1");
388 ncf.var("ubar").put_attr("coordinates","x_u y_u ocean_time");
389 ncf.var("ubar").put_attr("field","ubar-velocity, scalar, series");
390
391 ncf.def_var_fill("vbar", ncutils::NCDType::Real, { nt_name, ny_v_name, nx_v_name }, &fill_value);
392 ncf.var("vbar").put_attr("long_name","vertically integrated v-momentum component");
393 ncf.var("vbar").put_attr("units","meter second-1");
394 ncf.var("vbar").put_attr("time","ocean_time");
395 ncf.var("vbar").put_attr("grid","grid");
396 ncf.var("vbar").put_attr("location","edge2");
397 ncf.var("vbar").put_attr("coordinates","x_v y_v ocean_time");
398 ncf.var("vbar").put_attr("field","vbar-velocity, scalar, series");
399
400 ncf.def_var("sustr", ncutils::NCDType::Real, { nt_name, ny_u_name, nx_u_name });
401 ncf.var("sustr").put_attr("long_name","surface u-momentum stress");
402 ncf.var("sustr").put_attr("units","newton meter-2");
403 ncf.var("sustr").put_attr("time","ocean_time");
404 ncf.var("sustr").put_attr("grid","grid");
405 ncf.var("sustr").put_attr("location","edge1");
406 ncf.var("sustr").put_attr("coordinates","x_u y_u ocean_time");
407 ncf.var("sustr").put_attr("field","surface u-momentum stress, scalar, series");
408
409 ncf.def_var("svstr", ncutils::NCDType::Real, { nt_name, ny_v_name, nx_v_name });
410 ncf.var("svstr").put_attr("long_name","surface v-momentum stress");
411 ncf.var("svstr").put_attr("units","newton meter-2");
412 ncf.var("svstr").put_attr("time","ocean_time");
413 ncf.var("svstr").put_attr("grid","grid");
414 ncf.var("svstr").put_attr("location","edge2");
415 ncf.var("svstr").put_attr("coordinates","x_v y_v ocean_time");
416 ncf.var("svstr").put_attr("field","surface v-momentum stress, scalar, series");
417
418 // Right now this is hard-wired to {temp, salt, tracer, u, v}
419 ncf.put_attr("space_dimension", std::vector<int> { AMREX_SPACEDIM });
420// ncf.put_attr("current_time", std::vector<double> { time });
421 ncf.put_attr("start_time", std::vector<double> { start_bdy_time });
422 ncf.put_attr("CurrentLevel", std::vector<int> { flev });
423 ncf.put_attr("DefaultGeometry", std::vector<int> { amrex::DefaultGeometry().Coord() });
424
425 ncf.exit_def_mode();
426
427 // We are doing single-level writes but it doesn't have to be level 0
428 //
429 // Write out the header information.
430 //
431
432 Real dx[AMREX_SPACEDIM];
433 for (int i = 0; i < AMREX_SPACEDIM; i++) {
434 dx[i] = geom[lev].CellSize()[i];
435 }
436 const auto *base = geom[lev].ProbLo();
437 RealBox rb(subdomain, dx, base);
438
439 amrex::Vector<Real> probLo;
440 amrex::Vector<Real> probHi;
441 for (int i = 0; i < AMREX_SPACEDIM; i++) {
442 probLo.push_back(rb.lo(i));
443 probHi.push_back(rb.hi(i));
444 }
445
446 //nc_probLo.par_access(NC_COLLECTIVE);
447 // small variable data written by just the master proc
448 ncmpi_begin_indep_data(ncf.ncid);
449 if (amrex::ParallelDescriptor::IOProcessor()) // only master proc
450 {
451 auto nc_probLo = ncf.var("probLo");
452
453 nc_probLo.put(probLo.data(), { 0 }, { AMREX_SPACEDIM });
454
455 auto nc_probHi = ncf.var("probHi");
456 //nc_probHi.par_access(NC_COLLECTIVE);
457 nc_probHi.put(probHi.data(), { 0 }, { AMREX_SPACEDIM });
458
459 amrex::Vector<int> smallend;
460 amrex::Vector<int> bigend;
461 for (int i = lev; i < flev; i++) {
462 smallend.clear();
463 bigend.clear();
464 for (int j = 0; j < AMREX_SPACEDIM; j++) {
465 smallend.push_back(subdomain.smallEnd(j));
466 bigend.push_back(subdomain.bigEnd(j));
467 }
468 auto nc_Geom_smallend = ncf.var("Geom.smallend");
469 //nc_Geom_smallend.par_access(NC_COLLECTIVE);
470 nc_Geom_smallend.put(smallend.data(), { static_cast<long long int>(i - lev), 0 }, { 1,
471 AMREX_SPACEDIM });
472
473 auto nc_Geom_bigend = ncf.var("Geom.bigend");
474 //nc_Geom_bigend.par_access(NC_COLLECTIVE);
475 nc_Geom_bigend.put(bigend.data(), { static_cast<long long int>(i - lev), 0 }, { 1,
476 AMREX_SPACEDIM });
477 }
478
479 amrex::Vector<Real> CellSize;
480 for (int i = lev; i < flev; i++) {
481 CellSize.clear();
482 for (Real &j : dx) {
483 CellSize.push_back(amrex::Real(j));
484 }
485 auto nc_CellSize = ncf.var("CellSize");
486 //nc_CellSize.par_access(NC_COLLECTIVE);
487 nc_CellSize.put(CellSize.data(), { static_cast<long long int>(i - lev), 0 }, { 1,
488 AMREX_SPACEDIM });
489 }
490 Real hc = solverChoice.tcline;
491 Real theta_s = solverChoice.theta_s;
492 Real theta_b = solverChoice.theta_b;
493 ncf.var("hc").put(&hc);
494 ncf.var("theta_s").put(&theta_s);
495 ncf.var("theta_b").put(&theta_b);
496
497 }
498 ncmpi_end_indep_data(ncf.ncid);
499
500 } // end if write_header
501
502 ncmpi_begin_indep_data(ncf.ncid);
503 //
504 // We compute the offsets based on location of the box within the domain
505 //
506 long long adjusted_history_count = chunk_history_file ? history_count % steps_per_history_file : history_count;
507 long long local_start_nt = (is_history ? static_cast<long long>(adjusted_history_count) : static_cast<long long>(0));
508 long long local_nt = 1; // We write data for only one time
509
510 {
511 auto nc_plot_var = ncf.var("ocean_time");
512 //nc_plot_var.par_access(NC_COLLECTIVE);
513 nc_plot_var.put(&t_new[lev], { local_start_nt }, { local_nt });
514 }
515 // do all independent writes
516 //ncmpi_end_indep_data(ncf.ncid);
517
518 cons_new[lev]->FillBoundary(geom[lev].periodicity());
519
520 mask_arrays_for_write(lev, (Real) fill_value, 0.0_rt);
521
522 // Check whether there are any nans or infs in variables that we will write out
523 if (vec_Zt_avg1[lev]->contains_nan() || vec_Zt_avg1[lev]->contains_inf()) {
524 amrex::Abort("Found while writing output: zeta contains nan or inf");
525 }
526 if (cons_new[lev]->contains_nan(Temp_comp,1) || cons_new[lev]->contains_inf(Temp_comp,1)) {
527 amrex::Abort("Found while writing output: Temperature contains nan or inf");
528 }
529 if (cons_new[lev]->contains_nan(Salt_comp,1) || cons_new[lev]->contains_inf(Salt_comp,1)) {
530 amrex::Abort("Found while writing output: Salinity contains nan or inf");
531 }
532 if (cons_new[lev]->contains_nan(Scalar_comp,1) || cons_new[lev]->contains_inf(Scalar_comp,1)) {
533 amrex::Abort("Found while writing output: Passive tracer contains nan or inf");
534 }
535 if (xvel_new[lev]->contains_nan() || xvel_new[lev]->contains_inf()) {
536 amrex::Abort("Found while writing output: velocity u contains nan or inf");
537 }
538 if (vec_ubar[lev]->contains_nan(0,1) || vec_ubar[lev]->contains_inf(0,1)) {
539 amrex::Abort("Found while writing output: velocity ubar contains nan or inf");
540 }
541 if (yvel_new[lev]->contains_nan() || yvel_new[lev]->contains_inf()) {
542 amrex::Abort("Found while writing output: velocity v contains nan or inf");
543 }
544 if (vec_vbar[lev]->contains_nan(0,1) || vec_vbar[lev]->contains_inf(0,1)) {
545 amrex::Abort("Found while writing output: velocity vbar contains nan or inf");
546 }
547
548 for (MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi) {
549 auto bx = mfi.validbox();
550 if (subdomain.contains(bx)) {
551 //
552 // We only include one grow cell at subdomain boundaries, not internal grid boundaries
553 //
554 Box tmp_bx(bx);
555 if (tmp_bx.smallEnd()[0] == subdomain.smallEnd()[0])
556 tmp_bx.growLo(0, 1);
557 if (tmp_bx.smallEnd()[1] == subdomain.smallEnd()[1])
558 tmp_bx.growLo(1, 1);
559 if (tmp_bx.bigEnd()[0] == subdomain.bigEnd()[0])
560 tmp_bx.growHi(0, 1);
561 if (tmp_bx.bigEnd()[1] == subdomain.bigEnd()[1])
562 tmp_bx.growHi(1, 1);
563 // amrex::Print() << " BX " << bx << std::endl;
564 // amrex::Print() << "TMP_BX " << tmp_bx << std::endl;
565
566 Box tmp_bx_2d(tmp_bx);
567 tmp_bx_2d.makeSlab(2, 0);
568
569 Box tmp_bx_1d(tmp_bx);
570 tmp_bx_1d.makeSlab(0, 0);
571 tmp_bx_1d.makeSlab(1, 0);
572
573 //
574 // These are the dimensions of the data we write for only this box
575 //
576 long long local_nx = tmp_bx.length()[0];
577 long long local_ny = tmp_bx.length()[1];
578 long long local_nz = tmp_bx.length()[2];
579
580 // We do the "+1" because the offset needs to start at 0
581 long long local_start_x = static_cast<long long>(tmp_bx.smallEnd()[0] + 1);
582 long long local_start_y = static_cast<long long>(tmp_bx.smallEnd()[1] + 1);
583 long long local_start_z = static_cast<long long>(tmp_bx.smallEnd()[2]);
584
585 if (write_header) {
586 // Only write out s_rho and s_w at x=0,y=0 to avoid NaNs
587 if (bx.contains(IntVect(0,0,0)))
588 {
589 {
590 FArrayBox tmp_srho;
591 tmp_srho.resize(tmp_bx_1d, 1, amrex::The_Pinned_Arena());
592
593 tmp_srho.template copy<RunOn::Device>((*vec_s_r[lev])[mfi.index()], 0, 0, 1);
594 Gpu::streamSynchronize();
595
596 auto nc_plot_var = ncf.var("s_rho");
597 //nc_plot_var.par_access(NC_INDEPENDENT);
598 nc_plot_var.put(tmp_srho.dataPtr(), { local_start_z }, { local_nz });
599 }
600 {
601 FArrayBox tmp_sw;
602 tmp_sw.resize(convert(tmp_bx_1d,IntVect(0,0,1)), 1, amrex::The_Pinned_Arena());
603
604 tmp_sw.template copy<RunOn::Device>((*vec_s_w[lev])[mfi.index()], 0, 0, 1);
605 Gpu::streamSynchronize();
606
607 auto nc_plot_var = ncf.var("s_w");
608 //nc_plot_var.par_access(NC_INDEPENDENT);
609 nc_plot_var.put(tmp_sw.dataPtr(), { local_start_z }, { local_nz + 1});
610 }
611 {
612 FArrayBox tmp_Csrho;
613 tmp_Csrho.resize(tmp_bx_1d, 1, amrex::The_Pinned_Arena());
614
615 tmp_Csrho.template copy<RunOn::Device>((*vec_Cs_r[lev])[mfi.index()], 0, 0, 1);
616 Gpu::streamSynchronize();
617
618 auto nc_plot_var = ncf.var("Cs_r");
619 //nc_plot_var.par_access(NC_INDEPENDENT);
620 nc_plot_var.put(tmp_Csrho.dataPtr(), { local_start_z }, { local_nz });
621 }
622 {
623 FArrayBox tmp_Csw;
624 tmp_Csw.resize(convert(tmp_bx_1d,IntVect(0,0,1)), 1, amrex::The_Pinned_Arena());
625
626 tmp_Csw.template copy<RunOn::Device>((*vec_Cs_w[lev])[mfi.index()], 0, 0, 1);
627 Gpu::streamSynchronize();
628
629 auto nc_plot_var = ncf.var("Cs_w");
630 //nc_plot_var.par_access(NC_INDEPENDENT);
631 nc_plot_var.put(tmp_Csw.dataPtr(), { local_start_z }, { local_nz + 1});
632 }
633 }
634
635 {
636 FArrayBox tmp_bathy;
637 tmp_bathy.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
638
639 tmp_bathy.template copy<RunOn::Device>((*vec_hOfTheConfusingName[lev])[mfi.index()], 0, 0, 1);
640 Gpu::streamSynchronize();
641
642 auto nc_plot_var = ncf.var("h");
643 //nc_plot_var.par_access(NC_INDEPENDENT);
644 nc_plot_var.put(tmp_bathy.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
645 }
646
647 {
648 FArrayBox tmp_pm;
649 tmp_pm.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
650
651 tmp_pm.template copy<RunOn::Device>((*vec_pm[lev])[mfi.index()], 0, 0, 1);
652 Gpu::streamSynchronize();
653
654 auto nc_plot_var = ncf.var("pm");
655 //nc_plot_var.par_access(NC_INDEPENDENT);
656
657 nc_plot_var.put(tmp_pm.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
658 }
659
660 {
661 FArrayBox tmp_pn;
662 tmp_pn.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
663
664 tmp_pn.template copy<RunOn::Device>((*vec_pn[lev])[mfi.index()], 0, 0, 1);
665 Gpu::streamSynchronize();
666
667 auto nc_plot_var = ncf.var("pn");
668 //nc_plot_var.par_access(NC_INDEPENDENT);
669 nc_plot_var.put(tmp_pn.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
670 }
671
672 {
673 FArrayBox tmp_f;
674 tmp_f.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
675
676 tmp_f.template copy<RunOn::Device>((*vec_fcor[lev])[mfi.index()], 0, 0, 1);
677 Gpu::streamSynchronize();
678
679 auto nc_plot_var = ncf.var("f");
680 //nc_plot_var.par_access(NC_INDEPENDENT);
681 nc_plot_var.put(tmp_f.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
682 }
683
684 {
685 FArrayBox tmp_xr;
686 tmp_xr.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
687
688 tmp_xr.template copy<RunOn::Device>((*vec_xr[lev])[mfi.index()], 0, 0, 1);
689 Gpu::streamSynchronize();
690
691 auto nc_plot_var = ncf.var("x_rho");
692 //nc_plot_var.par_access(NC_INDEPENDENT);
693
694 nc_plot_var.put(tmp_xr.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
695 }
696
697 {
698 FArrayBox tmp_yr;
699 tmp_yr.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
700
701 tmp_yr.template copy<RunOn::Device>((*vec_yr[lev])[mfi.index()], 0, 0, 1);
702 Gpu::streamSynchronize();
703
704 auto nc_plot_var = ncf.var("y_rho");
705 //nc_plot_var.par_access(NC_INDEPENDENT);
706 nc_plot_var.put(tmp_yr.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
707 }
708 }
709
710 {
711 FArrayBox tmp_zeta;
712 tmp_zeta.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
713 tmp_zeta.template copy<RunOn::Device>((*vec_Zt_avg1[lev])[mfi.index()], 0, 0, 1);
714 Gpu::streamSynchronize();
715
716 auto nc_plot_var = ncf.var("zeta");
717 nc_plot_var.put(tmp_zeta.dataPtr(), { local_start_nt, local_start_y, local_start_x }, { local_nt, local_ny,
718 local_nx });
719 }
720
721 {
722 FArrayBox tmp_temp;
723 tmp_temp.resize(tmp_bx, 1, amrex::The_Pinned_Arena());
724 tmp_temp.template copy<RunOn::Device>((*cons_new[lev])[mfi.index()], Temp_comp, 0, 1);
725 Gpu::streamSynchronize();
726
727 auto nc_plot_var = ncf.var("temp");
728 nc_plot_var.put(tmp_temp.dataPtr(), { local_start_nt, local_start_z, local_start_y, local_start_x }, { local_nt,
729 local_nz, local_ny, local_nx });
730 }
731
732 {
733 FArrayBox tmp_salt;
734 tmp_salt.resize(tmp_bx, 1, amrex::The_Pinned_Arena());
735 tmp_salt.template copy<RunOn::Device>((*cons_new[lev])[mfi.index()], Salt_comp, 0, 1);
736 Gpu::streamSynchronize();
737
738 auto nc_plot_var = ncf.var("salt");
739 nc_plot_var.put(tmp_salt.dataPtr(), { local_start_nt, local_start_z, local_start_y, local_start_x }, { local_nt,
740 local_nz, local_ny, local_nx });
741 }
742
743 {
744 FArrayBox tmp_tracer;
745 tmp_tracer.resize(tmp_bx, 1, amrex::The_Pinned_Arena());
746 tmp_tracer.template copy<RunOn::Device>((*cons_new[lev])[mfi.index()], Scalar_comp, 0, 1);
747 Gpu::streamSynchronize();
748
749 auto nc_plot_var = ncf.var("tracer");
750 nc_plot_var.put(tmp_tracer.dataPtr(), { local_start_nt, local_start_z, local_start_y, local_start_x }, { local_nt,
751 local_nz, local_ny, local_nx });
752 }
753 } // subdomain
754 } // mfi
755
756 //ncf.wait_all(irq, &requests[0]);
757 //requests.resize(0);
758 //irq = 0;
759 // Writing u (we loop over cons to get cell-centered box)
760 for (MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi) {
761 Box bx = mfi.validbox();
762
763 if (subdomain.contains(bx)) {
764 //
765 // We only include one grow cell at subdomain boundaries, not internal grid boundaries
766 //
767 Box tmp_bx(bx);
768 tmp_bx.surroundingNodes(0);
769 if (tmp_bx.smallEnd()[1] == subdomain.smallEnd()[1])
770 tmp_bx.growLo(1, 1);
771 if (tmp_bx.bigEnd()[1] == subdomain.bigEnd()[1])
772 tmp_bx.growHi(1, 1);
773 Box tmp_bx_2d(tmp_bx);
774 tmp_bx_2d.makeSlab(2, 0);
775
776 //
777 // These are the dimensions of the data we write for only this box
778 //
779 long long local_nx = tmp_bx.length()[0];
780 long long local_ny = tmp_bx.length()[1];
781 long long local_nz = tmp_bx.length()[2];
782
783 // We do the "+1" because the offset needs to start at 0
784 long long local_start_x = static_cast<long long>(tmp_bx.smallEnd()[0]);
785 long long local_start_y = static_cast<long long>(tmp_bx.smallEnd()[1] + 1);
786 long long local_start_z = static_cast<long long>(tmp_bx.smallEnd()[2]);
787
788 if (write_header) {
789 {
790 FArrayBox tmp;
791 tmp.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
792 tmp.template copy<RunOn::Device>((*vec_xu[lev])[mfi.index()], 0, 0, 1);
793 Gpu::streamSynchronize();
794
795 auto nc_plot_var = ncf.var("x_u");
796 //nc_plot_var.par_access(NC_INDEPENDENT);
797 nc_plot_var.put(tmp.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
798 }
799 {
800 FArrayBox tmp;
801 tmp.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
802 tmp.template copy<RunOn::Device>((*vec_yu[lev])[mfi.index()], 0, 0, 1);
803 Gpu::streamSynchronize();
804
805 auto nc_plot_var = ncf.var("y_u");
806 //nc_plot_var.par_access(NC_INDEPENDENT);
807 nc_plot_var.put(tmp.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
808 }
809 }
810
811 {
812 FArrayBox tmp;
813 tmp.resize(tmp_bx, 1, amrex::The_Pinned_Arena());
814 tmp.template copy<RunOn::Device>((*xvel_new[lev])[mfi.index()], 0, 0, 1);
815 Gpu::streamSynchronize();
816
817 auto nc_plot_var = ncf.var("u");
818 nc_plot_var.put(tmp.dataPtr(), { local_start_nt, local_start_z, local_start_y, local_start_x }, { local_nt,
819 local_nz, local_ny, local_nx });
820 }
821
822 {
823 FArrayBox tmp;
824 tmp.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
825 tmp.template copy<RunOn::Device>((*vec_ubar[lev])[mfi.index()], 0, 0, 1);
826 Gpu::streamSynchronize();
827
828 auto nc_plot_var = ncf.var("ubar");
829 nc_plot_var.put(tmp.dataPtr(), { local_start_nt, local_start_y, local_start_x }, { local_nt, local_ny, local_nx });
830 }
831 {
832 FArrayBox tmp;
833 tmp.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
834 tmp.template copy<RunOn::Device>((*vec_sustr[lev])[mfi.index()], 0, 0, 1);
835 Gpu::streamSynchronize();
836
837 auto nc_plot_var = ncf.var("sustr");
838 nc_plot_var.put(tmp.dataPtr(), { local_start_nt, local_start_y, local_start_x }, { local_nt, local_ny, local_nx });
839 }
840 } // in subdomain
841 } // mfi
842
843 // Writing v (we loop over cons to get cell-centered box)
844 for (MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi) {
845 Box bx = mfi.validbox();
846
847 if (subdomain.contains(bx)) {
848 //
849 // We only include one grow cell at subdomain boundaries, not internal grid boundaries
850 //
851 Box tmp_bx(bx);
852 tmp_bx.surroundingNodes(1);
853 if (tmp_bx.smallEnd()[0] == subdomain.smallEnd()[0])
854 tmp_bx.growLo(0, 1);
855 if (tmp_bx.bigEnd()[0] == subdomain.bigEnd()[0])
856 tmp_bx.growHi(0, 1);
857 // amrex::Print() << " BX " << bx << std::endl;
858 // amrex::Print() << "TMP_BX " << tmp_bx << std::endl;
859
860 Box tmp_bx_2d(tmp_bx);
861 tmp_bx_2d.makeSlab(2, 0);
862
863 //
864 // These are the dimensions of the data we write for only this box
865 //
866 long long local_nx = tmp_bx.length()[0];
867 long long local_ny = tmp_bx.length()[1];
868 long long local_nz = tmp_bx.length()[2];
869
870 // We do the "+1" because the offset needs to start at 0
871 long long local_start_x = static_cast<long long>(tmp_bx.smallEnd()[0] + 1);
872 long long local_start_y = static_cast<long long>(tmp_bx.smallEnd()[1]);
873 long long local_start_z = static_cast<long long>(tmp_bx.smallEnd()[2]);
874
875 if (write_header) {
876 {
877 FArrayBox tmp;
878 tmp.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
879 tmp.template copy<RunOn::Device>((*vec_xv[lev])[mfi.index()], 0, 0, 1);
880 Gpu::streamSynchronize();
881
882 auto nc_plot_var = ncf.var("x_v");
883 //nc_plot_var.par_access(NC_INDEPENDENT);
884 nc_plot_var.put(tmp.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
885 }
886 {
887 FArrayBox tmp;
888 tmp.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
889 tmp.template copy<RunOn::Device>((*vec_yv[lev])[mfi.index()], 0, 0, 1);
890 Gpu::streamSynchronize();
891
892 auto nc_plot_var = ncf.var("y_v");
893 //nc_plot_var.par_access(NC_INDEPENDENT);
894 nc_plot_var.put(tmp.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
895 }
896
897 }
898 {
899 FArrayBox tmp;
900 tmp.resize(tmp_bx, 1, amrex::The_Pinned_Arena());
901 tmp.template copy<RunOn::Device>((*yvel_new[lev])[mfi.index()], 0, 0, 1);
902 Gpu::streamSynchronize();
903
904 auto nc_plot_var = ncf.var("v");
905 nc_plot_var.put(tmp.dataPtr(), { local_start_nt, local_start_z, local_start_y, local_start_x }, { local_nt,
906 local_nz, local_ny, local_nx });
907 }
908
909 {
910 FArrayBox tmp;
911 tmp.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
912 tmp.template copy<RunOn::Device>((*vec_vbar[lev])[mfi.index()], 0, 0, 1);
913 Gpu::streamSynchronize();
914
915 auto nc_plot_var = ncf.var("vbar");
916 nc_plot_var.put(tmp.dataPtr(), { local_start_nt, local_start_y, local_start_x }, { local_nt, local_ny, local_nx });
917 }
918 {
919 FArrayBox tmp;
920 tmp.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
921 tmp.template copy<RunOn::Device>((*vec_svstr[lev])[mfi.index()], 0, 0, 1);
922 Gpu::streamSynchronize();
923
924 auto nc_plot_var = ncf.var("svstr");
925 nc_plot_var.put(tmp.dataPtr(), { local_start_nt, local_start_y, local_start_x }, { local_nt, local_ny, local_nx });
926 }
927
928 } // in subdomain
929 } // mfi
930
931 for (MFIter mfi(*cons_new[lev], false); mfi.isValid(); ++mfi) {
932 Box bx = mfi.validbox();
933
934 if (subdomain.contains(bx)) {
935 //
936 // We only include one grow cell at subdomain boundaries, not internal grid boundaries
937 //
938 Box tmp_bx(bx);
939 tmp_bx.surroundingNodes(0);
940 tmp_bx.surroundingNodes(1);
941
942 Box tmp_bx_2d(tmp_bx);
943 tmp_bx_2d.makeSlab(2, 0);
944
945 //
946 // These are the dimensions of the data we write for only this box
947 //
948 long long local_nx = tmp_bx.length()[0];
949 long long local_ny = tmp_bx.length()[1];
950
951 // We do the "+1" because the offset needs to start at 0
952 long long local_start_x = static_cast<long long>(tmp_bx.smallEnd()[0]);
953 long long local_start_y = static_cast<long long>(tmp_bx.smallEnd()[1]);
954
955 if (write_header) {
956 {
957 FArrayBox tmp;
958 tmp.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
959 tmp.template copy<RunOn::Device>((*vec_xp[lev])[mfi.index()], 0, 0, 1);
960 Gpu::streamSynchronize();
961
962 auto nc_plot_var = ncf.var("x_psi");
963 //nc_plot_var.par_access(NC_INDEPENDENT);
964 nc_plot_var.put(tmp.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
965 }
966 {
967 FArrayBox tmp;
968 tmp.resize(tmp_bx_2d, 1, amrex::The_Pinned_Arena());
969 tmp.template copy<RunOn::Device>((*vec_yp[lev])[mfi.index()], 0, 0, 1);
970 Gpu::streamSynchronize();
971
972 auto nc_plot_var = ncf.var("y_psi");
973 //nc_plot_var.par_access(NC_INDEPENDENT);
974 nc_plot_var.put(tmp.dataPtr(), { local_start_y, local_start_x }, { local_ny, local_nx });
975 }
976
977 } // header
978 } // in subdomain
979 } // mfi
980
981 mask_arrays_for_write(lev, 0.0_rt, (Real) fill_value);
982
983 ncf.close();
984
985 REMORA::total_nc_plot_file_step += 1;
986}
987
REMORA_IndexDefines.H
Temp_comp
#define Temp_comp
Definition REMORA_IndexDefines.H:8
Scalar_comp
#define Scalar_comp
Definition REMORA_IndexDefines.H:10
Salt_comp
#define Salt_comp
Definition REMORA_IndexDefines.H:9
REMORA_NCInterface.H
REMORA_NCPlotFile.H
REMORA::vec_Cs_r
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_Cs_r
Stretching coefficients at rho points.
Definition REMORA.H:274
REMORA::vec_fcor
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_fcor
coriolis factor (2D)
Definition REMORA.H:363
REMORA::vec_hOfTheConfusingName
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_hOfTheConfusingName
Bathymetry data (2D, positive valued, h in ROMS)
Definition REMORA.H:232
REMORA::vec_pm
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_pm
horizontal scaling factor: 1 / dx (2D)
Definition REMORA.H:358
REMORA::vec_yv
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_yv
y_grid on v-points (2D)
Definition REMORA.H:378
REMORA::cons_new
amrex::Vector< amrex::MultiFab * > cons_new
multilevel data container for current step's scalar data: temperature, salinity, passive scalar
Definition REMORA.H:220
REMORA::write_history_file
static bool write_history_file
Whether to output NetCDF files as a single history file with several time steps.
Definition REMORA.H:1020
REMORA::history_count
int history_count
Counter for which time index we are writing to in the netcdf history file.
Definition REMORA.H:1226
REMORA::chunk_history_file
bool chunk_history_file
Whether to chunk netcdf history file.
Definition REMORA.H:1219
REMORA::vec_sustr
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_sustr
Surface stress in the u direction.
Definition REMORA.H:285
REMORA::vec_yp
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_yp
y_grid on psi-points (2D)
Definition REMORA.H:383
REMORA::vec_xr
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_xr
x_grid on rho points (2D)
Definition REMORA.H:366
REMORA::vec_xv
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_xv
x_grid on v-points (2D)
Definition REMORA.H:376
REMORA::boxes_at_level
amrex::Vector< amrex::Vector< amrex::Box > > boxes_at_level
the boxes specified at each level by tagging criteria
Definition REMORA.H:1107
REMORA::WriteNCPlotFile
void WriteNCPlotFile(int istep)
Write plotfile using NetCDF (wrapper)
Definition REMORA_NCPlotFile.cpp:24
REMORA::WriteNCPlotFile_which
void WriteNCPlotFile_which(int lev, int which_subdomain, bool write_header, ncutils::NCFile &ncf, bool is_history)
Write a particular NetCDF plotfile.
Definition REMORA_NCPlotFile.cpp:109
REMORA::yvel_new
amrex::Vector< amrex::MultiFab * > yvel_new
multilevel data container for current step's y velocities (v in ROMS)
Definition REMORA.H:224
REMORA::start_bdy_time
amrex::Real start_bdy_time
Start time in the time series of boundary data.
Definition REMORA.H:1015
REMORA::steps_per_history_file
int steps_per_history_file
Number of time steps per netcdf history file.
Definition REMORA.H:1224
REMORA::max_step
int max_step
maximum number of steps
Definition REMORA.H:1174
REMORA::xvel_new
amrex::Vector< amrex::MultiFab * > xvel_new
multilevel data container for current step's x velocities (u in ROMS)
Definition REMORA.H:222
REMORA::vec_s_w
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_s_w
Scaled vertical coordinate (range [0,1]) that transforms to z, defined at w-points (cell faces)
Definition REMORA.H:271
REMORA::plot_int
int plot_int
Plotfile output interval in iterations.
Definition REMORA.H:1208
REMORA::mask_arrays_for_write
void mask_arrays_for_write(int lev, amrex::Real fill_value, amrex::Real fill_where)
Mask data arrays before writing output.
Definition REMORA_Plotfile.cpp:670
REMORA::file_min_digits
static int file_min_digits
Minimum number of digits in plotfile name or chunked history file.
Definition REMORA.H:1272
REMORA::vec_svstr
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_svstr
Surface stress in the v direction.
Definition REMORA.H:287
REMORA::vec_Cs_w
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_Cs_w
Stretching coefficients at w points.
Definition REMORA.H:276
REMORA::t_new
amrex::Vector< amrex::Real > t_new
new time at each level
Definition REMORA.H:1114
REMORA::solverChoice
static SolverChoice solverChoice
Container for algorithmic choices.
Definition REMORA.H:1237
REMORA::total_nc_plot_file_step
static int total_nc_plot_file_step
Definition REMORA.H:926
REMORA::vec_xp
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_xp
x_grid on psi-points (2D)
Definition REMORA.H:381
REMORA::vec_vbar
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_vbar
barotropic y velocity (2D)
Definition REMORA.H:344
REMORA::vec_ubar
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_ubar
barotropic x velocity (2D)
Definition REMORA.H:342
REMORA::vec_yu
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_yu
y_grid on u-points (2D)
Definition REMORA.H:373
REMORA::vec_xu
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_xu
x_grid on u-points (2D)
Definition REMORA.H:371
REMORA::vec_pn
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_pn
horizontal scaling factor: 1 / dy (2D)
Definition REMORA.H:360
REMORA::plot_file_name
std::string plot_file_name
Plotfile prefix.
Definition REMORA.H:1206
REMORA::vec_Zt_avg1
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_Zt_avg1
Average of the free surface, zeta (2D)
Definition REMORA.H:282
REMORA::vec_yr
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_yr
y_grid on rho points (2D)
Definition REMORA.H:368
REMORA::vec_s_r
amrex::Vector< std::unique_ptr< amrex::MultiFab > > vec_s_r
Scaled vertical coordinate (range [0,1]) that transforms to z, defined at rho points (cell centers)
Definition REMORA.H:269
ncutils::NCFile::put_attr
void put_attr(const std::string &name, const std::string &value) const
Set file attribute to value.
Definition REMORA_NCInterface.cpp:355
ncutils::NCFile::enter_def_mode
void enter_def_mode() const
Enter definition mode (not needed for NetCDF4 format)
Definition REMORA_NCInterface.cpp:421
ncutils::NCFile::def_var
NCVar def_var(const std::string &name, const nc_type dtype, const std::vector< std::string > &dnames) const
Define a variable (wrapper for def_array)
Definition REMORA_NCInterface.H:334
ncutils::NCFile::def_dim
NCDim def_dim(const std::string &, const size_t len) const
Define new dimension.
Definition REMORA_NCInterface.cpp:275
ncutils::NCFile::create
static NCFile create(const std::string &name, const int cmode=NC_CLOBBER|NC_MPIIO, MPI_Comm comm=MPI_COMM_WORLD, MPI_Info info=MPI_INFO_NULL)
Create a file.
Definition REMORA_NCInterface.cpp:436
ncutils::NCFile::exit_def_mode
void exit_def_mode() const
Exit definition mode.
Definition REMORA_NCInterface.cpp:432
ncutils::NCFile::open
static NCFile open(const std::string &name, const int cmode=NC_NOWRITE, MPI_Comm comm=MPI_COMM_WORLD, MPI_Info info=MPI_INFO_NULL)
Open an existing file.
Definition REMORA_NCInterface.cpp:442
ncutils::NCFile::close
void close()
Close file object.
Definition REMORA_NCInterface.cpp:458
ncutils::NCFile::var
NCVar var(const std::string &) const
Get the variable instance by name.
Definition REMORA_NCInterface.cpp:313
ncutils::NCFile::def_var_fill
NCVar def_var_fill(const std::string &name, const nc_type dtype, const std::vector< std::string > &dnames, const void *fill_val) const
Define a variable (wrapper for def_array)
Definition REMORA_NCInterface.H:343
SolverChoice::theta_b
amrex::Real theta_b
Definition REMORA_DataStruct.H:608
SolverChoice::theta_s
amrex::Real theta_s
Definition REMORA_DataStruct.H:607
SolverChoice::tcline
amrex::Real tcline
Definition REMORA_DataStruct.H:609
ncutils::NCDType::Real
static constexpr nc_type Real
Definition REMORA_NCInterface.H:29
ncutils::NCVar::put
void put(const double *ptr) const
Write out the entire double variable.
Definition REMORA_NCInterface.cpp:59
ncutils::NCVar::put_attr
void put_attr(const std::string &name, const std::string &value) const
Set attribute "name" to "value".
Definition REMORA_NCInterface.cpp:223