37 #include "fft/turbulence.hpp"
39 #include "gravity/fft_gravity.hpp"
40 #include "gravity/mg_gravity.hpp"
42 #include "outputs/io_wrapper.hpp"
46 #include "particles/material_point.hpp"
52 MPI_Datatype MPI_PARTICLE;
55 #ifdef OPENMP_PARALLEL
63 int main(
int argc,
char *argv[]) {
64 std::string athena_version =
"version 19.0 - August 2019";
65 char *input_filename =
nullptr, *restart_filename =
nullptr;
66 char *prundir =
nullptr;
72 std::uint64_t mbcnt = 0;
79 #ifdef OPENMP_PARALLEL
81 if (MPI_SUCCESS != MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpiprv)) {
82 std::cout <<
"### FATAL ERROR in main" << std::endl
83 <<
"MPI Initialization failed." << std::endl;
86 if (mpiprv != MPI_THREAD_MULTIPLE) {
87 std::cout <<
"### FATAL ERROR in main" << std::endl
88 <<
"MPI_THREAD_MULTIPLE must be supported for the hybrid parallelzation. "
89 << MPI_THREAD_MULTIPLE <<
" : " << mpiprv
95 if (MPI_SUCCESS != MPI_Init(&argc, &argv)) {
96 std::cout <<
"### FATAL ERROR in main" << std::endl
97 <<
"MPI Initialization failed." << std::endl;
103 std::cout <<
"### FATAL ERROR in main" << std::endl
104 <<
"MPI_Comm_rank failed." << std::endl;
111 std::cout <<
"### FATAL ERROR in main" << std::endl
112 <<
"MPI_Comm_size failed." << std::endl;
118 MPI_Aint* tag_ub_ptr;
120 MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_TAG_UB, &tag_ub_ptr, &att_flag);
124 int counts[3] = {1, 2+NINT_PARTICLE_DATA, 8+NREAL_PARTICLE_DATA};
125 MPI_Datatype types[3] = {MPI_AINT, MPI_INT, MPI_ATHENA_REAL};
126 MPI_Aint disps[3] = {offsetof(MaterialPoint,
next), offsetof(MaterialPoint,
id),
127 offsetof(MaterialPoint, time)};
129 MPI_Type_create_struct(3, counts, disps, types, &MPI_PARTICLE);
130 MPI_Type_commit(&MPI_PARTICLE);
140 for (
int i=1; i<argc; i++) {
142 if (*argv[i] ==
'-' && *(argv[i]+1) !=
'\0' && *(argv[i]+2) ==
'\0') {
144 char opt_letter = *(argv[i]+1);
154 || (*argv[i+1] ==
'-') ) {
156 std::cout <<
"### FATAL ERROR in main" << std::endl
157 <<
"-" << opt_letter <<
" must be followed by a valid argument\n";
165 switch(*(argv[i]+1)) {
167 input_filename = argv[++i];
172 restart_filename = argv[++i];
181 mesh_flag =
static_cast<int>(std::strtol(argv[++i],
nullptr, 10));
185 std::sscanf(argv[++i],
"%d:%d:%d", &wth, &wtm, &wts);
186 wtlim = wth*3600 + wtm*60 + wts;
198 std::cout <<
"Athena++ " << athena_version << std::endl;
199 std::cout <<
"Usage: " << argv[0] <<
" [options] [block/par=value ...]\n";
200 std::cout <<
"Options:" << std::endl;
201 std::cout <<
" -i <file> specify input file [athinput]\n";
202 std::cout <<
" -r <file> restart with this file\n";
203 std::cout <<
" -d <directory> specify run dir [current dir]\n";
204 std::cout <<
" -n parse input file and quit\n";
205 std::cout <<
" -c show configuration and quit\n";
206 std::cout <<
" -m <nproc> output mesh structure and quit\n";
207 std::cout <<
" -t hh:mm:ss wall time limit for final output\n";
208 std::cout <<
" -h this help\n";
220 if (restart_filename ==
nullptr && input_filename ==
nullptr) {
222 std::cout <<
"### FATAL ERROR in main" << std::endl
223 <<
"No input file or restart file is specified." << std::endl;
241 IOWrapper infile, restartfile;
242 #ifdef ENABLE_EXCEPTIONS
247 restartfile.Open(restart_filename, IOWrapper::FileMode::read);
254 if (iarg_flag == 1) {
256 infile.Open(input_filename, IOWrapper::FileMode::read);
261 #ifdef ENABLE_EXCEPTIONS
263 catch(std::bad_alloc& ba) {
264 std::cout <<
"### FATAL ERROR in main" << std::endl
265 <<
"memory allocation failed initializing class ParameterInput: "
266 << ba.what() << std::endl;
267 if (res_flag == 1) restartfile.Close();
273 catch(std::exception
const& ex) {
274 std::cout << ex.what() << std::endl;
275 if (res_flag == 1) restartfile.Close();
287 #ifdef ENABLE_EXCEPTIONS
291 pmesh =
new Mesh(pinput, mesh_flag);
293 pmesh =
new Mesh(pinput, restartfile, mesh_flag);
295 #ifdef ENABLE_EXCEPTIONS
297 catch(std::bad_alloc& ba) {
298 std::cout <<
"### FATAL ERROR in main" << std::endl
299 <<
"memory allocation failed initializing class Mesh: "
300 << ba.what() << std::endl;
301 if (res_flag == 1) restartfile.Close();
307 catch(std::exception
const& ex) {
308 std::cout << ex.what() << std::endl;
309 if (res_flag == 1) restartfile.Close();
318 if (iarg_flag == 1 && res_flag == 1) {
326 if (res_flag == 1) restartfile.Close();
333 if (res_flag == 1) restartfile.Close();
348 #ifdef ENABLE_EXCEPTIONS
351 ptlist =
new MAIN_TASKLIST(pinput, pmesh);
353 #ifdef ENABLE_EXCEPTIONS
355 catch(std::bad_alloc& ba) {
356 std::cout <<
"### FATAL ERROR in main" << std::endl <<
"memory allocation failed "
357 <<
"in creating task list " << ba.what() << std::endl;
368 #ifdef ENABLE_EXCEPTIONS
372 #ifdef ENABLE_EXCEPTIONS
374 catch(std::bad_alloc& ba) {
375 std::cout <<
"### FATAL ERROR in main" << std::endl <<
"memory allocation failed "
376 <<
"in creating task list " << ba.what() << std::endl;
388 #ifdef ENABLE_EXCEPTIONS
392 #ifdef ENABLE_EXCEPTIONS
394 catch(std::bad_alloc& ba) {
395 std::cout <<
"### FATAL ERROR in main" << std::endl <<
"memory allocation failed "
396 <<
"in problem generator " << ba.what() << std::endl;
402 catch(std::exception
const& ex) {
403 std::cout << ex.what() << std::endl;
415 #ifdef ENABLE_EXCEPTIONS
419 pouts =
new Outputs(pmesh, pinput);
420 if (res_flag == 0) pouts->
MakeOutputs(pmesh, pinput);
421 #ifdef ENABLE_EXCEPTIONS
423 catch(std::bad_alloc& ba) {
424 std::cout <<
"### FATAL ERROR in main" << std::endl
425 <<
"memory allocation failed setting initial conditions: "
426 << ba.what() << std::endl;
432 catch(std::exception
const& ex) {
433 std::cout << ex.what() << std::endl;
445 std::cout <<
"\nSetup complete, entering main loop...\n" << std::endl;
448 clock_t tstart = clock();
449 #ifdef OPENMP_PARALLEL
450 double omp_start_time = omp_get_wtime();
453 while ((pmesh->
time < pmesh->
tlim) &&
463 static_cast<int>(0.5*(-1. + std::sqrt(1. + 8.*my_dt/dt_parabolic))) + 1;
466 for (
int stage=1; stage<=pststlist->
nstages; ++stage)
472 for (
int stage=1; stage<=ptlist->
nstages; ++stage) {
473 if (SELF_GRAVITY_ENABLED == 1)
474 pmesh->
pfgrd->Solve(stage, 0);
475 else if (SELF_GRAVITY_ENABLED == 2)
476 pmesh->
pmgrd->Solve(stage);
490 #ifdef ENABLE_EXCEPTIONS
495 #ifdef ENABLE_EXCEPTIONS
497 catch(std::bad_alloc& ba) {
498 std::cout <<
"### FATAL ERROR in main" << std::endl
499 <<
"memory allocation failed during output: " << ba.what() <<std::endl;
505 catch(std::exception
const& ex) {
506 std::cout << ex.what() << std::endl;
526 #ifdef ENABLE_EXCEPTIONS
530 #ifdef ENABLE_EXCEPTIONS
532 catch(std::bad_alloc& ba) {
533 std::cout <<
"### FATAL ERROR in main" << std::endl
534 <<
"memory allocation failed during output: " << ba.what() <<std::endl;
540 catch(std::exception
const& ex) {
541 std::cout << ex.what() << std::endl;
556 std::cout << std::endl <<
"Terminating on Terminate signal" << std::endl;
558 std::cout << std::endl <<
"Terminating on Interrupt signal" << std::endl;
560 std::cout << std::endl <<
"Terminating on wall-time limit" << std::endl;
562 std::cout << std::endl <<
"Terminating on cycle limit" << std::endl;
564 std::cout << std::endl <<
"Terminating on time limit" << std::endl;
567 std::cout <<
"time=" << pmesh->
time <<
" cycle=" << pmesh->
ncycle << std::endl;
568 std::cout <<
"tlim=" << pmesh->
tlim <<
" nlim=" << pmesh->
nlim << std::endl;
571 std::cout << std::endl <<
"Number of MeshBlocks = " << pmesh->
nbtotal
572 <<
"; " << pmesh->
nbnew <<
" created, " << pmesh->
nbdel
573 <<
" destroyed during this simulation." << std::endl;
577 #ifdef OPENMP_PARALLEL
578 double omp_time = omp_get_wtime() - omp_start_time;
580 clock_t tstop = clock();
581 double cpu_time = (tstop>tstart ?
static_cast<double> (tstop-tstart) :
582 1.0)/
static_cast<double> (CLOCKS_PER_SEC);
583 std::uint64_t zonecycles =
585 double zc_cpus =
static_cast<double> (zonecycles) / cpu_time;
587 std::cout << std::endl <<
"zone-cycles = " << zonecycles << std::endl;
588 std::cout <<
"cpu time used = " << cpu_time << std::endl;
589 std::cout <<
"zone-cycles/cpu_second = " << zc_cpus << std::endl;
590 #ifdef OPENMP_PARALLEL
591 double zc_omps =
static_cast<double> (zonecycles) / omp_time;
592 std::cout << std::endl <<
"omp wtime used = " << omp_time << std::endl;
593 std::cout <<
"zone-cycles/omp_wsecond = " << zc_omps << std::endl;
603 MPI_Type_free(&MPI_PARTICLE);
int GetNumberOfMeshBlockCells()
void OutputCycleDiagnostics()
void Initialize(int res_flag, ParameterInput *pin)
void LoadBalancingAndAdaptiveMeshRefinement(ParameterInput *pin)
void UserWorkAfterLoop(ParameterInput *pin)
void MakeOutputs(Mesh *pm, ParameterInput *pin, bool wtflag=false)
void DoTaskListOneStage(Mesh *pmesh, int stage)
int main(int argc, char *argv[])
void SetWallTimeAlarm(int t)
void CancelWallTimeAlarm()
void ChangeRunDir(const char *pdir)