From 6a460dfc5e01b1e52dc9b198c770681edd8fa2fb Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 3 Sep 2025 15:31:59 -0600 Subject: [PATCH 1/6] Allow for user overridable BCs --- src/bvals/comms/boundary_communication.cpp | 86 +++++++++++----------- src/bvals/comms/bvals_in_one.hpp | 49 ++++++++++-- src/mesh/mesh.cpp | 68 ++--------------- 3 files changed, 91 insertions(+), 112 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 78121cd3fac9b..8b8c6391d12a3 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -401,55 +401,51 @@ ProlongateBounds(std::shared_ptr> &); template TaskStatus ProlongateBounds(std::shared_ptr> &); -// Adds all relevant boundary communication to a single task list -template -TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl, - std::shared_ptr> &md, bool multilevel) { - // TODO(LFR): Splitting up the boundary tasks while doing prolongation can cause some - // possible issues for sparse fields. In particular, the order in which - // fields are allocated and then set could potentially result in different - // results if the default sparse value is non-zero. - // const auto any = BoundaryType::any; - static_assert(bounds == BoundaryType::any || bounds == BoundaryType::gmg_same); - // const auto local = BoundaryType::local; - // const auto nonlocal = BoundaryType::nonlocal; - - // auto send = tl.AddTask(dependency, SendBoundBufs, md); - // auto send_local = tl.AddTask(dependency, SendBoundBufs, md); - - // auto recv_local = tl.AddTask(dependency, ReceiveBoundBufs, md); - // auto set_local = tl.AddTask(recv_local, SetBounds, md); - - // auto recv = tl.AddTask(dependency, ReceiveBoundBufs, md); - // auto set = tl.AddTask(recv, SetBounds, md); - - // auto cbound = tl.AddTask(set, ApplyCoarseBoundaryConditions, md); - - // auto pro_local = tl.AddTask(cbound | set_local | set, ProlongateBounds, md); - // auto pro = tl.AddTask(cbound | set_local | set, ProlongateBounds, md); - - // auto out = (pro_local | pro); - - auto send = tl.AddTask(dependency, TF(SendBoundBufs), md); - auto recv = tl.AddTask(dependency, TF(ReceiveBoundBufs), md); - auto set = tl.AddTask(recv, TF(SetBounds), md); - - auto pro = set; - if (md->GetMeshPointer()->multilevel) { - auto cbound = tl.AddTask(set, TF(ApplyBoundaryConditionsOnCoarseOrFineMD), md, true); - pro = tl.AddTask(cbound, TF(ProlongateBounds), md); +template +TaskStatus ProlongateInternalBounds(std::shared_ptr> &md) { + PARTHENON_INSTRUMENT + + Mesh *pmesh = md->GetMeshPointer(); + auto &cache = md->GetBvarsCache().GetSubCache(bound_type, false); + + auto [rebuild, nbound] = CheckReceiveBufferCacheForRebuild(md); + + if (rebuild) { + if constexpr (bound_type == BoundaryType::gmg_prolongate_recv) { + RebuildBufferCache(md, nbound, BndInfo::GetSetBndInfo, + ProResInfo::GetInteriorProlongate); + } else if constexpr (bound_type == BoundaryType::gmg_restrict_recv) { + RebuildBufferCache(md, nbound, BndInfo::GetSetBndInfo, + ProResInfo::GetNull); + } else { + RebuildBufferCache(md, nbound, BndInfo::GetSetBndInfo, + ProResInfo::GetSet); + } } - auto fbound = tl.AddTask(pro, TF(ApplyBoundaryConditionsOnCoarseOrFineMD), md, false); - return fbound; + if (nbound > 0 && pmesh->multilevel && md->NumBlocks() > 0) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + StateDescriptor *resolved_packages = pmb->resolved_packages.get(); + + // Prolongate from coarse buffer + refinement::ProlongateInternal(resolved_packages, cache.prores_cache, pmb->cellbounds, + pmb->c_cellbounds); + } + return TaskStatus::complete; } -template TaskID -AddBoundaryExchangeTasks(TaskID, TaskList &, - std::shared_ptr> &, bool); +template TaskStatus +ProlongateInternalBounds(std::shared_ptr> &); +template TaskStatus +ProlongateInternalBounds(std::shared_ptr> &); +template TaskStatus +ProlongateInternalBounds(std::shared_ptr> &); +template TaskStatus +ProlongateInternalBounds(std::shared_ptr> &); -template TaskID -AddBoundaryExchangeTasks(TaskID, TaskList &, - std::shared_ptr> &, bool); + +bool IsMeshMultilevel(std::shared_ptr> &md) { + return md->GetMeshPointer()->multilevel; +} TaskID AddFluxCorrectionTasks(TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel) { diff --git a/src/bvals/comms/bvals_in_one.hpp b/src/bvals/comms/bvals_in_one.hpp index 0180bd6a83058..573baa711e915 100644 --- a/src/bvals/comms/bvals_in_one.hpp +++ b/src/bvals/comms/bvals_in_one.hpp @@ -23,6 +23,7 @@ #include #include "basic_types.hpp" +#include "bvals/boundary_conditions.hpp" #include "bvals/neighbor_block.hpp" #include "coordinates/coordinates.hpp" @@ -62,6 +63,8 @@ inline TaskStatus SetBoundaries(std::shared_ptr> &md) { template TaskStatus ProlongateBounds(std::shared_ptr> &md); +template +TaskStatus ProlongateInternalBounds(std::shared_ptr> &md); inline TaskStatus ProlongateBoundaries(std::shared_ptr> &md) { return ProlongateBounds(md); } @@ -79,15 +82,51 @@ static TaskStatus SetFluxCorrections(std::shared_ptr> &md) { return SetBounds(md); } -// Adds all relevant boundary communication to a single task list -template -TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl, - std::shared_ptr> &md, bool multilevel); - // Adds all relevant flux correction tasks to a single task list TaskID AddFluxCorrectionTasks(TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel); +using BValOnMDFunc_t = std::function> &, bool)>; +using BValOnMDTasks_t = std::function>, bool)>; +bool IsMeshMultilevel(std::shared_ptr> &md); + +// Adds all relevant boundary communication to a single task list +template +TaskID AddBoundaryExchangeTasks( + TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel, + BValOnMDTasks_t ApplyBCs) { + static_assert(bounds == BoundaryType::any || bounds == BoundaryType::gmg_same); + + auto send = tl.AddTask(dependency, TF(SendBoundBufs), md); + auto recv = tl.AddTask(dependency, TF(ReceiveBoundBufs), md); + auto set = tl.AddTask(recv, TF(SetBounds), md); + + auto pro = set; + if (IsMeshMultilevel(md)) { + auto cbound = ApplyBCs(set, &tl, md, true); + pro = tl.AddTask(cbound, TF(ProlongateBounds), md); + } + auto fbound = ApplyBCs(pro, &tl, md, false); + if (IsMeshMultilevel(md)) { + // Need to prolongate internal bounds after setting physical boundary + // conditions, since the internal prolongation uses the values of the + // normal buffer on shared elements (rather than values in the coarse) + // buffer for prolongation. + fbound = tl.AddTask(fbound, TF(ProlongateInternalBounds), md); + } + return fbound; +} + +template +TaskID AddBoundaryExchangeTasks( + TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel, + BValOnMDFunc_t ApplyBCs = ApplyBoundaryConditionsOnCoarseOrFineMD) { + return AddBoundaryExchangeTasks(dependency, tl, md, multilevel, + [&](TaskID id, TaskList *tl, std::shared_ptr> md, bool coarse){ + return tl->AddTask(id, TF(ApplyBCs), md, coarse); + }); +} + // These tasks should not be called in down stream code TaskStatus BuildBoundaryBuffers(std::shared_ptr> &md); TaskStatus BuildGMGBoundaryBuffers(std::shared_ptr> &md); diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index cd9bf146d0943..d3a5c5e637140 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -652,70 +652,14 @@ void Mesh::BuildTagMapAndBoundaryBuffers() { void Mesh::CommunicateBoundaries(std::string md_name, const std::vector &fields) { - const int num_partitions = DefaultNumPartitions(); - const int nmb = GetNumMeshBlocksThisRank(Globals::my_rank); - constexpr std::int64_t max_it = 1e10; - std::vector sent(num_partitions, false); - bool all_sent; - std::int64_t send_iters = 0; + TaskCollection tc; + TaskID none(0); auto partitions = GetDefaultBlockPartitions(); - do { - all_sent = true; - for (int i = 0; i < partitions.size(); ++i) { - auto &md = mesh_data.Add(md_name, partitions[i], fields); - if (!sent[i]) { - if (SendBoundaryBuffers(md) != TaskStatus::complete) { - all_sent = false; - } else { - sent[i] = true; - } - } - } - send_iters++; - } while (!all_sent && send_iters < max_it); - PARTHENON_REQUIRE( - send_iters < max_it, - "Too many iterations waiting to send boundary communication buffers."); - - // wait to receive FillGhost variables - // TODO(someone) evaluate if ReceiveWithWait kind of logic is better, also related to - // https://github.com/lanl/parthenon/issues/418 - std::vector received(num_partitions, false); - bool all_received; - std::int64_t receive_iters = 0; - do { - all_received = true; - for (int i = 0; i < partitions.size(); ++i) { - auto &md = mesh_data.Add(md_name, partitions[i], fields); - if (!received[i]) { - if (ReceiveBoundaryBuffers(md) != TaskStatus::complete) { - all_received = false; - } else { - received[i] = true; - } - } - } - receive_iters++; - } while (!all_received && receive_iters < max_it); - PARTHENON_REQUIRE( - receive_iters < max_it, - "Too many iterations waiting to receive boundary communication buffers."); - - for (auto &partition : partitions) { - auto &md = mesh_data.Add(md_name, partition, fields); - // unpack FillGhost variables - SetBoundaries(md); - } - - // Now do prolongation, compute primitives, apply BCs - for (auto &partition : partitions) { - auto &md = mesh_data.Add(md_name, partition, fields); - if (multilevel) { - ApplyBoundaryConditionsOnCoarseOrFineMD(md, true); - ProlongateBoundaries(md); - } - ApplyBoundaryConditionsOnCoarseOrFineMD(md, false); + TaskRegion ®ion = tc.AddRegion(partitions.size()); + for (int i = 0; i < partitions.size(); i++) { + auto &md = mesh_data.Add(md_name, partitions[i], fields); + AddBoundaryExchangeTasks(none, region[i], md, multilevel); } } From e082eec66ce2725385ff4631019c8f8ce861ec94 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 3 Sep 2025 15:45:58 -0600 Subject: [PATCH 2/6] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index afbb21f052f0f..3848cae565bbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Current develop ### Added (new features/APIs/variables/...) +- [[PR 1314]](https://github.com/parthenon-hpc-lab/parthenon/pull/1314) Add option of user specified BCs in AddBoundaryExchangeTasks - [[PR 1142]](https://github.com/parthenon-hpc-lab/parthenon/pull/1142) Unify par_dispatch, par_for_outer & par_for_inner overloads - [[PR 1255]](https://github.com/parthenon-hpc-lab/parthenon/pull/1255) RK34 low storage 3rd order 4 stage SSP integrator with CFL <= 2 from Spiteri & Ruuth 2002, SIAM Journal on Numerical Analysis, 40(2):469–491 - [[PR 1283]](https://github.com/parthenon-hpc-lab/parthenon/pull/1283) Ability to automatically document ParameterInputs From 35d99a62c1b616cd1e84375f261e745fae4100db Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 3 Sep 2025 16:57:29 -0600 Subject: [PATCH 3/6] actually execute the task list... --- src/mesh/mesh.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mesh/mesh.cpp b/src/mesh/mesh.cpp index d3a5c5e637140..2a049655b6562 100644 --- a/src/mesh/mesh.cpp +++ b/src/mesh/mesh.cpp @@ -661,6 +661,7 @@ void Mesh::CommunicateBoundaries(std::string md_name, auto &md = mesh_data.Add(md_name, partitions[i], fields); AddBoundaryExchangeTasks(none, region[i], md, multilevel); } + tc.Execute(); } void Mesh::PreCommFillDerived() { From 10ea7d4697da3ae8b57d5e160e5b9f4a3e1ea430 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Wed, 3 Sep 2025 17:00:00 -0600 Subject: [PATCH 4/6] format and lint --- src/bvals/comms/boundary_communication.cpp | 5 ++--- src/bvals/comms/bvals_in_one.hpp | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index 8b8c6391d12a3..a8a252a8410be 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -439,9 +439,8 @@ template TaskStatus ProlongateInternalBounds(std::shared_ptr> &); template TaskStatus ProlongateInternalBounds(std::shared_ptr> &); -template TaskStatus -ProlongateInternalBounds(std::shared_ptr> &); - +template TaskStatus ProlongateInternalBounds( + std::shared_ptr> &); bool IsMeshMultilevel(std::shared_ptr> &md) { return md->GetMeshPointer()->multilevel; diff --git a/src/bvals/comms/bvals_in_one.hpp b/src/bvals/comms/bvals_in_one.hpp index 573baa711e915..71fabcaa26852 100644 --- a/src/bvals/comms/bvals_in_one.hpp +++ b/src/bvals/comms/bvals_in_one.hpp @@ -87,14 +87,15 @@ TaskID AddFluxCorrectionTasks(TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel); using BValOnMDFunc_t = std::function> &, bool)>; -using BValOnMDTasks_t = std::function>, bool)>; +using BValOnMDTasks_t = + std::function>, bool)>; bool IsMeshMultilevel(std::shared_ptr> &md); // Adds all relevant boundary communication to a single task list template -TaskID AddBoundaryExchangeTasks( - TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel, - BValOnMDTasks_t ApplyBCs) { +TaskID AddBoundaryExchangeTasks(TaskID dependency, TaskList &tl, + std::shared_ptr> &md, bool multilevel, + BValOnMDTasks_t ApplyBCs) { static_assert(bounds == BoundaryType::any || bounds == BoundaryType::gmg_same); auto send = tl.AddTask(dependency, TF(SendBoundBufs), md); @@ -108,8 +109,8 @@ TaskID AddBoundaryExchangeTasks( } auto fbound = ApplyBCs(pro, &tl, md, false); if (IsMeshMultilevel(md)) { - // Need to prolongate internal bounds after setting physical boundary - // conditions, since the internal prolongation uses the values of the + // Need to prolongate internal bounds after setting physical boundary + // conditions, since the internal prolongation uses the values of the // normal buffer on shared elements (rather than values in the coarse) // buffer for prolongation. fbound = tl.AddTask(fbound, TF(ProlongateInternalBounds), md); @@ -121,10 +122,11 @@ template TaskID AddBoundaryExchangeTasks( TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel, BValOnMDFunc_t ApplyBCs = ApplyBoundaryConditionsOnCoarseOrFineMD) { - return AddBoundaryExchangeTasks(dependency, tl, md, multilevel, - [&](TaskID id, TaskList *tl, std::shared_ptr> md, bool coarse){ - return tl->AddTask(id, TF(ApplyBCs), md, coarse); - }); + return AddBoundaryExchangeTasks( + dependency, tl, md, multilevel, + [&](TaskID id, TaskList *tl, std::shared_ptr> md, bool coarse) { + return tl->AddTask(id, TF(ApplyBCs), md, coarse); + }); } // These tasks should not be called in down stream code From e2eebe12cf333c2dc8ebb7dbe58a2267b3e5b70c Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 4 Sep 2025 11:56:14 -0600 Subject: [PATCH 5/6] fix bug --- src/bvals/comms/boundary_communication.cpp | 11 +++++++++++ src/bvals/comms/bvals_in_one.hpp | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/bvals/comms/boundary_communication.cpp b/src/bvals/comms/boundary_communication.cpp index a8a252a8410be..db1c70a910170 100644 --- a/src/bvals/comms/boundary_communication.cpp +++ b/src/bvals/comms/boundary_communication.cpp @@ -168,6 +168,8 @@ template TaskStatus SendBoundBufs(std::shared_ptr> &); template TaskStatus SendBoundBufs(std::shared_ptr> &); +template TaskStatus +SendBoundBufs(std::shared_ptr> &); template TaskStatus StartReceiveBoundBufs(std::shared_ptr> &md) { @@ -196,6 +198,8 @@ template TaskStatus StartReceiveBoundBufs( std::shared_ptr> &); template TaskStatus StartReceiveBoundBufs(std::shared_ptr> &); +template TaskStatus +StartReceiveBoundBufs(std::shared_ptr> &); template TaskStatus ReceiveBoundBufs(std::shared_ptr> &md) { @@ -246,6 +250,8 @@ template TaskStatus ReceiveBoundBufs(std::shared_ptr> &); template TaskStatus ReceiveBoundBufs(std::shared_ptr> &); +template TaskStatus +ReceiveBoundBufs(std::shared_ptr> &); template TaskStatus SetBounds(std::shared_ptr> &md) { @@ -356,6 +362,7 @@ template TaskStatus SetBounds(std::shared_ptr> &); template TaskStatus SetBounds(std::shared_ptr> &); +template TaskStatus SetBounds(std::shared_ptr> &); template TaskStatus ProlongateBounds(std::shared_ptr> &md) { @@ -400,6 +407,8 @@ template TaskStatus ProlongateBounds(std::shared_ptr> &); template TaskStatus ProlongateBounds(std::shared_ptr> &); +template TaskStatus +ProlongateBounds(std::shared_ptr> &); template TaskStatus ProlongateInternalBounds(std::shared_ptr> &md) { @@ -441,6 +450,8 @@ template TaskStatus ProlongateInternalBounds(std::shared_ptr> &); template TaskStatus ProlongateInternalBounds( std::shared_ptr> &); +template TaskStatus +ProlongateInternalBounds(std::shared_ptr> &); bool IsMeshMultilevel(std::shared_ptr> &md) { return md->GetMeshPointer()->multilevel; diff --git a/src/bvals/comms/bvals_in_one.hpp b/src/bvals/comms/bvals_in_one.hpp index 71fabcaa26852..091a836cbbb30 100644 --- a/src/bvals/comms/bvals_in_one.hpp +++ b/src/bvals/comms/bvals_in_one.hpp @@ -122,7 +122,7 @@ template TaskID AddBoundaryExchangeTasks( TaskID dependency, TaskList &tl, std::shared_ptr> &md, bool multilevel, BValOnMDFunc_t ApplyBCs = ApplyBoundaryConditionsOnCoarseOrFineMD) { - return AddBoundaryExchangeTasks( + return AddBoundaryExchangeTasks( dependency, tl, md, multilevel, [&](TaskID id, TaskList *tl, std::shared_ptr> md, bool coarse) { return tl->AddTask(id, TF(ApplyBCs), md, coarse); From b65f33541d08236d3ea323a6eeb7ba9e8fe86527 Mon Sep 17 00:00:00 2001 From: Luke Roberts Date: Thu, 4 Sep 2025 13:24:51 -0600 Subject: [PATCH 6/6] Add documentation --- doc/sphinx/src/boundary_conditions.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/sphinx/src/boundary_conditions.rst b/doc/sphinx/src/boundary_conditions.rst index c97cc6b449dda..58077e19e4bfb 100644 --- a/doc/sphinx/src/boundary_conditions.rst +++ b/doc/sphinx/src/boundary_conditions.rst @@ -145,3 +145,17 @@ for a more complete example): pkg->UserBoundaryFunctions[BF::inner_x2].push_back(GetMyBC()); ... } + +User override of boundary conditions in `AddBoundaryExchangeTasks`. +------------------------------------------------------------------- + +Sometimes it is desirable to apply different boundary conditions when communicating +on different containers (e.g. when one container represents the value of a variable +and another container represents a Newton-Raphson correction to that variable). To +easily allow this, `AddBoundaryExchangeTasks` can take a final argument with type +`std::function> &, bool)>` or with type +`std::function>, bool)>`. +When this argument is specified, all other boundary conditions will be ignored and +the passed functions will be used as boundary conditions instead. The final boolean +arguments of the functions specify if the boundary condition needs to be applied on +the coarse buffer. \ No newline at end of file