Skip to content

NgenSimulation: Narrow interaction with MPI to just the routing path where it's used #987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
PhilMiller wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

NgenSimulation: Narrow interaction with MPI to just the routing path where it's used #987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions include/core/NgenSimulation.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
#include <Simulation_Time.hpp>
#include <Layer.hpp>

#if NGEN_WITH_MPI
#include <mpi.h>
#endif

namespace hy_features
{
class HY_Features;
Expand All @@ -28,9 +32,7 @@ class NgenSimulation
Simulation_Time const& sim_time,
std::vector<std::shared_ptr<ngen::Layer>> layers,
std::unordered_map<std::string, int> catchment_indexes,
std::unordered_map<std::string, int> nexus_indexes,
int mpi_rank,
int mpi_num_procs
std::unordered_map<std::string, int> nexus_indexes
);
NgenSimulation() = delete;

Expand Down Expand Up @@ -58,10 +60,12 @@ class NgenSimulation
*/
void run_routing(std::string const& t_route_config_file_with_path);

#if NGEN_WITH_MPI
/**
* Run t-route via BMI on the stored nexus outflow values for the full configured duration of the simulation
*/
void run_routing_bmi(hy_features_t &features, std::string const& t_route_config_file_with_path);
void run_routing_bmi(hy_features_t &features, std::string const& t_route_config_file_with_path, MPI_Comm mpi_comm);
#endif

int get_nexus_index(std::string const& nexus_id) const;
double get_nexus_outflow(int nexus_index, int timestep_index) const;
Expand All @@ -85,9 +89,6 @@ class NgenSimulation
std::unordered_map<std::string, int> nexus_indexes_;
std::vector<double> nexus_downstream_flows_;

int mpi_rank_;
int mpi_num_procs_;

#if NGEN_WITH_ROUTING
std::unique_ptr<routing_py_adapter::Routing_Py_Adapter> router_;
#endif
Expand Down
11 changes: 6 additions & 5 deletions src/NGen.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -612,9 +612,7 @@ int main(int argc, char* argv[]) {
auto simulation = std::make_unique<NgenSimulation>(*sim_time,
layers,
std::move(catchment_indexes),
std::move(nexus_indexes),
mpi_rank,
mpi_num_procs);
std::move(nexus_indexes));

auto time_done_init = std::chrono::steady_clock::now();
std::chrono::duration<double> time_elapsed_init = time_done_init - time_start;
Expand Down Expand Up @@ -642,9 +640,12 @@ int main(int argc, char* argv[]) {
if (manager->get_using_routing()) {
std::string t_route_config_file_with_path = manager->get_t_route_config_file_with_path();
#if NGEN_WITH_ROUTING_TROUTE_BMI
simulation->run_routing_bmi(features, t_route_config_file_with_path);
simulation->run_routing_bmi(features, t_route_config_file_with_path, MPI_COMM_WORLD);
#else
simulation->run_routing(t_route_config_file_with_path);
// Run t-route from single process
if (mpi_rank == 0) {
simulation->run_routing(t_route_config_file_with_path);
}
#endif // NGEN_WITH_ROUTING_TROUTE_BMI
}

Expand Down
34 changes: 15 additions & 19 deletions src/core/NgenSimulation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,13 @@ NgenSimulation::NgenSimulation(
Simulation_Time const& sim_time,
std::vector<std::shared_ptr<ngen::Layer>> layers,
std::unordered_map<std::string, int> catchment_indexes,
std::unordered_map<std::string, int> nexus_indexes,
int mpi_rank,
int mpi_num_procs
std::unordered_map<std::string, int> nexus_indexes
)
: simulation_step_(0)
, sim_time_(std::make_shared<Simulation_Time>(sim_time))
, layers_(std::move(layers))
, catchment_indexes_(std::move(catchment_indexes))
, nexus_indexes_(std::move(nexus_indexes))
, mpi_rank_(mpi_rank)
, mpi_num_procs_(mpi_num_procs)
{
#if NGEN_WITH_ROUTING && NGEN_WITH_ROUTING_TROUTE_BMI
catchment_outflows_.reserve(catchment_indexes_.size() * get_num_output_times());
Expand Down Expand Up @@ -131,10 +127,6 @@ double NgenSimulation::get_nexus_outflow(int nexus_index, int timestep_index) co
void NgenSimulation::run_routing(std::string const& t_route_config_file_with_path)
{
#if NGEN_WITH_ROUTING
// Run t-route from single process
if (mpi_rank_ != 0)
return;

router_ = std::make_unique<routing_py_adapter::Routing_Py_Adapter>(t_route_config_file_with_path);

// Note: Currently, delta_time is set in the t-route yaml configuration file, and the
Expand All @@ -148,7 +140,7 @@ void NgenSimulation::run_routing(std::string const& t_route_config_file_with_pat
#endif
}

void NgenSimulation::run_routing_bmi(NgenSimulation::hy_features_t &features, std::string const& t_route_config_file_with_path)
void NgenSimulation::run_routing_bmi(NgenSimulation::hy_features_t &features, std::string const& t_route_config_file_with_path, MPI_Comm mpi_comm)
{
#if NGEN_WITH_ROUTING && NGEN_WITH_ROUTING_TROUTE_BMI
std::vector<double> *routing_nexus_downflows = &nexus_downstream_flows_;
Expand All @@ -160,17 +152,21 @@ void NgenSimulation::run_routing_bmi(NgenSimulation::hy_features_t &features, st
}

#if NGEN_WITH_MPI
int mpi_rank, mpi_num_procs;
MPI_Comm_rank(mpi_comm, &mpi_rank);
MPI_Comm_size(mpi_comm, &mpi_num_procs);

std::vector<double> all_nexus_downflows;
std::unordered_map<std::string, int> all_nexus_indexes;

if (mpi_num_procs_ > 1) {
if (mpi_num_procs > 1) {
std::vector<std::string> local_nexus_ids;
for (const auto& nexus : nexus_indexes_) {
local_nexus_ids.push_back(nexus.first);
}
// MPI_Gather all nexus IDs into a single vector
std::vector<std::string> all_nexus_ids = parallel::gather_strings(local_nexus_ids, MPI_COMM_WORLD);
if (mpi_rank_ == 0) {
std::vector<std::string> all_nexus_ids = parallel::gather_strings(local_nexus_ids, mpi_comm);
if (mpi_rank == 0) {
// filter to only the unique IDs
std::sort(all_nexus_ids.begin(), all_nexus_ids.end());
all_nexus_ids.erase(
Expand All @@ -179,10 +175,10 @@ void NgenSimulation::run_routing_bmi(NgenSimulation::hy_features_t &features, st
);
}
// MPI_Broadcast so all processes share the nexus IDs
all_nexus_ids = std::move(parallel::broadcast_strings(all_nexus_ids, MPI_COMM_WORLD));
all_nexus_ids = std::move(parallel::broadcast_strings(all_nexus_ids, mpi_comm));

// MPI_Reduce to collect the results from processes
if (mpi_rank_ == 0) {
if (mpi_rank == 0) {
all_nexus_downflows.resize(number_of_timesteps * all_nexus_ids.size(), 0.0);
}
std::vector<double> local_buffer(number_of_timesteps);
Expand All @@ -200,8 +196,8 @@ void NgenSimulation::run_routing_bmi(NgenSimulation::hy_features_t &features, st
// if this process does not have the id, fill with 0 to make sure it doesn't affect reduce sum
std::fill(local_buffer.begin(), local_buffer.end(), 0.0);
}
MPI_Reduce(local_buffer.data(), receive_buffer.data(), number_of_timesteps, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
if (mpi_rank_ == 0) {
MPI_Reduce(local_buffer.data(), receive_buffer.data(), number_of_timesteps, MPI_DOUBLE, MPI_SUM, 0, mpi_comm);
if (mpi_rank == 0) {
// copy reduce values to a combined downflows vector
all_nexus_indexes[nexus_id] = i;
for (int step = 0; step < number_of_timesteps; ++step) {
Expand All @@ -212,15 +208,15 @@ void NgenSimulation::run_routing_bmi(NgenSimulation::hy_features_t &features, st
}
}

if (mpi_rank_ == 0) {
if (mpi_rank == 0) {
// update root's local data for running t-route below
routing_nexus_indexes = &all_nexus_indexes;
routing_nexus_downflows = &all_nexus_downflows;
}
}
#endif // NGEN_WITH_MPI

if (mpi_rank_ == 0) { // Run t-route from single process
if (mpi_rank == 0) { // Run t-route from single process
//LOG(LogLevel::INFO, "Running T-Route on nexus outflows.");

// model for routing
Expand Down
4 changes: 1 addition & 3 deletions test/core/NgenSimulationTests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ TEST(NgenSimulation_Test, Construction)
sim_time,
layers,
catchment_indexes,
nexus_indexes,
mpi_rank,
mpi_num_procs
nexus_indexes
)};
}
Loading