diff --git a/core/distributed/vector.cpp b/core/distributed/vector.cpp index dac9e80d073..6f4509429f8 100644 --- a/core/distributed/vector.cpp +++ b/core/distributed/vector.cpp @@ -39,21 +39,6 @@ dim<2> compute_global_size(std::shared_ptr exec, return {num_global_rows, local_size[1]}; } - -template -void Vector::apply_impl(const LinOp* b, LinOp* x) const -{ - GKO_NOT_SUPPORTED(this); -} - - -template -void Vector::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const -{ - GKO_NOT_SUPPORTED(this); -} - template Vector::Vector(std::shared_ptr exec, mpi::communicator comm, dim<2> global_size, @@ -66,7 +51,7 @@ template Vector::Vector(std::shared_ptr exec, mpi::communicator comm, dim<2> global_size, dim<2> local_size, size_type stride) - : LinOp{exec, global_size}, + : EnableMultiVector{exec, global_size}, DistributedBase{comm}, local_{exec, local_size, stride} { @@ -77,7 +62,9 @@ template Vector::Vector(std::shared_ptr exec, mpi::communicator comm, dim<2> global_size, std::unique_ptr local_vector) - : LinOp{exec, global_size}, DistributedBase{comm}, local_{exec} + : EnableMultiVector{exec, global_size}, + DistributedBase{comm}, + local_{exec} { local_vector->move_to(&local_); } @@ -87,7 +74,7 @@ template Vector::Vector(std::shared_ptr exec, mpi::communicator comm, std::unique_ptr local_vector) - : LinOp{exec, {}}, DistributedBase{comm}, local_{exec} + : EnableMultiVector{exec, {}}, DistributedBase{comm}, local_{exec} { this->set_size(compute_global_size(exec, comm, local_vector->get_size())); local_vector->move_to(&local_); @@ -159,36 +146,6 @@ std::unique_ptr> Vector::create_const( } -template -std::unique_ptr> Vector::create_with_config_of( - ptr_param other) -{ - // De-referencing `other` before calling the functions (instead of - // using operator `->`) is currently required to be compatible with - // CUDA 10.1. - // Otherwise, it results in a compile error. - return (*other).create_with_same_config(); -} - - -template -std::unique_ptr> Vector::create_with_type_of( - ptr_param other, std::shared_ptr exec) -{ - return (*other).create_with_type_of_impl(exec, {}, {}, 0); -} - - -template -std::unique_ptr> Vector::create_with_type_of( - ptr_param other, std::shared_ptr exec, - const dim<2>& global_size, const dim<2>& local_size, size_type stride) -{ - return (*other).create_with_type_of_impl(exec, global_size, local_size, - stride); -} - - template template void Vector::read_distributed_impl( @@ -211,135 +168,110 @@ void Vector::read_distributed_impl( template -void Vector::read_distributed( - const device_matrix_data& data, - ptr_param> partition) -{ - this->read_distributed_impl(data, partition.get()); -} - - -template -void Vector::read_distributed( - const device_matrix_data& data, - ptr_param> partition) +void Vector::compute_absolute_inplace_impl() { - this->read_distributed_impl(data, partition.get()); + local_.compute_absolute_inplace(); } template -void Vector::read_distributed( - const device_matrix_data& data, - ptr_param> partition) +std::unique_ptr> +Vector::create_with_same_config_impl() const { - this->read_distributed_impl(data, partition.get()); + return Vector::create( + this->get_executor(), this->get_communicator(), this->get_size(), + this->get_local_vector()->get_size(), this->get_stride()); } template -void Vector::read_distributed( - const matrix_data& data, - ptr_param> partition) +std::unique_ptr> Vector::create_with_type_of_impl( + std::shared_ptr exec, const dim<2>& global_size, + const dim<2>& local_size, size_type stride) const { - this->read_distributed( - device_matrix_data::create_from_host( - this->get_executor(), data), - partition); + return Vector::create(exec, this->get_communicator(), global_size, + local_size, stride); } template -void Vector::read_distributed( - const matrix_data& data, - ptr_param> partition) +std::unique_ptr> Vector::create_subview_impl( + local_span rows, local_span columns) { - this->read_distributed( - device_matrix_data::create_from_host( - this->get_executor(), data), - partition); + auto exec = this->get_executor(); + auto comm = this->get_communicator(); + auto global_rows = this->get_size()[0]; + auto global_cols = this->get_size()[1]; + comm.all_reduce(exec, &global_rows, 1, MPI_SUM); + comm.all_reduce(exec, &global_cols, 1, MPI_SUM); + return create_subview_impl(rows, columns, {global_rows, global_cols}); } template -void Vector::read_distributed( - const matrix_data& data, - ptr_param> partition) +std::unique_ptr> Vector::create_subview_impl( + local_span rows, local_span columns) const { - this->read_distributed( - device_matrix_data::create_from_host( - this->get_executor(), data), - partition); + auto exec = this->get_executor(); + auto comm = this->get_communicator(); + auto global_rows = this->get_size()[0]; + auto global_cols = this->get_size()[1]; + comm.all_reduce(exec, &global_rows, 1, MPI_SUM); + comm.all_reduce(exec, &global_cols, 1, MPI_SUM); + return create_subview_impl(rows, columns, {global_rows, global_cols}); } template -void Vector::fill(const ValueType value) +std::unique_ptr> Vector::create_subview_impl( + local_span rows, local_span columns, dim<2> global_size) { - local_.fill(value); + return create(this->get_executor(), this->get_communicator(), global_size, + local_.create_subview(rows, columns)); } template -void Vector::convert_to( - Vector>* result) const +std::unique_ptr> Vector::create_subview_impl( + local_span rows, local_span columns, dim<2> global_size) const { - GKO_ASSERT(this->get_communicator().size() == - result->get_communicator().size()); - result->set_size(this->get_size()); - this->get_local_vector()->convert_to(&result->local_); + return create( + this->get_executor(), this->get_communicator(), global_size, + const_cast(local_).create_subview(rows, columns)); } template -void Vector::move_to(Vector>* result) +std::unique_ptr::real_type> +Vector::create_real_view_impl() const { - this->convert_to(result); -} - + const auto num_global_rows = this->get_size()[0]; + const auto num_cols = is_complex() ? 2 * this->get_size()[1] + : this->get_size()[1]; -#if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16 -template -void Vector::convert_to( - Vector>* result) const -{ - GKO_ASSERT(this->get_communicator().size() == - result->get_communicator().size()); - result->set_size(this->get_size()); - this->get_local_vector()->convert_to(&result->local_); + return real_type::create_const( + this->get_executor(), this->get_communicator(), + dim<2>{num_global_rows, num_cols}, local_.create_real_view()); } template -void Vector::move_to(Vector>* result) +std::unique_ptr::real_type> +Vector::create_real_view_impl() { - this->convert_to(result); -} -#endif - + const auto num_global_rows = this->get_size()[0]; + const auto num_cols = is_complex() ? 2 * this->get_size()[1] + : this->get_size()[1]; -#if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 -template -void Vector::convert_to( - Vector>* result) const -{ - GKO_ASSERT(this->get_communicator().size() == - result->get_communicator().size()); - result->set_size(this->get_size()); - this->get_local_vector()->convert_to(&result->local_); + return real_type::create(this->get_executor(), this->get_communicator(), + dim<2>{num_global_rows, num_cols}, + local_.create_real_view()); } -template -void Vector::move_to(Vector>* result) -{ - this->convert_to(result); -} -#endif - template std::unique_ptr::absolute_type> -Vector::compute_absolute() const +Vector::compute_absolute_impl() const { auto exec = this->get_executor(); @@ -356,23 +288,15 @@ Vector::compute_absolute() const template -void Vector::compute_absolute_inplace() +void Vector::compute_absolute_impl(absolute_type* result) const { - local_.compute_absolute_inplace(); -} - - -template -const typename Vector::local_vector_type* -Vector::get_local_vector() const -{ - return &local_; + local_.compute_absolute(&result->local_); } template std::unique_ptr::complex_type> -Vector::make_complex() const +Vector::make_complex_impl() const { auto result = complex_type::create( this->get_executor(), this->get_communicator(), this->get_size(), @@ -384,111 +308,113 @@ Vector::make_complex() const template -void Vector::make_complex( - ptr_param result) const +std::unique_ptr::real_type> +Vector::get_real_impl() const { - this->get_local_vector()->make_complex(&result->local_); + auto result = real_type::create(this->get_executor(), + this->get_communicator(), this->get_size(), + this->get_local_vector()->get_size(), + this->get_local_vector()->get_stride()); + this->get_real(result); + return result; } template std::unique_ptr::real_type> -Vector::get_real() const +Vector::get_imag_impl() const { auto result = real_type::create(this->get_executor(), this->get_communicator(), this->get_size(), this->get_local_vector()->get_size(), this->get_local_vector()->get_stride()); - this->get_real(result); + this->get_imag(result); return result; } template -void Vector::get_real(ptr_param result) const +void Vector::make_complex_impl(complex_type* result) const { - this->get_local_vector()->get_real(&result->local_); + this->get_local_vector()->make_complex(&result->local_); } template -std::unique_ptr::real_type> -Vector::get_imag() const +void Vector::get_real_impl(real_type* result) const { - auto result = real_type::create(this->get_executor(), - this->get_communicator(), this->get_size(), - this->get_local_vector()->get_size(), - this->get_local_vector()->get_stride()); - this->get_imag(result); - return result; + this->get_local_vector()->get_real(&result->local_); } template -void Vector::get_imag(ptr_param result) const +void Vector::get_imag_impl(real_type* result) const { this->get_local_vector()->get_imag(&result->local_); } template -void Vector::scale(ptr_param alpha) +void Vector::fill_impl(value_type value) { - local_.scale(alpha); + local_.fill(value); } template -void Vector::inv_scale(ptr_param alpha) +void Vector::scale_impl(scaling_param alpha) { - local_.inv_scale(alpha); + std::visit([this](auto alpha_v) { local_.scale(alpha_v); }, alpha.variant); } template -void Vector::add_scaled(ptr_param alpha, - ptr_param b) +void Vector::inv_scale_impl(scaling_param alpha) { - auto dense_b = as(b); - local_.add_scaled(alpha, dense_b->get_local_vector()); + std::visit([this](auto alpha_v) { local_.inv_scale(alpha_v); }, + alpha.variant); } template -void Vector::sub_scaled(ptr_param alpha, - ptr_param b) +void Vector::add_scaled_impl(scaling_param alpha, + const Vector* b) { - auto dense_b = as(b); - local_.sub_scaled(alpha, dense_b->get_local_vector()); + std::visit( + [this, b](auto alpha_v) { + local_.add_scaled(alpha_v, b->get_local_vector()); + }, + alpha.variant); } template -void Vector::compute_dot(ptr_param b, - ptr_param result) const +void Vector::sub_scaled_impl(scaling_param alpha, + const Vector* b) { - array tmp{this->get_executor()}; - this->compute_dot(b, result, tmp); + std::visit( + [this, b](auto alpha_v) { + local_.sub_scaled(alpha_v, b->get_local_vector()); + }, + alpha.variant); } template -void Vector::compute_dot(ptr_param b, - ptr_param result, - array& tmp) const +void Vector::compute_dot_impl( + const Vector* b, matrix::MultiVector* result, + array& tmp) const { - GKO_ASSERT_EQUAL_DIMENSIONS(result, dim<2>(1, this->get_size()[1])); auto exec = this->get_executor(); const auto comm = this->get_communicator(); - auto dense_res = - make_temporary_clone(exec, as>(result)); + auto dense_res = as>(result); this->get_local_vector()->compute_dot(as(b)->get_local_vector(), - dense_res.get(), tmp); + dense_res, tmp); exec->synchronize(); - auto sum_op = gko::experimental::mpi::sum(); + auto sum_op = gko::experimental::mpi::sum(); if (mpi::requires_host_buffer(exec, comm)) { host_reduction_buffer_.init(exec->get_master(), dense_res->get_size()); - host_reduction_buffer_->copy_from(dense_res.get()); + host_reduction_buffer_->copy_from(dense_res); comm.all_reduce(exec->get_master(), host_reduction_buffer_->get_values(), static_cast(this->get_size()[1]), sum_op.get_op()); @@ -501,31 +427,20 @@ void Vector::compute_dot(ptr_param b, template -void Vector::compute_conj_dot(ptr_param b, - ptr_param result) const +void Vector::compute_conj_dot_impl( + const Vector* b, matrix::MultiVector* result, + array& tmp) const { - array tmp{this->get_executor()}; - this->compute_conj_dot(b, result, tmp); -} - - -template -void Vector::compute_conj_dot(ptr_param b, - ptr_param result, - array& tmp) const -{ - GKO_ASSERT_EQUAL_DIMENSIONS(result, dim<2>(1, this->get_size()[1])); auto exec = this->get_executor(); const auto comm = this->get_communicator(); - auto dense_res = - make_temporary_clone(exec, as>(result)); + auto dense_res = as>(result); this->get_local_vector()->compute_conj_dot( - as(b)->get_local_vector(), dense_res.get(), tmp); + as(b)->get_local_vector(), dense_res, tmp); exec->synchronize(); - auto sum_op = gko::experimental::mpi::sum(); + auto sum_op = gko::experimental::mpi::sum(); if (mpi::requires_host_buffer(exec, comm)) { host_reduction_buffer_.init(exec->get_master(), dense_res->get_size()); - host_reduction_buffer_->copy_from(dense_res.get()); + host_reduction_buffer_->copy_from(dense_res); comm.all_reduce(exec->get_master(), host_reduction_buffer_->get_values(), static_cast(this->get_size()[1]), sum_op.get_op()); @@ -538,49 +453,35 @@ void Vector::compute_conj_dot(ptr_param b, template -void Vector::compute_norm2(ptr_param result) const -{ - array tmp{this->get_executor()}; - this->compute_norm2(result, tmp); -} - - -template -void Vector::compute_norm2(ptr_param result, - array& tmp) const +void Vector::compute_norm2_impl(norm_type* result, + array& tmp) const { using NormVector = typename local_vector_type::absolute_type; auto exec = this->get_executor(); const auto comm = this->get_communicator(); - auto dense_res = make_temporary_clone(exec, as(result)); - this->compute_squared_norm2(dense_res.get(), tmp); + auto dense_res = as(result); + this->compute_squared_norm2(dense_res, tmp); exec->run(vector::make_compute_sqrt(dense_res->get_device_view())); } template -void Vector::compute_norm1(ptr_param result) const -{ - array tmp{this->get_executor()}; - this->compute_norm1(result, tmp); -} - - -template -void Vector::compute_norm1(ptr_param result, - array& tmp) const +void Vector::compute_squared_norm2_impl(norm_type* result, + array& tmp) const { using NormVector = typename local_vector_type::absolute_type; - GKO_ASSERT_EQUAL_DIMENSIONS(result, dim<2>(1, this->get_size()[1])); auto exec = this->get_executor(); const auto comm = this->get_communicator(); - auto dense_res = make_temporary_clone(exec, as(result)); - this->get_local_vector()->compute_norm1(dense_res.get()); + auto dense_res = as(result); + exec->run(vector::make_compute_squared_norm2( + this->get_local_vector()->get_const_device_view(), + dense_res->get_device_view(), tmp)); exec->synchronize(); - auto norm_sum_op = gko::experimental::mpi::sum>(); + auto norm_sum_op = + gko::experimental::mpi::sum>(); if (mpi::requires_host_buffer(exec, comm)) { host_norm_buffer_.init(exec->get_master(), dense_res->get_size()); - host_norm_buffer_->copy_from(dense_res.get()); + host_norm_buffer_->copy_from(dense_res); comm.all_reduce(exec->get_master(), host_norm_buffer_->get_values(), static_cast(this->get_size()[1]), norm_sum_op.get_op()); @@ -594,30 +495,20 @@ void Vector::compute_norm1(ptr_param result, template -void Vector::compute_squared_norm2(ptr_param result) const -{ - array tmp{this->get_executor()}; - this->compute_squared_norm2(result, tmp); -} - - -template -void Vector::compute_squared_norm2(ptr_param result, - array& tmp) const +void Vector::compute_norm1_impl(norm_type* result, + array& tmp) const { using NormVector = typename local_vector_type::absolute_type; - GKO_ASSERT_EQUAL_DIMENSIONS(result, dim<2>(1, this->get_size()[1])); auto exec = this->get_executor(); const auto comm = this->get_communicator(); - auto dense_res = make_temporary_clone(exec, as(result)); - exec->run(vector::make_compute_squared_norm2( - this->get_local_vector()->get_const_device_view(), - dense_res->get_device_view(), tmp)); + auto dense_res = as(result); + this->get_local_vector()->compute_norm1(dense_res); exec->synchronize(); - auto norm_sum_op = gko::experimental::mpi::sum>(); + auto norm_sum_op = + gko::experimental::mpi::sum>(); if (mpi::requires_host_buffer(exec, comm)) { host_norm_buffer_.init(exec->get_master(), dense_res->get_size()); - host_norm_buffer_->copy_from(dense_res.get()); + host_norm_buffer_->copy_from(dense_res); comm.all_reduce(exec->get_master(), host_norm_buffer_->get_values(), static_cast(this->get_size()[1]), norm_sum_op.get_op()); @@ -631,7 +522,207 @@ void Vector::compute_squared_norm2(ptr_param result, template -void Vector::compute_mean(ptr_param result) const +typename Vector::device_view +Vector::get_local_device_view_impl() +{ + return local_.get_device_view(); +} + + +template +typename Vector::const_device_view +Vector::get_const_local_device_view_impl() const +{ + return local_.get_const_device_view(); +} + + +template +void Vector::read_distributed( + const device_matrix_data& data, + ptr_param> partition) +{ + this->read_distributed_impl(data, partition.get()); +} + + +template +void Vector::read_distributed( + const device_matrix_data& data, + ptr_param> partition) +{ + this->read_distributed_impl(data, partition.get()); +} + + +template +void Vector::read_distributed( + const device_matrix_data& data, + ptr_param> partition) +{ + this->read_distributed_impl(data, partition.get()); +} + + +template +void Vector::read_distributed( + const matrix_data& data, + ptr_param> partition) +{ + this->read_distributed( + device_matrix_data::create_from_host( + this->get_executor(), data), + partition); +} + + +template +void Vector::read_distributed( + const matrix_data& data, + ptr_param> partition) +{ + this->read_distributed( + device_matrix_data::create_from_host( + this->get_executor(), data), + partition); +} + + +template +void Vector::read_distributed( + const matrix_data& data, + ptr_param> partition) +{ + this->read_distributed( + device_matrix_data::create_from_host( + this->get_executor(), data), + partition); +} + + +template +void Vector::convert_to( + Vector>* result) const +{ + GKO_ASSERT(this->get_communicator().size() == + result->get_communicator().size()); + result->set_size(this->get_size()); + this->get_local_vector()->convert_to(&result->local_); +} + + +template +void Vector::move_to(Vector>* result) +{ + this->convert_to(result); +} + + +#if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16 +template +void Vector::convert_to( + Vector>* result) const +{ + GKO_ASSERT(this->get_communicator().size() == + result->get_communicator().size()); + result->set_size(this->get_size()); + this->get_local_vector()->convert_to(&result->local_); +} + + +template +void Vector::move_to(Vector>* result) +{ + this->convert_to(result); +} +#endif + + +#if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 +template +void Vector::convert_to( + Vector>* result) const +{ + GKO_ASSERT(this->get_communicator().size() == + result->get_communicator().size()); + result->set_size(this->get_size()); + this->get_local_vector()->convert_to(&result->local_); +} + + +template +void Vector::move_to(Vector>* result) +{ + this->convert_to(result); +} +#endif + + +template +const typename Vector::local_vector_type* +Vector::get_local_vector() const +{ + return &local_; +} + + +template +template +temporary_conversion> Vector::as_precision() +{ + // See the implementation of MultiVector::as_precision for details + if constexpr (is_complex() == is_complex()) { + return temporary_conversion>::create(this); + } else if constexpr (is_complex() && + std::is_same_v, + ValueType>) { + return temporary_conversion>::create( + this->create_real_view()); + } else { + GKO_NOT_IMPLEMENTED; + } +} + +#define GKO_DECLARE_VECTOR_AS_PRECISION(ValueType, OtherValueType) \ + auto Vector::as_precision() \ + ->temporary_conversion> +#define GKO_DECLARE_VECTOR_AS_PRECISION_same(ValueType) \ + GKO_DECLARE_VECTOR_AS_PRECISION(ValueType, ValueType) +GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION(GKO_DECLARE_VECTOR_AS_PRECISION); +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_VECTOR_AS_PRECISION_same); + + +template +template +temporary_conversion> +Vector::as_precision() const +{ + // See the implementation of MultiVector::as_precision for details + if constexpr (is_complex() == is_complex()) { + return temporary_conversion>::create(this); + } else if constexpr (is_complex() && + std::is_same_v, + ValueType>) { + return temporary_conversion>::create( + this->create_real_view()); + } else { + GKO_NOT_IMPLEMENTED; + } +} + +#define GKO_DECLARE_VECTOR_CONST_AS_PRECISION(ValueType, OtherValueType) \ + auto Vector::as_precision() \ + const->temporary_conversion> +#define GKO_DECLARE_VECTOR_CONST_AS_PRECISION_same(ValueType) \ + GKO_DECLARE_VECTOR_CONST_AS_PRECISION(ValueType, ValueType) +GKO_INSTANTIATE_FOR_EACH_VALUE_CONVERSION( + GKO_DECLARE_VECTOR_CONST_AS_PRECISION); +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_VECTOR_CONST_AS_PRECISION_same); + + +template +void Vector::compute_mean( + ptr_param result) const { array tmp{this->get_executor()}; this->compute_mean(result, tmp); @@ -639,7 +730,7 @@ void Vector::compute_mean(ptr_param result) const template -void Vector::compute_mean(ptr_param result, +void Vector::compute_mean(ptr_param result, array& tmp) const { using MeanVector = local_vector_type; @@ -674,42 +765,46 @@ void Vector::compute_mean(ptr_param result, } template -ValueType& Vector::at_local(size_type row, size_type col) noexcept +typename Vector::value_type& Vector::at_local( + size_type row, size_type col) noexcept { return local_.at(row, col); } template -ValueType Vector::at_local(size_type row, - size_type col) const noexcept +typename Vector::value_type Vector::at_local( + size_type row, size_type col) const noexcept { return local_.at(row, col); } template -ValueType& Vector::at_local(size_type idx) noexcept +typename Vector::value_type& Vector::at_local( + size_type idx) noexcept { return local_.at(idx); } template -ValueType Vector::at_local(size_type idx) const noexcept +typename Vector::value_type Vector::at_local( + size_type idx) const noexcept { return local_.at(idx); } template -ValueType* Vector::get_local_values() +typename Vector::value_type* Vector::get_local_values() { return local_.get_values(); } template -const ValueType* Vector::get_const_local_values() const +const typename Vector::value_type* +Vector::get_const_local_values() const { return local_.get_const_values(); } @@ -725,71 +820,6 @@ void Vector::resize(dim<2> global_size, dim<2> local_size) } -template -std::unique_ptr::real_type> -Vector::create_real_view() const -{ - const auto num_global_rows = this->get_size()[0]; - const auto num_cols = - is_complex() ? 2 * this->get_size()[1] : this->get_size()[1]; - - return real_type::create_const( - this->get_executor(), this->get_communicator(), - dim<2>{num_global_rows, num_cols}, local_.create_real_view()); -} - - -template -std::unique_ptr::real_type> -Vector::create_real_view() -{ - const auto num_global_rows = this->get_size()[0]; - const auto num_cols = - is_complex() ? 2 * this->get_size()[1] : this->get_size()[1]; - - return real_type::create(this->get_executor(), this->get_communicator(), - dim<2>{num_global_rows, num_cols}, - local_.create_real_view()); -} - - -template -std::unique_ptr> Vector::create_submatrix( - local_span rows, local_span columns, dim<2> global_size) -{ - return this->create_submatrix_impl(rows, columns, global_size); -} - - -template -std::unique_ptr> Vector::create_with_same_config() - const -{ - return Vector::create( - this->get_executor(), this->get_communicator(), this->get_size(), - this->get_local_vector()->get_size(), this->get_stride()); -} - - -template -std::unique_ptr> Vector::create_with_type_of_impl( - std::shared_ptr exec, const dim<2>& global_size, - const dim<2>& local_size, size_type stride) const -{ - return Vector::create(exec, this->get_communicator(), global_size, - local_size, stride); -} - - -template -std::unique_ptr> Vector::create_submatrix_impl( - local_span rows, local_span columns, dim<2> global_size) -{ - return Vector::create(this->get_executor(), this->get_communicator(), - global_size, local_.create_submatrix(rows, columns)); -} - - #define GKO_DECLARE_DISTRIBUTED_VECTOR(ValueType) class Vector GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DISTRIBUTED_VECTOR); diff --git a/include/ginkgo/core/distributed/vector.hpp b/include/ginkgo/core/distributed/vector.hpp index bfdf34d2a8f..46c9769c017 100644 --- a/include/ginkgo/core/distributed/vector.hpp +++ b/include/ginkgo/core/distributed/vector.hpp @@ -64,19 +64,16 @@ class Partition; * @ingroup LinOp */ template -class Vector - : public LinOp, - public EnableCloneable>, - public ConvertibleTo>>, +class Vector : public EnableMultiVector>, + public ConvertibleTo>>, #if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16 - public ConvertibleTo>>, + public ConvertibleTo>>, #endif #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 - public ConvertibleTo>>, + public ConvertibleTo>>, #endif - public EnableAbsoluteComputation>>, - public DistributedBase { - friend class EnableCloneable; + public DistributedBase { + friend class EnableMultiVector; friend class Vector>; friend class Vector>; friend class Vector>; @@ -84,57 +81,21 @@ class Vector GKO_ASSERT_SUPPORTED_VALUE_TYPE; public: - using EnableCloneable::convert_to; - using EnableCloneable::move_to; + using EnableMultiVector::convert_to; + using EnableMultiVector::move_to; using ConvertibleTo>>::convert_to; using ConvertibleTo>>::move_to; - using value_type = ValueType; - using absolute_type = remove_complex; - using real_type = absolute_type; - using complex_type = Vector>; + using value_type = typename EnableMultiVector::value_type; + using absolute_type = typename EnableMultiVector::absolute_type; + using real_type = typename EnableMultiVector::real_type; + using complex_type = typename EnableMultiVector::complex_type; + using norm_type = typename EnableMultiVector::norm_type; + using device_view = typename EnableMultiVector::device_view; + using const_device_view = + typename EnableMultiVector::const_device_view; using local_vector_type = gko::matrix::MultiVector; - /** - * Creates a distributed Vector with the same size and stride as another - * Vector. - * - * @param other The other vector whose configuration needs to copied. - */ - static std::unique_ptr create_with_config_of( - ptr_param other); - - - /** - * Creates an empty Vector with the same type as another Vector, but on a - * different executor. - * - * @param other The other multi-vector whose type we target. - * @param exec The executor of the new multi-vector. - * - * @note The new multi-vector uses the same communicator as other. - * - * @returns an empty Vector with the type of other. - */ - static std::unique_ptr create_with_type_of( - ptr_param other, std::shared_ptr exec); - - /** - * Creates an Vector with the same type as another Vector, but on a - * different executor and with a different size. - * - * @param other The other multi-vector whose type we target. - * @param exec The executor of the new multi-vector. - * @param global_size The global size of the multi-vector. - * @param local_size The local size of the multi-vector. - * @param stride The stride of the new multi-vector. - * - * @returns a Vector of specified size with the type of other. - */ - static std::unique_ptr create_with_type_of( - ptr_param other, std::shared_ptr exec, - const dim<2>& global_size, const dim<2>& local_size, size_type stride); - /** * Reads a vector from the device_matrix_data structure and a global row * partition. @@ -202,217 +163,6 @@ class Vector void move_to(Vector>* result) override; #endif - std::unique_ptr compute_absolute() const override; - - void compute_absolute_inplace() override; - - /** - * Creates a complex copy of the original vectors. If the original vectors - * were real, the imaginary part of the result will be zero. - */ - std::unique_ptr make_complex() const; - - /** - * Writes a complex copy of the original vectors to given complex vectors. - * If the original vectors were real, the imaginary part of the result will - * be zero. - */ - void make_complex(ptr_param result) const; - - /** - * Creates new real vectors and extracts the real part of the original - * vectors into that. - */ - std::unique_ptr get_real() const; - - /** - * Extracts the real part of the original vectors into given real vectors. - */ - void get_real(ptr_param result) const; - - /** - * Creates new real vectors and extracts the imaginary part of the - * original vectors into that. - */ - std::unique_ptr get_imag() const; - - /** - * Extracts the imaginary part of the original vectors into given real - * vectors. - */ - void get_imag(ptr_param result) const; - - /** - * Fill the distributed vectors with a given value. - * - * @param value the value to be filled - */ - void fill(ValueType value); - - /** - * Scales the vectors with a scalar (aka: BLAS scal). - * - * @param alpha If alpha is 1x1 MultiVector matrx, the all vectors are - * scaled by alpha. If it is a MultiVector row vector of values, then i-th - * column vector is scaled with the i-th element of alpha (the number of - * columns of alpha has to match the number of vectors). - */ - void scale(ptr_param alpha); - - /** - * Scales the vectors with the inverse of a scalar. - * - * @param alpha If alpha is 1x1 MultiVector, the all vectors are scaled - * by 1 / alpha. If it is a MultiVector row vector of values, - * then i-th column vector is scaled with the inverse - * of the i-th element of alpha (the number of columns of - * alpha has to match the number of vectors). - */ - void inv_scale(ptr_param alpha); - - /** - * Adds `b` scaled by `alpha` to the vectors (aka: BLAS axpy). - * - * @param alpha If alpha is 1x1 MultiVector, the all vectors of b are - * scaled by alpha. If it is a MultiVector row vector of values, then i-th - * column vector of b is scaled with the i-th element of alpha (the number - * of columns of alpha has to match the number of vectors). - * @param b a (multi-)vector of the same dimension as this - */ - void add_scaled(ptr_param alpha, ptr_param b); - - /** - * Subtracts `b` scaled by `alpha` from the vectors (aka: BLAS axpy). - * - * @param alpha If alpha is 1x1 MultiVector, the all vectors of b are - * scaled by alpha. If it is a MultiVector row vector of values, then i-th - * column vector of b is scaled with the i-th element of alpha (the number - * of c - * @param b a (multi-)vector of the same dimension as this - */ - void sub_scaled(ptr_param alpha, ptr_param b); - - /** - * Computes the column-wise dot product of this (multi-)vector and `b` using - * a global reduction. - * - * @param b a (multi-)vector of same dimension as this - * @param result a row MultiVector, used to store the dot product - * (the number of column in result must match the number - * of columns of this) - */ - void compute_dot(ptr_param b, ptr_param result) const; - - /** - * Computes the column-wise dot product of this (multi-)vector and `b` using - * a global reduction. - * - * @param b a (multi-)vector of same dimension as this - * @param result a row MultiVector, used to store the dot product - * (the number of column in result must match the number - * of columns of this) - * @param tmp the temporary storage to use for partial sums during the - * reduction computation. It may be resized and/or reset to the - * correct executor. - */ - void compute_dot(ptr_param b, ptr_param result, - array& tmp) const; - - /** - * Computes the column-wise dot product of this (multi-)vector and `conj(b)` - * using a global reduction. - * - * @param b a (multi-)vector of same dimension as this - * @param result a row MultiVector, used to store the dot product - * (the number of column in result must match the number - * of columns of this) - */ - void compute_conj_dot(ptr_param b, - ptr_param result) const; - - /** - * Computes the column-wise dot product of this (multi-)vector and `conj(b)` - * using a global reduction. - * - * @param b a (multi-)vector of same dimension as this - * @param result a row MultiVector, used to store the dot product - * (the number of column in result must match the number - * of columns of this) - * @param tmp the temporary storage to use for partial sums during the - * reduction computation. It may be resized and/or reset to the - * correct executor. - */ - void compute_conj_dot(ptr_param b, ptr_param result, - array& tmp) const; - - /** - * Computes the square of the column-wise Euclidean ($L^2$) norm of this - * (multi-)vector using a global reduction. - * - * @param result a MultiVector row vector, used to store the norm - * (the number of columns in the vector must match the number - * of columns of this) - */ - void compute_squared_norm2(ptr_param result) const; - - /** - * Computes the square of the column-wise Euclidean ($L^2$) norm of this - * (multi-)vector using a global reduction. - * - * @param result a MultiVector row vector, used to store the norm - * (the number of columns in the vector must match the - * number of columns of this) - * @param tmp the temporary storage to use for partial sums during the - * reduction computation. It may be resized and/or reset to the - * correct executor. - */ - void compute_squared_norm2(ptr_param result, array& tmp) const; - - /** - * Computes the Euclidean (L^2) norm of this (multi-)vector using a global - * reduction. - * - * @param result a row MultiVector, used to store the norm - * (the number of columns in result must match the number - * of columns of this) - */ - void compute_norm2(ptr_param result) const; - - /** - * Computes the Euclidean (L^2) norm of this (multi-)vector using a global - * reduction. - * - * @param result a row MultiVector, used to store the norm - * (the number of columns in result must match the number - * of columns of this) - * @param tmp the temporary storage to use for partial sums during the - * reduction computation. It may be resized and/or reset to the - * correct executor. - */ - void compute_norm2(ptr_param result, array& tmp) const; - - /** - * Computes the column-wise (L^1) norm of this (multi-)vector. - * - * @param result a row MultiVector, used to store the norm - * (the number of columns in result must match the number - * of columns of this) - */ - void compute_norm1(ptr_param result) const; - - /** - * Computes the column-wise (L^1) norm of this (multi-)vector using a global - * reduction. - * - * @param result a row MultiVector, used to store the norm - * (the number of columns in result must match the number - * of columns of this) - * @param tmp the temporary storage to use for partial sums during the - * reduction computation. It may be resized and/or reset to the - * correct executor. - */ - void compute_norm1(ptr_param result, array& tmp) const; - /** * Computes the column-wise mean of this (multi-)vector using a global * reduction. @@ -421,7 +171,7 @@ class Vector * (the number of columns in result must match the number * of columns of this) */ - void compute_mean(ptr_param result) const; + void compute_mean(ptr_param result) const; /** * Computes the column-wise arithmetic mean of this (multi-)vector using a @@ -434,7 +184,8 @@ class Vector * reduction computation. It may be resized and/or reset to the * correct executor. */ - void compute_mean(ptr_param result, array& tmp) const; + void compute_mean(ptr_param result, + array& tmp) const; /** * Returns a single element of the multi-vector. @@ -467,12 +218,12 @@ class Vector * stored at (e.g. trying to call this method on a GPU matrix from * the OMP results in a runtime error) */ - ValueType& at_local(size_type idx) noexcept; + value_type& at_local(size_type idx) noexcept; /** * @copydoc Vector::at(size_type) */ - ValueType at_local(size_type idx) const noexcept; + value_type at_local(size_type idx) const noexcept; /** * Returns a pointer to the array of local values of the multi-vector. @@ -497,34 +248,14 @@ class Vector */ const local_vector_type* get_local_vector() const; - /** - * Create a real view of the (potentially) complex original multi-vector. - * If the original vector is real, nothing changes. If the original vector - * is complex, the result is created by viewing the complex vector with as - * real with a reinterpret_cast with twice the number of columns and - * double the stride. - */ - std::unique_ptr create_real_view() const; - - /** - * @copydoc create_real_view - */ - std::unique_ptr create_real_view(); + size_type get_stride() const noexcept { return local_.get_stride(); } - /** - * Creates a view of a submatrix of this vector. - * - * @param rows The local rows of the submatrix - * @param columns The local columns of the submatrix - * @param global_size The global size of the submatrix - * - * @return A view of a submatrix. - */ - std::unique_ptr create_submatrix(local_span rows, - local_span columns, - dim<2> global_size); + template + [[nodiscard]] temporary_conversion> as_precision(); - size_type get_stride() const noexcept { return local_.get_stride(); } + template + [[nodiscard]] temporary_conversion> + as_precision() const; /** * Creates an empty distributed vector with a specified size @@ -655,41 +386,80 @@ class Vector const device_matrix_data& data, const Partition* partition); - void apply_impl(const LinOp*, LinOp*) const override; + void compute_absolute_inplace_impl() override; - void apply_impl(const LinOp*, const LinOp*, const LinOp*, - LinOp*) const override; - - /** - * Creates a distributed vector with the same size and stride as the callers - * vector. - * - * @returns a Vector with the same size and stride as the caller. - */ - virtual std::unique_ptr create_with_same_config() const; + [[nodiscard]] std::unique_ptr create_with_same_config_impl() + const override; - /** - * Creates a Vector with the same type as the callers multi-vector. - * - * @note The new vector will use the same communicator as the caller. - * - * @param exec the executor of the new vector. - * @param global_size global_size of the vector. - * @param local_size the size of the local MultiVector vector. - * @param stride the stride of the local MultiVector vector. - * - * @returns a Vector with the same type as the caller. - */ - virtual std::unique_ptr create_with_type_of_impl( + [[nodiscard]] std::unique_ptr create_with_type_of_impl( std::shared_ptr exec, const dim<2>& global_size, - const dim<2>& local_size, size_type stride) const; + const dim<2>& local_size, size_type stride) const override; - /** - * @copydoc create_submatrix - */ - virtual std::unique_ptr create_submatrix_impl(local_span rows, - local_span columns, - dim<2> global_size); + [[nodiscard]] std::unique_ptr create_subview_impl( + local_span rows, local_span columns) override; + + [[nodiscard]] std::unique_ptr create_subview_impl( + local_span rows, local_span columns) const override; + + [[nodiscard]] std::unique_ptr create_subview_impl( + local_span rows, local_span columns, dim<2> global_size) override; + + [[nodiscard]] std::unique_ptr create_subview_impl( + local_span rows, local_span columns, dim<2> global_size) const override; + + [[nodiscard]] std::unique_ptr create_real_view_impl() + const override; + + [[nodiscard]] std::unique_ptr create_real_view_impl() override; + + [[nodiscard]] std::unique_ptr compute_absolute_impl() + const override; + + void compute_absolute_impl(absolute_type* result) const override; + + [[nodiscard]] std::unique_ptr make_complex_impl() + const override; + + [[nodiscard]] std::unique_ptr get_real_impl() const override; + + [[nodiscard]] std::unique_ptr get_imag_impl() const override; + + void make_complex_impl(complex_type* result) const override; + + void get_real_impl(real_type* result) const override; + + void get_imag_impl(real_type* result) const override; + + void fill_impl(value_type value) override; + + void scale_impl(scaling_param alpha) override; + + void inv_scale_impl(scaling_param alpha) override; + + void add_scaled_impl(scaling_param alpha, + const Vector* b) override; + + void sub_scaled_impl(scaling_param alpha, + const Vector* b) override; + + void compute_dot_impl(const Vector* b, + matrix::MultiVector* result, + array& tmp) const override; + + void compute_conj_dot_impl(const Vector* b, + matrix::MultiVector* result, + array& tmp) const override; + + void compute_norm2_impl(norm_type* result, array& tmp) const override; + + void compute_squared_norm2_impl(norm_type* result, + array& tmp) const override; + + void compute_norm1_impl(norm_type* result, array& tmp) const override; + + device_view get_local_device_view_impl() override; + + const_device_view get_const_local_device_view_impl() const override; private: local_vector_type local_; diff --git a/test/mpi/distributed/vector.cpp b/test/mpi/distributed/vector.cpp index 1380ec4464f..bdc13bee80c 100644 --- a/test/mpi/distributed/vector.cpp +++ b/test/mpi/distributed/vector.cpp @@ -800,38 +800,6 @@ class VectorLocalOps : public CommonMpiTestFixture { TYPED_TEST_SUITE(VectorLocalOps, gko::test::ValueTypes, TypenameNameGenerator); -TYPED_TEST(VectorLocalOps, ApplyNotSupported) -{ - using dist_vec_type = typename TestFixture::dist_vec_type; - auto a = dist_vec_type::create(this->exec, this->comm, gko::dim<2>{2, 2}, - gko::dim<2>{2, 2}); - auto b = dist_vec_type::create(this->exec, this->comm, gko::dim<2>{2, 2}, - gko::dim<2>{2, 2}); - auto c = dist_vec_type::create(this->exec, this->comm, gko::dim<2>{2, 2}, - gko::dim<2>{2, 2}); - - ASSERT_THROW(a->apply(b, c), gko::NotSupported); -} - - -TYPED_TEST(VectorLocalOps, AdvancedApplyNotSupported) -{ - using dist_vec_type = typename TestFixture::dist_vec_type; - auto a = dist_vec_type::create(this->exec, this->comm, gko::dim<2>{2, 2}, - gko::dim<2>{2, 2}); - auto b = dist_vec_type::create(this->exec, this->comm, gko::dim<2>{1, 1}, - gko::dim<2>{1, 1}); - auto c = dist_vec_type::create(this->exec, this->comm, gko::dim<2>{2, 2}, - gko::dim<2>{2, 2}); - auto d = dist_vec_type::create(this->exec, this->comm, gko::dim<2>{1, 1}, - gko::dim<2>{1, 1}); - auto e = dist_vec_type::create(this->exec, this->comm, gko::dim<2>{2, 2}, - gko::dim<2>{2, 2}); - - ASSERT_THROW(a->apply(b, c, d, e), gko::NotSupported); -} - - TYPED_TEST(VectorLocalOps, ConvertsToPrecision) { using T = typename TestFixture::value_type;