From 2aaee636531faef8a02f0011f451fe49766edbcd Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 15:12:17 +0200 Subject: [PATCH 01/21] change ScaledIdentityAddable to multivector --- core/base/lin_op.cpp | 14 +++++++++++++- core/matrix/csr.cpp | 11 +++++++---- core/matrix/dense.cpp | 28 ++++++++++++++-------------- include/ginkgo/core/base/lin_op.hpp | 21 +++++++++++---------- include/ginkgo/core/matrix/csr.hpp | 3 ++- include/ginkgo/core/matrix/dense.hpp | 3 ++- 6 files changed, 49 insertions(+), 31 deletions(-) diff --git a/core/base/lin_op.cpp b/core/base/lin_op.cpp index 3db49ce32d1..2accfde94ce 100644 --- a/core/base/lin_op.cpp +++ b/core/base/lin_op.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: BSD-3-Clause #include +#include namespace gko { @@ -23,7 +24,18 @@ LinOp::LinOp(std::shared_ptr exec, const dim<2>& size, precision LinOp::get_precision() const noexcept { return value_t_; } -void LinOp::set_precision(precision p) noexcept { value_t_ = p; } +void ScaledIdentityAddable::add_scaled_identity( + ptr_param a, + ptr_param b) +{ + GKO_ASSERT_IS_SCALAR(a); + GKO_ASSERT_IS_SCALAR(b); + auto ae = + make_temporary_clone(as(this)->get_executor(), a); + auto be = + make_temporary_clone(as(this)->get_executor(), b); + add_scaled_identity_impl(ae.get(), be.get()); +} } // namespace gko diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp index a40b8340de2..f44f43305c4 100644 --- a/core/matrix/csr.cpp +++ b/core/matrix/csr.cpp @@ -38,6 +38,7 @@ #include "core/matrix/hybrid_kernels.hpp" #include "core/matrix/permutation.hpp" #include "core/matrix/sellp_kernels.hpp" +#include "ginkgo/core/base/multivector.hpp" namespace gko { @@ -1811,8 +1812,8 @@ void Csr::inv_scale_impl(const LinOp* alpha) template -void Csr::add_scaled_identity_impl(const LinOp* a, - const LinOp* b) +void Csr::add_scaled_identity_impl( + const AbstractMultiVector* a, const AbstractMultiVector* b) { bool has_diags{false}; this->get_executor()->run( @@ -1822,8 +1823,10 @@ void Csr::add_scaled_identity_impl(const LinOp* a, "The matrix has one or more structurally zero diagonal entries!"); } this->get_executor()->run(csr::make_add_scaled_identity( - make_temporary_conversion(a)->get_const_device_view(), - make_temporary_conversion(b)->get_const_device_view(), + a->as_precision(this->get_precision()) + ->template get_const_local_device_view(), + b->as_precision(this->get_precision()) + ->template get_const_local_device_view(), this)); } diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp index bcb3bdbeecf..9a64a46c469 100644 --- a/core/matrix/dense.cpp +++ b/core/matrix/dense.cpp @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: BSD-3-Clause +#include #include #include #include @@ -540,20 +541,6 @@ void Dense::convert_impl( } -template -void Dense::add_scaled_identity_impl(const LinOp* a, const LinOp* b) -{ - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_beta, auto dense_x) { - this->get_executor()->run(dense::make_add_scaled_identity( - dense_alpha->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); - }, - a, b, this); -} - - template void Dense::convert_to(SparsityCsr* result) const { @@ -853,6 +840,19 @@ Dense::get_const_device_view() const } +template +void Dense::add_scaled_identity_impl(const AbstractMultiVector* a, + const AbstractMultiVector* b) +{ + this->get_executor()->run(dense::make_add_scaled_identity( + a->as_precision(this->get_precision()) + ->template get_const_local_device_view(), + b->as_precision(this->get_precision()) + ->template get_const_local_device_view(), + this->get_device_view())); +} + + template ValueType& Dense::at(size_type row, size_type col) { diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp index 232a930c200..258e33be184 100644 --- a/include/ginkgo/core/base/lin_op.hpp +++ b/include/ginkgo/core/base/lin_op.hpp @@ -26,6 +26,11 @@ namespace gko { + + +class AbstractMultiVector; + + namespace matrix { @@ -779,6 +784,8 @@ class EnableAbsoluteComputation : public AbsoluteComputable { */ class ScaledIdentityAddable { public: + virtual ~ScaledIdentityAddable() = default; + /** * Scales this and adds another scalar times the identity to it. * @@ -786,18 +793,12 @@ class ScaledIdentityAddable { * @param b Scalar to multiply this before adding the scaled identity to * it. */ - void add_scaled_identity(ptr_param const a, - ptr_param const b) - { - GKO_ASSERT_IS_SCALAR(a); - GKO_ASSERT_IS_SCALAR(b); - auto ae = make_temporary_clone(as(this)->get_executor(), a); - auto be = make_temporary_clone(as(this)->get_executor(), b); - add_scaled_identity_impl(ae.get(), be.get()); - } + void add_scaled_identity(ptr_param a, + ptr_param b); private: - virtual void add_scaled_identity_impl(const LinOp* a, const LinOp* b) = 0; + virtual void add_scaled_identity_impl(const AbstractMultiVector* a, + const AbstractMultiVector* b) = 0; }; diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp index db175a401f3..723b27aaed9 100644 --- a/include/ginkgo/core/matrix/csr.hpp +++ b/include/ginkgo/core/matrix/csr.hpp @@ -1181,7 +1181,8 @@ class Csr : public LinOp, array srow_; index_type max_nnz_per_row_; - void add_scaled_identity_impl(const LinOp* a, const LinOp* b) override; + void add_scaled_identity_impl(const AbstractMultiVector* a, + const AbstractMultiVector* b) override; }; diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp index 43e8f4b7646..6c548898fb7 100644 --- a/include/ginkgo/core/matrix/dense.hpp +++ b/include/ginkgo/core/matrix/dense.hpp @@ -387,7 +387,8 @@ class Dense : public LinOp, size_type stride_; array values_; - void add_scaled_identity_impl(const LinOp* a, const LinOp* b) override; + void add_scaled_identity_impl(const AbstractMultiVector* a, + const AbstractMultiVector* b) override; }; From 2b2f6853366ba1aff556c838ebd9570ac916c743 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Thu, 16 Jul 2026 17:56:03 +0200 Subject: [PATCH 02/21] change linop::apply signature --- core/base/lin_op.cpp | 78 ++++++++++++++++++++++++++- include/ginkgo/core/base/lin_op.hpp | 81 +++++++++-------------------- 2 files changed, 103 insertions(+), 56 deletions(-) diff --git a/core/base/lin_op.cpp b/core/base/lin_op.cpp index 2accfde94ce..167ca645862 100644 --- a/core/base/lin_op.cpp +++ b/core/base/lin_op.cpp @@ -8,6 +8,51 @@ namespace gko { +void LinOp::apply(ptr_param b, + ptr_param x) const +{ + this->template log(this, b.get(), + x.get()); + this->validate_application_parameters(b.get(), x.get()); + auto exec = this->get_executor(); + this->apply_impl(make_temporary_clone(exec, b).get(), + make_temporary_clone(exec, x).get()); + this->template log(this, b.get(), + x.get()); +} + + +void LinOp::apply(ptr_param alpha, + ptr_param b, + ptr_param beta, + ptr_param x) const + +{ + this->template log( + this, alpha.get(), b.get(), beta.get(), x.get()); + this->validate_application_parameters(alpha.get(), b.get(), beta.get(), + x.get()); + auto exec = this->get_executor(); + this->apply_impl(make_temporary_clone(exec, alpha).get(), + make_temporary_clone(exec, b).get(), + make_temporary_clone(exec, beta).get(), + make_temporary_clone(exec, x).get()); + this->template log( + this, alpha.get(), b.get(), beta.get(), x.get()); +} + + +LinOp& LinOp::operator=(LinOp&& other) +{ + if (this != &other) { + PolymorphicObject::operator=(std::move(other)); + this->set_size(other.get_size()); + other.set_size({}); + } + return *this; +} + + LinOp::LinOp(LinOp&& other) : PolymorphicObject(std::move(other)), size_{std::exchange(other.size_, dim<2>{})}, @@ -21,7 +66,38 @@ LinOp::LinOp(std::shared_ptr exec, const dim<2>& size, {} -precision LinOp::get_precision() const noexcept { return value_t_; } +void LinOp::set_size(const dim<2>& value) noexcept { size_ = value; } + + +void LinOp::validate_application_parameters(const AbstractMultiVector* b, + const AbstractMultiVector* x) const + +{ + GKO_ASSERT_CONFORMANT(this, b); + GKO_ASSERT_EQUAL_ROWS(this, x); + GKO_ASSERT_EQUAL_COLS(b, x); +} + + +void LinOp::validate_application_parameters(const LinOp* b, + const LinOp* x) const +{ + GKO_ASSERT_CONFORMANT(this, b); + GKO_ASSERT_EQUAL_ROWS(this, x); + GKO_ASSERT_EQUAL_COLS(b, x); +} + + +void LinOp::validate_application_parameters(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + const AbstractMultiVector* x) const + +{ + this->validate_application_parameters(b, x); + GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1)); + GKO_ASSERT_EQUAL_DIMENSIONS(beta, dim<2>(1, 1)); +} void ScaledIdentityAddable::add_scaled_identity( diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp index 258e33be184..771268a30a5 100644 --- a/include/ginkgo/core/base/lin_op.hpp +++ b/include/ginkgo/core/base/lin_op.hpp @@ -131,18 +131,8 @@ class LinOp : public PolymorphicObject { * @param b the input vector(s) on which the operator is applied * @param x the output vector(s) where the result is stored */ - void apply(ptr_param b, ptr_param x) const - { - this->template log(this, b.get(), - x.get()); - this->validate_application_parameters(b.get(), x.get()); - auto exec = this->get_executor(); - this->apply_impl(make_temporary_clone(exec, b).get(), - make_temporary_clone(exec, x).get()); - this->template log(this, b.get(), - x.get()); - } - + void apply(ptr_param b, + ptr_param x) const; /** * Performs the operation x = alpha * op(b) + beta * x. @@ -152,21 +142,10 @@ class LinOp : public PolymorphicObject { * @param beta scaling of the input x * @param x output vector(s) */ - void apply(ptr_param alpha, ptr_param b, - ptr_param beta, ptr_param x) const - { - this->template log( - this, alpha.get(), b.get(), beta.get(), x.get()); - this->validate_application_parameters(alpha.get(), b.get(), beta.get(), - x.get()); - auto exec = this->get_executor(); - this->apply_impl(make_temporary_clone(exec, alpha).get(), - make_temporary_clone(exec, b).get(), - make_temporary_clone(exec, beta).get(), - make_temporary_clone(exec, x).get()); - this->template log( - this, alpha.get(), b.get(), beta.get(), x.get()); - } + void apply(ptr_param alpha, + ptr_param b, + ptr_param beta, + ptr_param x) const; /** * Returns the size of the operator. @@ -184,7 +163,7 @@ class LinOp : public PolymorphicObject { */ virtual bool apply_uses_initial_guess() const { return false; } - [[nodiscard]] precision get_precision() const noexcept; + [[nodiscard]] precision get_precision() const noexcept { return value_t_; } /** Copy-assigns a LinOp. Preserves the executor and copies the size. */ LinOp& operator=(const LinOp&) = default; @@ -194,15 +173,7 @@ class LinOp : public PolymorphicObject { * The moved-from object has size 0x0 afterwards, but its executor is * unchanged. */ - LinOp& operator=(LinOp&& other) - { - if (this != &other) { - PolymorphicObject::operator=(std::move(other)); - this->set_size(other.get_size()); - other.set_size({}); - } - return *this; - } + LinOp& operator=(LinOp&& other); /** Copy-constructs a LinOp. Inherits executor and size from the input. */ LinOp(const LinOp&) = default; @@ -229,7 +200,7 @@ class LinOp : public PolymorphicObject { * * @param value the new size of the operator */ - void set_size(const dim<2>& value) noexcept { size_ = value; } + void set_size(const dim<2>& value) noexcept; void set_precision(precision p) noexcept; @@ -242,7 +213,8 @@ class LinOp : public PolymorphicObject { * @param b the input vector(s) on which the operator is applied * @param x the output vector(s) where the result is stored */ - virtual void apply_impl(const LinOp* b, LinOp* x) const = 0; + virtual void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const = 0; /** * Implementers of LinOp should override this function instead @@ -253,8 +225,10 @@ class LinOp : public PolymorphicObject { * @param beta scaling of the input x * @param x output vector(s) */ - virtual void apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const = 0; + virtual void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const = 0; /** * Throws a DimensionMismatch exception if the parameters to `apply` are of @@ -263,12 +237,13 @@ class LinOp : public PolymorphicObject { * @param b vector(s) on which the operator is applied * @param x output vector(s) */ - void validate_application_parameters(const LinOp* b, const LinOp* x) const - { - GKO_ASSERT_CONFORMANT(this, b); - GKO_ASSERT_EQUAL_ROWS(this, x); - GKO_ASSERT_EQUAL_COLS(b, x); - } + void validate_application_parameters(const AbstractMultiVector* b, + const AbstractMultiVector* x) const; + + /** + * @copydoc validate_application_parameters + */ + void validate_application_parameters(const LinOp* b, const LinOp* x) const; /** * Throws a DimensionMismatch exception if the parameters to `apply` are of @@ -279,14 +254,10 @@ class LinOp : public PolymorphicObject { * @param beta scaling of the input x * @param x output vector(s) */ - void validate_application_parameters(const LinOp* alpha, const LinOp* b, - const LinOp* beta, - const LinOp* x) const - { - this->validate_application_parameters(b, x); - GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1)); - GKO_ASSERT_EQUAL_DIMENSIONS(beta, dim<2>(1, 1)); - } + void validate_application_parameters(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + const AbstractMultiVector* x) const; private: dim<2> size_{}; From 03ff0b8fdbd9556f1ac7575692c4ec730377a149 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Fri, 7 Aug 2026 17:04:29 +0200 Subject: [PATCH 03/21] deprecate apply using Dense --- core/test/base/lin_op.cpp | 29 +++++++++++ include/ginkgo/core/base/lin_op.hpp | 75 +++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp index d7eef135c2c..c933f25c768 100644 --- a/core/test/base/lin_op.cpp +++ b/core/test/base/lin_op.cpp @@ -12,6 +12,7 @@ #include #include +#include #include "core/test/utils.hpp" @@ -320,6 +321,34 @@ TEST_F(LinOpApply, AdvancedIsLogged) } +GKO_BEGIN_DISABLE_DEPRECATION_WARNINGS + + +TEST_F(LinOpApply, SimpleApplyToDense) +{ + auto op = gko::matrix::Dense<>::create(this->ref, gko::dim<2>{3, 5}); + auto b = gko::matrix::Dense<>::create(this->ref, gko::dim<2>{5, 1}); + auto x = gko::matrix::Dense<>::create(this->ref, gko::dim<2>{3, 1}); + + EXPECT_NO_THROW(op->apply(b, x)); +} + + +TEST_F(LinOpApply, AdvancedApplyToDense) +{ + auto op = gko::matrix::Dense<>::create(this->ref, gko::dim<2>{3, 5}); + auto alpha = gko::matrix::Dense<>::create(this->ref, gko::dim<2>{1, 1}); + auto beta = gko::matrix::Dense<>::create(this->ref, gko::dim<2>{1, 1}); + auto b = gko::matrix::Dense<>::create(this->ref, gko::dim<2>{5, 1}); + auto x = gko::matrix::Dense<>::create(this->ref, gko::dim<2>{3, 1}); + + EXPECT_NO_THROW(op->apply(alpha, b, beta, x)); +} + + +GKO_END_DISABLE_DEPRECATION_WARNINGS + + template class DummyLinOpWithFactory : public gko::LinOp { public: diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp index 771268a30a5..d57e557555f 100644 --- a/include/ginkgo/core/base/lin_op.hpp +++ b/include/ginkgo/core/base/lin_op.hpp @@ -37,10 +37,55 @@ namespace matrix { template class Diagonal; +template +class Dense; + } // namespace matrix +namespace detail { + + +template +struct is_dense_ptr : std::false_type {}; + +template +struct is_dense_ptr : is_dense_ptr> {}; + +template +struct is_dense_ptr : is_dense_ptr> {}; + +template +struct is_dense_ptr*> : std::true_type {}; + +template +struct is_dense_ptr*> : std::true_type {}; + +template +struct is_dense_ptr>> + : std::true_type {}; + +template +struct is_dense_ptr>> + : std::true_type {}; + +template +struct is_dense_ptr>> + : std::true_type {}; + +template +struct is_dense_ptr>> + : std::true_type {}; + + +} // namespace detail + + +template +constexpr bool is_dense_ptr = detail::is_dense_ptr::value; + + /** * @addtogroup LinOp * @@ -147,6 +192,36 @@ class LinOp : public PolymorphicObject { ptr_param beta, ptr_param x) const; + template && + is_dense_ptr>> + [[deprecated( + "Use apply(ptr_param b, " + "ptr_param x) by storing vectors as " + "matrix::MultiVector")]] void + apply(const DenseIn& b, DenseOut&& x) const + { + apply(b->as_const_multivector_view(), x->as_multivector_view()); + } + + template && is_dense_ptr && + is_dense_ptr && is_dense_ptr>> + [[deprecated( + "Use apply(ptr_param alpha, ptr_param b, ptr_param beta, " + "ptr_param x) by storing vectors as " + "matrix::MultiVector")]] void + apply(const DenseAlpha& alpha, const DenseIn& b, const DenseBeta& beta, + DenseOut&& x) const + { + apply(alpha->as_const_multivector_view(), + b->as_const_multivector_view(), beta->as_multivector_view(), + x->as_multivector_view()); + } + /** * Returns the size of the operator. * From 400be05a78db5507630095f09a2179e77fd826ad Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:35:38 +0200 Subject: [PATCH 04/21] add default implementation for advanced apply --- core/base/lin_op.cpp | 22 ++++++++++++++++++++++ include/ginkgo/core/base/lin_op.hpp | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/base/lin_op.cpp b/core/base/lin_op.cpp index 167ca645862..232ecfc8386 100644 --- a/core/base/lin_op.cpp +++ b/core/base/lin_op.cpp @@ -4,6 +4,7 @@ #include #include +#include namespace gko { @@ -69,6 +70,27 @@ LinOp::LinOp(std::shared_ptr exec, const dim<2>& size, void LinOp::set_size(const dim<2>& value) noexcept { size_ = value; } +void LinOp::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const +{ + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + std::visit( + [&](auto p) { + using value_type = std::decay_t; + auto dense_alpha = alpha->as_precision(this); + auto dense_beta = beta->as_precision(this); + auto x_clone = converted_x->clone(); + this->apply_impl(converted_b.get(), x_clone.get()); + converted_x->scale(dense_beta.get()); + converted_x->add_scaled(dense_alpha.get(), x_clone); + }, + precision_to_variant(this->get_precision())); +} + + void LinOp::validate_application_parameters(const AbstractMultiVector* b, const AbstractMultiVector* x) const diff --git a/include/ginkgo/core/base/lin_op.hpp b/include/ginkgo/core/base/lin_op.hpp index d57e557555f..4a6283874c7 100644 --- a/include/ginkgo/core/base/lin_op.hpp +++ b/include/ginkgo/core/base/lin_op.hpp @@ -303,7 +303,7 @@ class LinOp : public PolymorphicObject { virtual void apply_impl(const AbstractMultiVector* alpha, const AbstractMultiVector* b, const AbstractMultiVector* beta, - AbstractMultiVector* x) const = 0; + AbstractMultiVector* x) const; /** * Throws a DimensionMismatch exception if the parameters to `apply` are of From 8cbb4edadc989f44825712e5b0cb557b20f62185 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 15:17:45 +0200 Subject: [PATCH 05/21] change logger apply event --- include/ginkgo/core/log/logger.hpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp index 623eb9a6d11..9e8be442811 100644 --- a/include/ginkgo/core/log/logger.hpp +++ b/include/ginkgo/core/log/logger.hpp @@ -24,6 +24,7 @@ class array; class Executor; class LinOp; class LinOpFactory; +class AbstractMultiVector; class PolymorphicObject; class Operation; class stopping_status; @@ -274,7 +275,8 @@ public: \ * @param x the output vector(s) */ GKO_LOGGER_REGISTER_EVENT(13, linop_apply_started, const LinOp* A, - const LinOp* b, const LinOp* x) + const AbstractMultiVector* b, + const AbstractMultiVector* x) /** * LinOp's apply completed event. @@ -284,7 +286,8 @@ public: \ * @param x the output vector(s) */ GKO_LOGGER_REGISTER_EVENT(14, linop_apply_completed, const LinOp* A, - const LinOp* b, const LinOp* x) + const AbstractMultiVector* b, + const AbstractMultiVector* x) /** * LinOp's advanced apply started event. @@ -296,8 +299,10 @@ public: \ * @param x the output vector(s) */ GKO_LOGGER_REGISTER_EVENT(15, linop_advanced_apply_started, const LinOp* A, - const LinOp* alpha, const LinOp* b, - const LinOp* beta, const LinOp* x) + const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + const AbstractMultiVector* x) /** * LinOp's advanced apply completed event. @@ -309,8 +314,10 @@ public: \ * @param x the output vector(s) */ GKO_LOGGER_REGISTER_EVENT(16, linop_advanced_apply_completed, - const LinOp* A, const LinOp* alpha, - const LinOp* b, const LinOp* beta, const LinOp* x) + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + const AbstractMultiVector* x) /** * LinOp Factory's generate started event. From 3c77adda358f28b6f73c6272f6b9a6adcfda46a1 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Wed, 1 Apr 2026 16:55:20 +0200 Subject: [PATCH 06/21] add dispatch helper for apply functions --- core/base/dispatch_helper.hpp | 189 ++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) diff --git a/core/base/dispatch_helper.hpp b/core/base/dispatch_helper.hpp index 748add2842c..d2e22811214 100644 --- a/core/base/dispatch_helper.hpp +++ b/core/base/dispatch_helper.hpp @@ -246,6 +246,195 @@ auto run(std::shared_ptr obj, Func&& f, Args&&... args) std::forward(args)...); } +/** + * Helper to dispatch vectors to the expected precision. + * Also handles complex->real conversion if necessary. + * + * @tparam ValueType Value type to convert the inputs to + * @tparam Fn Function type, has signature void(const AbstractMultiVector*, + * AbstractMultiVector*) + * + * @param fn Function to apply to the converted inputs + * @param b Input vector + * @param x Output vector + */ +template +void precision_dispatch(Fn&& fn, const AbstractMultiVector* b, + AbstractMultiVector* x) +{ + auto p = precision_v; + if constexpr (!is_complex()) { + fn(b->create_real_view()->as_precision(p).get(), + x->create_real_view()->as_precision(p).get()); + } else { + fn(b->as_precision(p).get(), x->as_precision(p).get()); + } +} + + +/** + * Specialization for precision_dispatch for operator apply. + * + * Note: the function needs to have the following signature: + * fn(device_view_type, device_view_type) + */ +template +void apply_precision_dispatch(Fn&& fn, const AbstractMultiVector* b, + AbstractMultiVector* x) +{ + precision_dispatch( + [&fn](auto b_, auto x_) { + fn(b_->template get_const_local_device_view(), + x_->template get_local_device_view()); + }, + b, x); +} + + +/** + * Same as apply_dispatch(Fn, const AbstractMultiVector*, AbstractMultiVector*), + * except for the additional alpha and beta scalars. + * + * Note: the function needs to have the following signature: + * fn(MultiVector*, device_view_type, + * MultiVector*, device_view_type) + */ +template +void apply_precision_dispatch(Fn&& fn, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) +{ + auto p = precision_v; + auto dense_alpha = + as>(alpha->as_precision(p)); + auto dense_beta = as>(beta->as_precision(p)); + precision_dispatch( + [&fn, &dense_alpha, &dense_beta](auto b_, auto x_) { + fn(dense_alpha.get(), + b_->template get_const_local_device_view(), + dense_beta.get(), + x_->template get_local_device_view()); + }, + b, x); +} + + +/** + * Helper function for mixed precision dispatch. + * Falls back to apply_dispatch if GINKGO_MIXED_PRECISION is not defined. + * + * The input vectors will be _not_ be converted to the precision of the + * operator. Instead, the underlying precision of each vector will be used. + * Exception: If the operator is complex, the vectors will be converted to their + * corresponding real precision. + * + * @tparam ValueType Value type to ensure compatibility with + * @tparam Fn Function type, has signature + * void(const AbstractMultiVector* b, AbstractMultiVector* x, + * ValueTypeIn, ValueTypeOut) + * + * @param fn Function to apply to the inputs + * @param b Input vector + * @param x Output vector + */ +template +void mixed_precision_dispatch(Fn&& fn, const AbstractMultiVector* b, + AbstractMultiVector* x) +{ +#ifdef GINKGO_MIXED_PRECISION + auto precision_b = precision_to_variant(b->get_precision()); + auto precision_x = precision_to_variant(x->get_precision()); + std::visit( + [&fn, b, x](auto p_b, auto p_x) { + using fst_value_type = std::decay_t; + using snd_value_type = std::decay_t; + if constexpr (is_complex() == + is_complex() && + is_complex() == + is_complex()) { + // Either all precisions are real or all precisions are complex + fn(b, x, p_b, p_x); + } else if constexpr (!is_complex() && + is_complex() && + is_complex()) { + // ValueType is real and both other precisions are complex + fn(b->create_real_view().get(), x->create_real_view().get(), + remove_complex(), + remove_complex()); + } else { + // real ValueType and one real and one complex precision are not + // supported + GKO_NOT_IMPLEMENTED; + } + }, + precision_b, precision_x); +#else + precision_dispatch( + [&fn](auto b_, auto x_, auto...) { + fn(b_, x_, ValueType(), ValueType()); + }, + b, x); +#endif +} + +/** + * Specialization for mixed_precision_dispatch for operator apply. + * + * Note: the function needs to have the following signature: + * fn(device_view_type, device_view_type, + * ValueTypeIn, ValueTypeOut) + */ +template +void apply_mixed_precision_dispatch(Fn&& fn, const AbstractMultiVector* b, + AbstractMultiVector* x) +{ + mixed_precision_dispatch( + [&fn](auto b_, auto x_, auto p_b, auto p_x) { + using fst_value_type = std::decay_t; + using snd_value_type = std::decay_t; + fn(b_->template get_const_local_device_view(), + x_->template get_local_device_view(), p_b, p_x); + }, + b, x); +} + + +/** + * Same as mixed_precision_apply_dispatch(Fn, const AbstractMultiVector*, + * AbstractMultiVector*), except for the additional alpha and beta scalars. + * + * @note the function needs to have the following signature: + * fn(MultiVector, device_view_type, + * MultiVector, device_view_type, + * ValueTypeIn, ValueTypeOut) + * + * @param alpha input scalar converted to precision ValueType if necessary + * @param beta input scalar converted to precision of x if necessary + */ +template +void apply_mixed_precision_dispatch(Fn&& fn, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) +{ + auto dense_alpha = as>( + alpha->as_precision(precision_v)); + + mixed_precision_dispatch( + [&fn, &dense_alpha, beta](auto b_, auto x_, auto p_b, auto p_x) { + using fst_value_type = std::decay_t; + using snd_value_type = std::decay_t; + auto dense_beta = as>( + beta->as_precision(precision_v)); + fn(dense_alpha.get(), + b_->template get_const_local_device_view(), + dense_beta.get(), + x_->template get_local_device_view(), p_b, p_x); + }, + b, x); +} + } // namespace gko From e4c652df3fde90402caa883744da33eaf80d3dc1 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 16:58:00 +0200 Subject: [PATCH 07/21] fix hierarchy change for loggers --- core/log/convergence.cpp | 149 ++++++++++---- core/log/papi.cpp | 56 +++-- core/log/profiler_hook.cpp | 78 +++---- core/log/record.cpp | 233 +++++++++++++++++---- core/log/solver_progress.cpp | 122 ++++++----- core/log/stream.cpp | 67 +++--- include/ginkgo/core/log/convergence.hpp | 75 +++---- include/ginkgo/core/log/logger.hpp | 46 +++-- include/ginkgo/core/log/papi.hpp | 52 ++--- include/ginkgo/core/log/profiler_hook.hpp | 71 ++++--- include/ginkgo/core/log/record.hpp | 239 ++++++++-------------- include/ginkgo/core/log/stream.hpp | 63 +++--- 12 files changed, 724 insertions(+), 527 deletions(-) diff --git a/core/log/convergence.cpp b/core/log/convergence.cpp index 9931e4a1236..4cb42fb2af3 100644 --- a/core/log/convergence.cpp +++ b/core/log/convergence.cpp @@ -22,11 +22,12 @@ namespace log { template void Convergence::on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* implicit_sq_resnorm, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized, - const array* status, const bool& one_changed, - const bool& stopped) const + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_resnorm, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized, const array* status, + const bool& one_changed, const bool& stopped) const { this->on_iteration_complete(nullptr, nullptr, solution, num_iterations, residual, residual_norm, implicit_sq_resnorm, @@ -37,10 +38,11 @@ void Convergence::on_criterion_check_completed( template void Convergence::on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized, - const array* status, const bool& one_changed, - const bool& stopped) const + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized, const array* status, + const bool& one_changed, const bool& stopped) const { this->on_criterion_check_completed( criterion, num_iterations, residual, residual_norm, nullptr, solution, @@ -50,9 +52,11 @@ void Convergence::on_criterion_check_completed( template void Convergence::on_iteration_complete( - const LinOp* solver, const LinOp* b, const LinOp* x, - const size_type& num_iterations, const LinOp* residual, - const LinOp* residual_norm, const LinOp* implicit_resnorm_sq, + const LinOp* solver, const AbstractMultiVector* b, + const AbstractMultiVector* x, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_resnorm_sq, const array* status, const bool stopped) const { if (stopped) { @@ -67,26 +71,18 @@ void Convergence::on_iteration_complete( } this->num_iterations_ = num_iterations; if (residual != nullptr) { - this->residual_.reset( - as(as(residual)->clone()).release()); + this->residual_ = residual->clone(); } if (implicit_resnorm_sq != nullptr) { - this->implicit_sq_resnorm_.reset( - as(as(implicit_resnorm_sq)->clone()) - .release()); + this->implicit_sq_resnorm_ = implicit_resnorm_sq->clone(); } if (residual_norm != nullptr) { - this->residual_norm_.reset( - as(as(residual_norm)->clone()).release()); + this->residual_norm_ = residual_norm->clone(); } else if (residual != nullptr) { using NormVector = matrix::MultiVector>; - detail::vector_dispatch( - residual, [&](const auto* dense_r) { - this->residual_norm_ = - NormVector::create(residual->get_executor(), - dim<2>{1, residual->get_size()[1]}); - dense_r->compute_norm2(this->residual_norm_); - }); + this->residual_norm_ = NormVector::create( + residual->get_executor(), dim<2>{1, residual->get_size()[1]}); + residual->compute_norm2(this->residual_norm_); } else if (dynamic_cast( solver) && b != nullptr && x != nullptr) { @@ -95,23 +91,100 @@ void Convergence::on_iteration_complete( ->get_system_matrix(); using Vector = matrix::MultiVector; using NormVector = matrix::MultiVector>; - detail::vector_dispatch(b, [&](const auto* dense_b) { - detail::vector_dispatch(x, [&](const auto* dense_x) { - auto exec = system_mtx->get_executor(); - auto residual = dense_b->clone(); - this->residual_norm_ = NormVector::create( - exec, dim<2>{1, residual->get_size()[1]}); - system_mtx->apply(initialize({-1.0}, exec), dense_x, - initialize({1.0}, exec), - residual); - residual->compute_norm2(this->residual_norm_); - }); - }); + auto converted_b = b->as_precision(precision_v); + auto exec = system_mtx->get_executor(); + auto residual_tmp = converted_b->clone(); + this->residual_norm_ = NormVector::create( + exec, dim<2>{1, residual_tmp->get_size()[1]}); + system_mtx->apply(initialize({-1.0}, exec), + x->as_precision(precision_v).get(), + initialize({1.0}, exec), residual_tmp); + residual_tmp->compute_norm2(this->residual_norm_); } } } +template +std::unique_ptr> Convergence::create( + std::shared_ptr, const mask_type& enabled_events) + +{ + return std::unique_ptr(new Convergence(enabled_events)); +} + + +template +std::unique_ptr> Convergence::create( + const mask_type& enabled_events) + +{ + return std::unique_ptr(new Convergence(enabled_events)); +} + + +template +bool Convergence::has_converged() const noexcept +{ + return convergence_status_; +} + + +template +void Convergence::reset_convergence_status() +{ + this->convergence_status_ = false; +} + + +template +const size_type& Convergence::get_num_iterations() const noexcept + +{ + return num_iterations_; +} + + +template +const AbstractMultiVector* Convergence::get_residual() const noexcept +{ + return residual_.get(); +} + + +template +const AbstractMultiVector* Convergence::get_residual_norm() + const noexcept + +{ + return residual_norm_.get(); +} + + +template +const AbstractMultiVector* Convergence::get_implicit_sq_resnorm() + const noexcept + +{ + return implicit_sq_resnorm_.get(); +} + + +template +Convergence::Convergence(std::shared_ptr, + const mask_type& enabled_events) + + : Logger(enabled_events) +{} + + +template +Convergence::Convergence(const mask_type& enabled_events) + + : Logger(enabled_events) +{} + + #define GKO_DECLARE_CONVERGENCE(ValueType) class Convergence GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_CONVERGENCE); diff --git a/core/log/papi.cpp b/core/log/papi.cpp index 0ed5d9a981c..66237ca5fdd 100644 --- a/core/log/papi.cpp +++ b/core/log/papi.cpp @@ -148,38 +148,36 @@ void Papi::on_polymorphic_object_deleted( template -void Papi::on_linop_apply_started(const LinOp* A, const LinOp* b, - const LinOp* x) const +void Papi::on_linop_apply_started(const LinOp* A, + const MultiVector* b, + const MultiVector* x) const { linop_apply_started.get_counter(A) += 1; } template -void Papi::on_linop_apply_completed(const LinOp* A, const LinOp* b, - const LinOp* x) const +void Papi::on_linop_apply_completed(const LinOp* A, + const MultiVector* b, + const MultiVector* x) const { linop_apply_completed.get_counter(A) += 1; } template -void Papi::on_linop_advanced_apply_started(const LinOp* A, - const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - const LinOp* x) const +void Papi::on_linop_advanced_apply_started( + const LinOp* A, const MultiVector* alpha, const MultiVector* b, + const MultiVector* beta, const MultiVector* x) const { linop_advanced_apply_started.get_counter(A) += 1; } template -void Papi::on_linop_advanced_apply_completed(const LinOp* A, - const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - const LinOp* x) const +void Papi::on_linop_advanced_apply_completed( + const LinOp* A, const MultiVector* alpha, const MultiVector* b, + const MultiVector* beta, const MultiVector* x) const { linop_advanced_apply_completed.get_counter(A) += 1; } @@ -204,10 +202,10 @@ void Papi::on_linop_factory_generate_completed( template void Papi::on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, const LinOp* solution, - const uint8& stoppingId, const bool& setFinalized, - const array* status, const bool& oneChanged, - const bool& converged) const + const MultiVector* residual, const MultiVector* residual_norm, + const MultiVector* solution, const uint8& stoppingId, + const bool& setFinalized, const array* status, + const bool& oneChanged, const bool& converged) const { using Vector = matrix::MultiVector; double residual_norm_d = 0.0; @@ -244,9 +242,9 @@ void Papi::on_criterion_check_completed( template void Papi::on_iteration_complete( - const LinOp* solver, const LinOp* b, const LinOp* solution, - const size_type& num_iterations, const LinOp* residual, - const LinOp* residual_norm, const LinOp* implicit_resnorm_sq, + const LinOp* solver, const MultiVector* b, const MultiVector* solution, + const size_type& num_iterations, const MultiVector* residual, + const MultiVector* residual_norm, const MultiVector* implicit_resnorm_sq, const array* status, bool stopped) const { iteration_complete.get_counter(solver) = num_iterations; @@ -254,11 +252,10 @@ void Papi::on_iteration_complete( template -void Papi::on_iteration_complete(const LinOp* solver, - const size_type& num_iterations, - const LinOp* residual, - const LinOp* solution, - const LinOp* residual_norm) const +void Papi::on_iteration_complete( + const LinOp* solver, const size_type& num_iterations, + const MultiVector* residual, const MultiVector* solution, + const MultiVector* residual_norm) const { this->on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, nullptr, nullptr, @@ -268,9 +265,10 @@ void Papi::on_iteration_complete(const LinOp* solver, template void Papi::on_iteration_complete( - const LinOp* solver, const size_type& num_iterations, const LinOp* residual, - const LinOp* solution, const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm) const + const LinOp* solver, const size_type& num_iterations, + const MultiVector* residual, const MultiVector* solution, + const MultiVector* residual_norm, + const MultiVector* implicit_sq_residual_norm) const { this->on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, diff --git a/core/log/profiler_hook.cpp b/core/log/profiler_hook.cpp index 1559d45266a..4dd8e0c63af 100644 --- a/core/log/profiler_hook.cpp +++ b/core/log/profiler_hook.cpp @@ -140,8 +140,9 @@ void ProfilerHook::on_polymorphic_object_move_completed( } -void ProfilerHook::on_linop_apply_started(const LinOp* A, const LinOp* b, - const LinOp* x) const +void ProfilerHook::on_linop_apply_started(const LinOp* A, + const AbstractMultiVector* b, + const AbstractMultiVector* x) const { std::stringstream ss; ss << "apply(" << stringify_object(A) << " * " << stringify_object(b) @@ -153,8 +154,9 @@ void ProfilerHook::on_linop_apply_started(const LinOp* A, const LinOp* b, } -void ProfilerHook::on_linop_apply_completed(const LinOp* A, const LinOp* b, - const LinOp* x) const +void ProfilerHook::on_linop_apply_completed(const LinOp* A, + const AbstractMultiVector* b, + const AbstractMultiVector* x) const { std::stringstream ss; ss << "apply(" << stringify_object(A) << " * " << stringify_object(b) @@ -166,11 +168,10 @@ void ProfilerHook::on_linop_apply_completed(const LinOp* A, const LinOp* b, } -void ProfilerHook::on_linop_advanced_apply_started(const LinOp* A, - const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - const LinOp* x) const +void ProfilerHook::on_linop_advanced_apply_started( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const { std::stringstream ss; ss << "advanced_apply(" << stringify_object(alpha) << " * " @@ -183,11 +184,10 @@ void ProfilerHook::on_linop_advanced_apply_started(const LinOp* A, } -void ProfilerHook::on_linop_advanced_apply_completed(const LinOp* A, - const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - const LinOp* x) const +void ProfilerHook::on_linop_advanced_apply_completed( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const { std::stringstream ss; ss << "advanced_apply(" << stringify_object(alpha) << " * " @@ -220,8 +220,10 @@ void ProfilerHook::on_linop_factory_generate_completed( void ProfilerHook::on_criterion_check_started( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized) const + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized) const { std::stringstream ss; ss << "check(" << stringify_object(criterion) << ")"; @@ -231,10 +233,11 @@ void ProfilerHook::on_criterion_check_started( void ProfilerHook::on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized, - const array* status, const bool& one_changed, - const bool& all_converged) const + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized, const array* status, + const bool& one_changed, const bool& all_converged) const { this->on_criterion_check_completed( criterion, num_iterations, residual, residual_norm, nullptr, solution, @@ -243,11 +246,12 @@ void ProfilerHook::on_criterion_check_completed( void ProfilerHook::on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* implicit_sq_resnorm, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized, - const array* status, const bool& one_changed, - const bool& all_stopped) const + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_resnorm, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized, const array* status, + const bool& one_changed, const bool& all_stopped) const { std::stringstream ss; ss << "check(" << stringify_object(criterion) << ")"; @@ -256,9 +260,11 @@ void ProfilerHook::on_criterion_check_completed( void ProfilerHook::on_iteration_complete( - const LinOp* solver, const LinOp* right_hand_side, const LinOp* solution, - const size_type& num_iterations, const LinOp* residual, - const LinOp* residual_norm, const LinOp* implicit_sq_residual_norm, + const LinOp* solver, const AbstractMultiVector* right_hand_side, + const AbstractMultiVector* solution, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm, const array* status, bool stopped) const { if (num_iterations > 0 && @@ -269,11 +275,10 @@ void ProfilerHook::on_iteration_complete( } -void ProfilerHook::on_iteration_complete(const LinOp* solver, - const size_type& num_iterations, - const LinOp* residual, - const LinOp* solution, - const LinOp* residual_norm) const +void ProfilerHook::on_iteration_complete( + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm) const { on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, nullptr, nullptr, false); @@ -281,9 +286,10 @@ void ProfilerHook::on_iteration_complete(const LinOp* solver, void ProfilerHook::on_iteration_complete( - const LinOp* solver, const size_type& num_iterations, const LinOp* residual, - const LinOp* solution, const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm) const + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm) const { on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, implicit_sq_residual_norm, nullptr, diff --git a/core/log/record.cpp b/core/log/record.cpp index 0d810c05fa0..808d805a603 100644 --- a/core/log/record.cpp +++ b/core/log/record.cpp @@ -1,10 +1,11 @@ -// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause #include "ginkgo/core/log/record.hpp" #include +#include #include #include @@ -13,6 +14,122 @@ namespace gko { namespace log { +template +std::unique_ptr clone_or_nullptr(T* input) +{ + // whether throw exception if input is not cloneable? + if (auto tmp = dynamic_cast(input)) { + return as(tmp->clone()); + } + return nullptr; +} + + +iteration_complete_data::iteration_complete_data( + const LinOp* solver, const AbstractMultiVector* right_hand_side, + const AbstractMultiVector* solution, const size_type num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm, + const gko::array* status, bool all_stopped) + + : num_iterations{num_iterations}, all_stopped(all_stopped) +{ + this->solver = clone_or_nullptr(solver); + this->solution = clone_or_nullptr(solution); + if (right_hand_side != nullptr) { + this->right_hand_side = clone_or_nullptr(right_hand_side); + } + if (residual != nullptr) { + this->residual = clone_or_nullptr(residual); + } + if (residual_norm != nullptr) { + this->residual_norm = clone_or_nullptr(residual_norm); + } + if (implicit_sq_residual_norm != nullptr) { + this->implicit_sq_residual_norm = + clone_or_nullptr(implicit_sq_residual_norm); + } + if (status != nullptr) { + this->status = *status; + } +} + + +polymorphic_object_data::polymorphic_object_data( + const Executor* exec, const PolymorphicObject* input, + const PolymorphicObject* output) + + : exec{exec} +{ + this->input = clone_or_nullptr(input); + if (output != nullptr) { + this->output = clone_or_nullptr(output); + } +} + + +linop_data::linop_data(const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + const AbstractMultiVector* x) + +{ + this->A = clone_or_nullptr(A); + if (alpha != nullptr) { + this->alpha = clone_or_nullptr(alpha); + } + this->b = clone_or_nullptr(b); + if (beta != nullptr) { + this->beta = clone_or_nullptr(beta); + } + this->x = clone_or_nullptr(x); +} + + +linop_factory_data::linop_factory_data(const LinOpFactory* factory, + const LinOp* input, const LinOp* output) + + : factory{factory} +{ + this->input = clone_or_nullptr(input); + if (output != nullptr) { + this->output = clone_or_nullptr(output); + } +} + + +criterion_data::criterion_data(const stop::Criterion* criterion, + const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, + const uint8 stopping_id, + const bool set_finalized, + const array* status, + const bool oneChanged, const bool converged) + + : criterion{criterion}, + num_iterations{num_iterations}, + residual{nullptr}, + residual_norm{nullptr}, + solution{nullptr}, + stopping_id{stopping_id}, + set_finalized{set_finalized}, + status{status}, + oneChanged{oneChanged}, + converged{converged} +{ + if (residual != nullptr) { + this->residual = residual->clone(); + } + if (residual_norm != nullptr) { + this->residual_norm = residual_norm->clone(); + } + if (solution != nullptr) { + this->solution = solution->clone(); + } +} void Record::on_allocation_started(const Executor* exec, const size_type& num_bytes) const { @@ -160,8 +277,9 @@ void Record::on_polymorphic_object_deleted(const Executor* exec, } -void Record::on_linop_apply_started(const LinOp* A, const LinOp* b, - const LinOp* x) const +void Record::on_linop_apply_started(const LinOp* A, + const AbstractMultiVector* b, + const AbstractMultiVector* x) const { append_deque(data_.linop_apply_started, (std::unique_ptr( @@ -169,8 +287,9 @@ void Record::on_linop_apply_started(const LinOp* A, const LinOp* b, } -void Record::on_linop_apply_completed(const LinOp* A, const LinOp* b, - const LinOp* x) const +void Record::on_linop_apply_completed(const LinOp* A, + const AbstractMultiVector* b, + const AbstractMultiVector* x) const { append_deque(data_.linop_apply_completed, (std::unique_ptr( @@ -178,9 +297,11 @@ void Record::on_linop_apply_completed(const LinOp* A, const LinOp* b, } -void Record::on_linop_advanced_apply_started(const LinOp* A, const LinOp* alpha, - const LinOp* b, const LinOp* beta, - const LinOp* x) const +void Record::on_linop_advanced_apply_started(const LinOp* A, + const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + const AbstractMultiVector* x) const { append_deque( data_.linop_advanced_apply_started, @@ -188,11 +309,10 @@ void Record::on_linop_advanced_apply_started(const LinOp* A, const LinOp* alpha, } -void Record::on_linop_advanced_apply_completed(const LinOp* A, - const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - const LinOp* x) const +void Record::on_linop_advanced_apply_completed( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const { append_deque( data_.linop_advanced_apply_completed, @@ -219,10 +339,47 @@ void Record::on_linop_factory_generate_completed(const LinOpFactory* factory, } +std::unique_ptr Record::create(std::shared_ptr exec, + const mask_type& enabled_events, + size_type max_storage) + +{ + return std::unique_ptr(new Record(enabled_events, max_storage)); +} + + +std::unique_ptr Record::create(const mask_type& enabled_events, + size_type max_storage) + +{ + return std::unique_ptr(new Record(enabled_events, max_storage)); +} + + +const Record::logged_data& Record::get() const noexcept { return data_; } + + +Record::logged_data& Record::get() noexcept { return data_; } + + +Record::Record(std::shared_ptr exec, + const mask_type& enabled_events, size_type max_storage) + + : Record(enabled_events, max_storage) +{} + + +Record::Record(const mask_type& enabled_events, size_type max_storage) + : Logger(enabled_events), max_storage_{max_storage} +{} + + void Record::on_criterion_check_started( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized) const + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized) const { append_deque(data_.criterion_check_started, (std::unique_ptr(new criterion_data{ @@ -233,11 +390,12 @@ void Record::on_criterion_check_started( void Record::on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* implicit_residual_norm_sq, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized, - const array* status, const bool& oneChanged, - const bool& converged) const + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_residual_norm_sq, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized, const array* status, + const bool& oneChanged, const bool& converged) const { append_deque( data_.criterion_check_completed, @@ -249,10 +407,11 @@ void Record::on_criterion_check_completed( void Record::on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized, - const array* status, const bool& oneChanged, - const bool& converged) const + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized, const array* status, + const bool& oneChanged, const bool& converged) const { this->on_criterion_check_completed( criterion, num_iterations, residual, residual_norm, nullptr, solution, @@ -260,10 +419,10 @@ void Record::on_criterion_check_completed( } -void Record::on_iteration_complete(const LinOp* solver, - const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm) const +void Record::on_iteration_complete( + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm) const { this->on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, nullptr, nullptr, @@ -271,11 +430,11 @@ void Record::on_iteration_complete(const LinOp* solver, } -void Record::on_iteration_complete(const LinOp* solver, - const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm) const +void Record::on_iteration_complete( + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm) const { this->on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, @@ -284,9 +443,11 @@ void Record::on_iteration_complete(const LinOp* solver, void Record::on_iteration_complete( - const LinOp* solver, const LinOp* right_hand_side, const LinOp* solution, - const size_type& num_iterations, const LinOp* residual, - const LinOp* residual_norm, const LinOp* implicit_resnorm_sq, + const LinOp* solver, const AbstractMultiVector* right_hand_side, + const AbstractMultiVector* solution, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_resnorm_sq, const array* status, bool stopped) const { append_deque( diff --git a/core/log/solver_progress.cpp b/core/log/solver_progress.cpp index acb391d659a..07b7bb3eca3 100644 --- a/core/log/solver_progress.cpp +++ b/core/log/solver_progress.cpp @@ -25,7 +25,7 @@ namespace log { namespace { -bool is_dense(const LinOp* value) +bool is_dense(const AbstractMultiVector* value) { using conv_to_double = ConvertibleTo>; using conv_to_complex = @@ -40,17 +40,19 @@ class SolverProgressPrint : public SolverProgress { public: /* Internal solver events */ - void on_linop_apply_started(const LinOp* solver, const LinOp* in, - const LinOp* out) const override + void on_linop_apply_started(const LinOp* solver, + const AbstractMultiVector* in, + const AbstractMultiVector* out) const override { printed_header_ = false; } void on_iteration_complete( - const LinOp* solver, const LinOp* right_hand_side, - const LinOp* solution, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm, + const LinOp* solver, const AbstractMultiVector* right_hand_side, + const AbstractMultiVector* solution, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm, const array* status, bool stopped) const override { using solver_base = solver::detail::SolverBaseLinOp; @@ -104,10 +106,11 @@ class SolverProgressPrint : public SolverProgress { GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") - void on_iteration_complete(const LinOp* solver, - const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm) const override + void on_iteration_complete( + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm) const override { on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, nullptr, nullptr, false); @@ -118,9 +121,10 @@ class SolverProgressPrint : public SolverProgress { "information.") void on_iteration_complete( const LinOp* solver, const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm) const override + const AbstractMultiVector* residual, + const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm) const override { on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, @@ -128,7 +132,8 @@ class SolverProgressPrint : public SolverProgress { } private: - void print_scalar(const LinOp* value, std::ostream& stream) const + void print_scalar(const AbstractMultiVector* value, + std::ostream& stream) const { if (separator_) { stream << separator_; @@ -177,8 +182,9 @@ class SolverProgressStore : public SolverProgress { public: /* Internal solver events */ - void on_linop_apply_started(const LinOp* solver, const LinOp* in, - const LinOp* out) const override + void on_linop_apply_started(const LinOp* solver, + const AbstractMultiVector* in, + const AbstractMultiVector* out) const override { using solver_base = solver::detail::SolverBaseLinOp; auto dynamic_type = name_demangling::get_dynamic_type(*solver); @@ -189,10 +195,11 @@ class SolverProgressStore : public SolverProgress { } void on_iteration_complete( - const LinOp* solver, const LinOp* right_hand_side, - const LinOp* solution, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm, + const LinOp* solver, const AbstractMultiVector* right_hand_side, + const AbstractMultiVector* solution, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm, const array* status, bool stopped) const override { using solver_base = solver::detail::SolverBaseLinOp; @@ -213,10 +220,11 @@ class SolverProgressStore : public SolverProgress { GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") - void on_iteration_complete(const LinOp* solver, - const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm) const override + void on_iteration_complete( + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm) const override { on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, nullptr, nullptr, false); @@ -227,9 +235,10 @@ class SolverProgressStore : public SolverProgress { "information.") void on_iteration_complete( const LinOp* solver, const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm) const override + const AbstractMultiVector* residual, + const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm) const override { on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, @@ -237,28 +246,36 @@ class SolverProgressStore : public SolverProgress { } private: - void store_vector(const LinOp* value, const std::string& name) const + template + void store_generic(const T* value, const std::string& name) const { const auto filename = output_file_prefix_ + "_" + name + (binary_ ? ".bin" : ".mtx"); if (!value) { return; } - // putting MultiVector first here causes gko::write to use dense output - run, gko::matrix::MultiVector, - gko::matrix::MultiVector>, - gko::matrix::MultiVector>, + run(value, [&](auto vector) { + std::ofstream output{ + filename, + binary_ ? (std::ios::out | std::ios::binary) : std::ios::out}; + if (binary_) { + gko::write_binary(output, vector); + } else { + gko::write(output, vector); + } + }); + } + + void store_vector(const LinOp* value, const std::string& name) const + { + store_generic< #if GINKGO_ENABLE_HALF - gko::matrix::MultiVector, - gko::matrix::MultiVector>, gko::WritableToMatrixData, gko::WritableToMatrixData, int32>, gko::WritableToMatrixData, gko::WritableToMatrixData, int64>, #endif #if GINKGO_ENABLE_BFLOAT16 - gko::matrix::MultiVector, - gko::matrix::MultiVector>, gko::WritableToMatrixData, gko::WritableToMatrixData, int32>, gko::WritableToMatrixData, @@ -272,20 +289,27 @@ class SolverProgressStore : public SolverProgress { gko::WritableToMatrixData, gko::WritableToMatrixData, gko::WritableToMatrixData, int64>, - gko::WritableToMatrixData, int64>>( - value, [&](auto vector) { - std::ofstream output{ - filename, binary_ ? (std::ios::out | std::ios::binary) - : std::ios::out}; - if (binary_) { - gko::write_binary(output, vector); - } else { - gko::write(output, vector); - } - }); + gko::WritableToMatrixData, int64>>(value, name); + } + + void store_vector(const AbstractMultiVector* value, + const std::string& name) const + { + store_generic< +#if GINKGO_ENABLE_HALF + gko::matrix::MultiVector, + gko::matrix::MultiVector>, +#endif +#if GINKGO_ENABLE_BFLOAT16 + gko::matrix::MultiVector, + gko::matrix::MultiVector>, +#endif + gko::matrix::MultiVector, gko::matrix::MultiVector, + gko::matrix::MultiVector>, + gko::matrix::MultiVector>>(value, name); } - void store_vector(const LinOp* value, size_type iteration, + void store_vector(const AbstractMultiVector* value, size_type iteration, const std::string& name) const { store_vector(value, std::to_string(iteration) + "_" + name); diff --git a/core/log/stream.cpp b/core/log/stream.cpp index 228c5ea720c..b27e7859b5f 100644 --- a/core/log/stream.cpp +++ b/core/log/stream.cpp @@ -254,8 +254,9 @@ void Stream::on_polymorphic_object_deleted( template -void Stream::on_linop_apply_started(const LinOp* A, const LinOp* b, - const LinOp* x) const +void Stream::on_linop_apply_started( + const LinOp* A, const AbstractMultiVector* b, + const AbstractMultiVector* x) const { *os_ << prefix_ << "apply started on A " << demangle_name(A) << " with b " << demangle_name(b) << " and x " << demangle_name(x) << std::endl; @@ -271,8 +272,9 @@ void Stream::on_linop_apply_started(const LinOp* A, const LinOp* b, template -void Stream::on_linop_apply_completed(const LinOp* A, const LinOp* b, - const LinOp* x) const +void Stream::on_linop_apply_completed( + const LinOp* A, const AbstractMultiVector* b, + const AbstractMultiVector* x) const { *os_ << prefix_ << "apply completed on A " << demangle_name(A) << " with b " << demangle_name(b) << " and x " << demangle_name(x) << std::endl; @@ -288,11 +290,10 @@ void Stream::on_linop_apply_completed(const LinOp* A, const LinOp* b, template -void Stream::on_linop_advanced_apply_started(const LinOp* A, - const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - const LinOp* x) const +void Stream::on_linop_advanced_apply_started( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const { *os_ << prefix_ << "advanced apply started on A " << demangle_name(A) << " with alpha " << demangle_name(alpha) << " b " << demangle_name(b) @@ -314,11 +315,10 @@ void Stream::on_linop_advanced_apply_started(const LinOp* A, template -void Stream::on_linop_advanced_apply_completed(const LinOp* A, - const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - const LinOp* x) const +void Stream::on_linop_advanced_apply_completed( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const { *os_ << prefix_ << "advanced apply completed on A " << demangle_name(A) << " with alpha " << demangle_name(alpha) << " b " << demangle_name(b) @@ -361,8 +361,10 @@ void Stream::on_linop_factory_generate_completed( template void Stream::on_criterion_check_started( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized) const + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized) const { *os_ << prefix_ << "check started for " << demangle_name(criterion) << " at iteration " << num_iterations << " with ID " @@ -391,10 +393,11 @@ void Stream::on_criterion_check_started( template void Stream::on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, const LinOp* solution, - const uint8& stoppingId, const bool& setFinalized, - const array* status, const bool& oneChanged, - const bool& converged) const + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stoppingId, + const bool& setFinalized, const array* status, + const bool& oneChanged, const bool& converged) const { *os_ << prefix_ << "check completed for " << demangle_name(criterion) << " at iteration " << num_iterations << " with ID " @@ -427,9 +430,11 @@ void Stream::on_criterion_check_completed( template void Stream::on_iteration_complete( - const LinOp* solver, const LinOp* right_hand_side, const LinOp* solution, - const size_type& num_iterations, const LinOp* residual, - const LinOp* residual_norm, const LinOp* implicit_resnorm_sq, + const LinOp* solver, const AbstractMultiVector* right_hand_side, + const AbstractMultiVector* solution, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_resnorm_sq, const array* status, bool stopped) const { *os_ << prefix_ << "iteration " << num_iterations @@ -472,11 +477,10 @@ void Stream::on_iteration_complete( template -void Stream::on_iteration_complete(const LinOp* solver, - const size_type& num_iterations, - const LinOp* residual, - const LinOp* solution, - const LinOp* residual_norm) const +void Stream::on_iteration_complete( + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm) const { on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, nullptr, nullptr, false); @@ -485,9 +489,10 @@ void Stream::on_iteration_complete(const LinOp* solver, template void Stream::on_iteration_complete( - const LinOp* solver, const size_type& num_iterations, const LinOp* residual, - const LinOp* solution, const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm) const + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm) const { on_iteration_complete(solver, nullptr, solution, num_iterations, residual, residual_norm, implicit_sq_residual_norm, nullptr, diff --git a/include/ginkgo/core/log/convergence.hpp b/include/ginkgo/core/log/convergence.hpp index ca4c6433294..f0a8eafe0d7 100644 --- a/include/ginkgo/core/log/convergence.hpp +++ b/include/ginkgo/core/log/convergence.hpp @@ -40,24 +40,28 @@ class Convergence : public Logger { public: void on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* solution, const uint8& stopping_id, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, const bool& set_finalized, const array* status, const bool& one_changed, const bool& all_stopped) const override; void on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* implicit_sq_resnorm, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized, - const array* status, const bool& one_changed, - const bool& all_stopped) const override; - - void on_iteration_complete(const LinOp* solver, const LinOp* b, - const LinOp* x, const size_type& num_iterations, - const LinOp* residual, - const LinOp* residual_norm, - const LinOp* implicit_resnorm_sq, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_resnorm, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized, const array* status, + const bool& one_changed, const bool& all_stopped) const override; + + void on_iteration_complete(const LinOp* solver, + const AbstractMultiVector* b, + const AbstractMultiVector* x, + const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_resnorm_sq, const array* status, bool stopped) const override; @@ -75,10 +79,7 @@ class Convergence : public Logger { static std::unique_ptr create( std::shared_ptr, const mask_type& enabled_events = Logger::criterion_events_mask | - Logger::iteration_complete_mask) - { - return std::unique_ptr(new Convergence(enabled_events)); - } + Logger::iteration_complete_mask); /** * Creates a convergence logger. This dynamically allocates the memory, @@ -91,59 +92,47 @@ class Convergence : public Logger { */ static std::unique_ptr create( const mask_type& enabled_events = Logger::criterion_events_mask | - Logger::iteration_complete_mask) - { - return std::unique_ptr(new Convergence(enabled_events)); - } + Logger::iteration_complete_mask); /** * Returns true if the solver has converged. * * @return the bool flag for convergence status */ - bool has_converged() const noexcept { return convergence_status_; } + bool has_converged() const noexcept; /** * Resets the convergence status to false. */ - void reset_convergence_status() { this->convergence_status_ = false; } + void reset_convergence_status(); /** * Returns the number of iterations * * @return the number of iterations */ - const size_type& get_num_iterations() const noexcept - { - return num_iterations_; - } + const size_type& get_num_iterations() const noexcept; /** * Returns the residual * * @return the residual */ - const LinOp* get_residual() const noexcept { return residual_.get(); } + const AbstractMultiVector* get_residual() const noexcept; /** * Returns the residual norm * * @return the residual norm */ - const LinOp* get_residual_norm() const noexcept - { - return residual_norm_.get(); - } + const AbstractMultiVector* get_residual_norm() const noexcept; /** * Returns the implicit squared residual norm * * @return the implicit squared residual norm */ - const LinOp* get_implicit_sq_resnorm() const noexcept - { - return implicit_sq_resnorm_.get(); - } + const AbstractMultiVector* get_implicit_sq_resnorm() const noexcept; protected: /** @@ -157,9 +146,7 @@ class Convergence : public Logger { explicit Convergence( std::shared_ptr, const mask_type& enabled_events = Logger::criterion_events_mask | - Logger::iteration_complete_mask) - : Logger(enabled_events) - {} + Logger::iteration_complete_mask); /** * Creates a Convergence logger. @@ -169,16 +156,14 @@ class Convergence : public Logger { */ explicit Convergence( const mask_type& enabled_events = Logger::criterion_events_mask | - Logger::iteration_complete_mask) - : Logger(enabled_events) - {} + Logger::iteration_complete_mask); private: mutable bool convergence_status_{false}; mutable size_type num_iterations_{}; - mutable std::unique_ptr residual_{}; - mutable std::unique_ptr residual_norm_{}; - mutable std::unique_ptr implicit_sq_resnorm_{}; + mutable std::unique_ptr residual_{}; + mutable std::unique_ptr residual_norm_{}; + mutable std::unique_ptr implicit_sq_resnorm_{}; }; diff --git a/include/ginkgo/core/log/logger.hpp b/include/ginkgo/core/log/logger.hpp index 9e8be442811..0cef264535d 100644 --- a/include/ginkgo/core/log/logger.hpp +++ b/include/ginkgo/core/log/logger.hpp @@ -354,8 +354,9 @@ public: \ */ GKO_LOGGER_REGISTER_EVENT(19, criterion_check_started, const stop::Criterion* criterion, - const size_type& it, const LinOp* r, - const LinOp* tau, const LinOp* x, + const size_type& it, const AbstractMultiVector* r, + const AbstractMultiVector* tau, + const AbstractMultiVector* x, const uint8& stopping_id, const bool& set_finalized) @@ -381,7 +382,8 @@ public: \ */ GKO_LOGGER_REGISTER_EVENT( 20, criterion_check_completed, const stop::Criterion* criterion, - const size_type& it, const LinOp* r, const LinOp* tau, const LinOp* x, + const size_type& it, const AbstractMultiVector* r, + const AbstractMultiVector* tau, const AbstractMultiVector* x, const uint8& stopping_id, const bool& set_finalized, const array* status, const bool& one_changed, const bool& all_converged) @@ -404,11 +406,12 @@ public: \ * @param all_converged whether all right hand sides are converged */ virtual void on_criterion_check_completed( - const stop::Criterion* criterion, const size_type& it, const LinOp* r, - const LinOp* tau, const LinOp* implicit_tau_sq, const LinOp* x, - const uint8& stopping_id, const bool& set_finalized, - const array* status, const bool& one_changed, - const bool& all_converged) const + const stop::Criterion* criterion, const size_type& it, + const AbstractMultiVector* r, const AbstractMultiVector* tau, + const AbstractMultiVector* implicit_tau_sq, + const AbstractMultiVector* x, const uint8& stopping_id, + const bool& set_finalized, const array* status, + const bool& one_changed, const bool& all_converged) const { this->on_criterion_check_completed(criterion, it, r, tau, x, stopping_id, set_finalized, status, @@ -444,9 +447,10 @@ public: \ GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") - virtual void on_iteration_complete(const LinOp* solver, const size_type& it, - const LinOp* r, const LinOp* x = nullptr, - const LinOp* tau = nullptr) const + virtual void on_iteration_complete( + const LinOp* solver, const size_type& it, const AbstractMultiVector* r, + const AbstractMultiVector* x = nullptr, + const AbstractMultiVector* tau = nullptr) const {} /** @@ -465,10 +469,10 @@ public: \ GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") - virtual void on_iteration_complete(const LinOp* solver, const size_type& it, - const LinOp* r, const LinOp* x, - const LinOp* tau, - const LinOp* implicit_tau_sq) const + virtual void on_iteration_complete( + const LinOp* solver, const size_type& it, const AbstractMultiVector* r, + const AbstractMultiVector* x, const AbstractMultiVector* tau, + const AbstractMultiVector* implicit_tau_sq) const { GKO_BEGIN_DISABLE_DEPRECATION_WARNINGS this->on_iteration_complete(solver, it, r, x, tau); @@ -490,12 +494,12 @@ public: \ * @param stopped whether all right hand sides have stopped (invalid if * status is not provided) */ - virtual void on_iteration_complete(const LinOp* solver, const LinOp* b, - const LinOp* x, const size_type& it, - const LinOp* r, const LinOp* tau, - const LinOp* implicit_tau_sq, - const array* status, - bool stopped) const + virtual void on_iteration_complete( + const LinOp* solver, const AbstractMultiVector* b, + const AbstractMultiVector* x, const size_type& it, + const AbstractMultiVector* r, const AbstractMultiVector* tau, + const AbstractMultiVector* implicit_tau_sq, + const array* status, bool stopped) const { GKO_BEGIN_DISABLE_DEPRECATION_WARNINGS this->on_iteration_complete(solver, it, r, x, tau, implicit_tau_sq); diff --git a/include/ginkgo/core/log/papi.hpp b/include/ginkgo/core/log/papi.hpp index 8b346df96ba..8c83746c463 100644 --- a/include/ginkgo/core/log/papi.hpp +++ b/include/ginkgo/core/log/papi.hpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -119,19 +119,23 @@ class Papi : public Logger { const Executor* exec, const PolymorphicObject* po) const override; /* LinOp events */ - void on_linop_apply_started(const LinOp* A, const LinOp* b, - const LinOp* x) const override; + void on_linop_apply_started(const LinOp* A, const MultiVector* b, + const MultiVector* x) const override; - void on_linop_apply_completed(const LinOp* A, const LinOp* b, - const LinOp* x) const override; + void on_linop_apply_completed(const LinOp* A, const MultiVector* b, + const MultiVector* x) const override; - void on_linop_advanced_apply_started(const LinOp* A, const LinOp* alpha, - const LinOp* b, const LinOp* beta, - const LinOp* x) const override; + void on_linop_advanced_apply_started(const LinOp* A, + const MultiVector* alpha, + const MultiVector* b, + const MultiVector* beta, + const MultiVector* x) const override; - void on_linop_advanced_apply_completed(const LinOp* A, const LinOp* alpha, - const LinOp* b, const LinOp* beta, - const LinOp* x) const override; + void on_linop_advanced_apply_completed(const LinOp* A, + const MultiVector* alpha, + const MultiVector* b, + const MultiVector* beta, + const MultiVector* x) const override; /* LinOpFactory events */ void on_linop_factory_generate_started(const LinOpFactory* factory, @@ -143,17 +147,18 @@ class Papi : public Logger { void on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* solution, const uint8& stopping_id, + const MultiVector* residual, const MultiVector* residual_norm, + const MultiVector* solution, const uint8& stopping_id, const bool& set_finalized, const array* status, const bool& one_changed, const bool& all_converged) const override; /* Internal solver events */ - void on_iteration_complete(const LinOp* solver, const LinOp* b, - const LinOp* x, const size_type& num_iterations, - const LinOp* residual, - const LinOp* residual_norm, - const LinOp* implicit_resnorm_sq, + void on_iteration_complete(const LinOp* solver, const MultiVector* b, + const MultiVector* x, + const size_type& num_iterations, + const MultiVector* residual, + const MultiVector* residual_norm, + const MultiVector* implicit_resnorm_sq, const array* status, bool stopped) const override; @@ -162,17 +167,18 @@ class Papi : public Logger { "information.") void on_iteration_complete(const LinOp* solver, const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm) const override; + const MultiVector* residual, + const MultiVector* solution, + const MultiVector* residual_norm) const override; GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") void on_iteration_complete( const LinOp* solver, const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm) const override; + const MultiVector* residual, const MultiVector* solution, + const MultiVector* residual_norm, + const MultiVector* implicit_sq_residual_norm) const override; /** * Creates a Papi Logger. diff --git a/include/ginkgo/core/log/profiler_hook.hpp b/include/ginkgo/core/log/profiler_hook.hpp index c5dc9dcbab6..438ae845237 100644 --- a/include/ginkgo/core/log/profiler_hook.hpp +++ b/include/ginkgo/core/log/profiler_hook.hpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -105,19 +105,21 @@ class ProfilerHook : public Logger { const PolymorphicObject* to) const override; /* LinOp events */ - void on_linop_apply_started(const LinOp* A, const LinOp* b, - const LinOp* x) const override; + void on_linop_apply_started(const LinOp* A, const AbstractMultiVector* b, + const AbstractMultiVector* x) const override; - void on_linop_apply_completed(const LinOp* A, const LinOp* b, - const LinOp* x) const override; + void on_linop_apply_completed(const LinOp* A, const AbstractMultiVector* b, + const AbstractMultiVector* x) const override; - void on_linop_advanced_apply_started(const LinOp* A, const LinOp* alpha, - const LinOp* b, const LinOp* beta, - const LinOp* x) const override; + void on_linop_advanced_apply_started( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const override; - void on_linop_advanced_apply_completed(const LinOp* A, const LinOp* alpha, - const LinOp* b, const LinOp* beta, - const LinOp* x) const override; + void on_linop_advanced_apply_completed( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const override; /* LinOpFactory events */ void on_linop_factory_generate_started(const LinOpFactory* factory, @@ -130,51 +132,56 @@ class ProfilerHook : public Logger { /* Criterion events */ void on_criterion_check_started(const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, - const LinOp* residual_norm, - const LinOp* solution, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, const bool& set_finalized) const override; void on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* solution, const uint8& stopping_id, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, const bool& set_finalized, const array* status, const bool& one_changed, const bool& all_stopped) const override; void on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* implicit_sq_resnorm, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized, - const array* status, const bool& one_changed, - const bool& all_stopped) const override; + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_resnorm, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized, const array* status, + const bool& one_changed, const bool& all_stopped) const override; /* Internal solver events */ void on_iteration_complete( - const LinOp* solver, const LinOp* right_hand_side, - const LinOp* solution, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm, + const LinOp* solver, const AbstractMultiVector* right_hand_side, + const AbstractMultiVector* solution, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm, const array* status, bool stopped) const override; GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") - void on_iteration_complete(const LinOp* solver, - const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm) const override; + void on_iteration_complete( + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm) const override; GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") void on_iteration_complete( const LinOp* solver, const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm) const override; + const AbstractMultiVector* residual, + const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm) const override; bool needs_propagation() const override; diff --git a/include/ginkgo/core/log/record.hpp b/include/ginkgo/core/log/record.hpp index 3a903517e12..e4748816007 100644 --- a/include/ginkgo/core/log/record.hpp +++ b/include/ginkgo/core/log/record.hpp @@ -21,21 +21,6 @@ namespace gko { * @ingroup log */ namespace log { -namespace detail { - - -template -std::unique_ptr clone_or_nullptr(T* input) -{ - // whether throw exception if input is not cloneable? - if (auto tmp = dynamic_cast(input)) { - return as(tmp->clone()); - } - return nullptr; -} - - -} // namespace detail /** @@ -43,44 +28,23 @@ std::unique_ptr clone_or_nullptr(T* input) */ struct iteration_complete_data { std::unique_ptr solver; - std::unique_ptr right_hand_side; - std::unique_ptr solution; + std::unique_ptr right_hand_side; + std::unique_ptr solution; const size_type num_iterations; - std::unique_ptr residual; - std::unique_ptr residual_norm; - std::unique_ptr implicit_sq_residual_norm; + std::unique_ptr residual; + std::unique_ptr residual_norm; + std::unique_ptr implicit_sq_residual_norm; array status; bool all_stopped; - iteration_complete_data(const LinOp* solver, const LinOp* right_hand_side, - const LinOp* solution, - const size_type num_iterations, - const LinOp* residual = nullptr, - const LinOp* residual_norm = nullptr, - const LinOp* implicit_sq_residual_norm = nullptr, - const gko::array* status = nullptr, - bool all_stopped = false) - : num_iterations{num_iterations}, all_stopped(all_stopped) - { - this->solver = detail::clone_or_nullptr(solver); - this->solution = detail::clone_or_nullptr(solution); - if (right_hand_side != nullptr) { - this->right_hand_side = detail::clone_or_nullptr(right_hand_side); - } - if (residual != nullptr) { - this->residual = detail::clone_or_nullptr(residual); - } - if (residual_norm != nullptr) { - this->residual_norm = detail::clone_or_nullptr(residual_norm); - } - if (implicit_sq_residual_norm != nullptr) { - this->implicit_sq_residual_norm = - detail::clone_or_nullptr(implicit_sq_residual_norm); - } - if (status != nullptr) { - this->status = *status; - } - } + iteration_complete_data( + const LinOp* solver, const AbstractMultiVector* right_hand_side, + const AbstractMultiVector* solution, const size_type num_iterations, + const AbstractMultiVector* residual = nullptr, + const AbstractMultiVector* residual_norm = nullptr, + const AbstractMultiVector* implicit_sq_residual_norm = nullptr, + const gko::array* status = nullptr, + bool all_stopped = false); }; @@ -113,14 +77,7 @@ struct polymorphic_object_data { polymorphic_object_data(const Executor* exec, const PolymorphicObject* input, - const PolymorphicObject* output = nullptr) - : exec{exec} - { - this->input = detail::clone_or_nullptr(input); - if (output != nullptr) { - this->output = detail::clone_or_nullptr(output); - } - } + const PolymorphicObject* output = nullptr); }; @@ -129,24 +86,14 @@ struct polymorphic_object_data { */ struct linop_data { std::unique_ptr A; - std::unique_ptr alpha; - std::unique_ptr b; - std::unique_ptr beta; - std::unique_ptr x; - - linop_data(const LinOp* A, const LinOp* alpha, const LinOp* b, - const LinOp* beta, const LinOp* x) - { - this->A = detail::clone_or_nullptr(A); - if (alpha != nullptr) { - this->alpha = detail::clone_or_nullptr(alpha); - } - this->b = detail::clone_or_nullptr(b); - if (beta != nullptr) { - this->beta = detail::clone_or_nullptr(beta); - } - this->x = detail::clone_or_nullptr(x); - } + std::unique_ptr alpha; + std::unique_ptr b; + std::unique_ptr beta; + std::unique_ptr x; + + linop_data(const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x); }; @@ -159,14 +106,7 @@ struct linop_factory_data { std::unique_ptr output; linop_factory_data(const LinOpFactory* factory, const LinOp* input, - const LinOp* output) - : factory{factory} - { - this->input = detail::clone_or_nullptr(input); - if (output != nullptr) { - this->output = detail::clone_or_nullptr(output); - } - } + const LinOp* output); }; @@ -176,9 +116,9 @@ struct linop_factory_data { struct criterion_data { const stop::Criterion* criterion; const size_type num_iterations; - std::unique_ptr residual; - std::unique_ptr residual_norm; - std::unique_ptr solution; + std::unique_ptr residual; + std::unique_ptr residual_norm; + std::unique_ptr solution; const uint8 stopping_id; const bool set_finalized; const array* status; @@ -186,32 +126,13 @@ struct criterion_data { const bool converged; criterion_data(const stop::Criterion* criterion, - const size_type& num_iterations, const LinOp* residual, - const LinOp* residual_norm, const LinOp* solution, - const uint8 stopping_id, const bool set_finalized, + const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8 stopping_id, + const bool set_finalized, const array* status = nullptr, - const bool oneChanged = false, const bool converged = false) - : criterion{criterion}, - num_iterations{num_iterations}, - residual{nullptr}, - residual_norm{nullptr}, - solution{nullptr}, - stopping_id{stopping_id}, - set_finalized{set_finalized}, - status{status}, - oneChanged{oneChanged}, - converged{converged} - { - if (residual != nullptr) { - this->residual = detail::clone_or_nullptr(residual); - } - if (residual_norm != nullptr) { - this->residual_norm = detail::clone_or_nullptr(residual_norm); - } - if (solution != nullptr) { - this->solution = detail::clone_or_nullptr(solution); - } - } + const bool oneChanged = false, const bool converged = false); }; @@ -333,19 +254,21 @@ class Record : public Logger { const Executor* exec, const PolymorphicObject* po) const override; /* LinOp events */ - void on_linop_apply_started(const LinOp* A, const LinOp* b, - const LinOp* x) const override; + void on_linop_apply_started(const LinOp* A, const AbstractMultiVector* b, + const AbstractMultiVector* x) const override; - void on_linop_apply_completed(const LinOp* A, const LinOp* b, - const LinOp* x) const override; + void on_linop_apply_completed(const LinOp* A, const AbstractMultiVector* b, + const AbstractMultiVector* x) const override; - void on_linop_advanced_apply_started(const LinOp* A, const LinOp* alpha, - const LinOp* b, const LinOp* beta, - const LinOp* x) const override; + void on_linop_advanced_apply_started( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const override; - void on_linop_advanced_apply_completed(const LinOp* A, const LinOp* alpha, - const LinOp* b, const LinOp* beta, - const LinOp* x) const override; + void on_linop_advanced_apply_completed( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const override; /* LinOpFactory events */ void on_linop_factory_generate_started(const LinOpFactory* factory, @@ -358,50 +281,58 @@ class Record : public Logger { /* Criterion events */ void on_criterion_check_started(const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, - const LinOp* residual_norm, - const LinOp* solution, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, const bool& set_finalized) const override; void on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* implicit_residual_norm_sq, const LinOp* solution, - const uint8& stopping_id, const bool& set_finalized, - const array* status, const bool& one_changed, - const bool& all_converged) const override; + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_residual_norm_sq, + const AbstractMultiVector* solution, const uint8& stopping_id, + const bool& set_finalized, const array* status, + const bool& one_changed, const bool& all_converged) const override; void on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* solution, const uint8& stopping_id, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, const bool& set_finalized, const array* status, const bool& one_changed, const bool& all_converged) const override; /* Internal solver events */ - void on_iteration_complete( - const LinOp* solver, const LinOp* right_hand_side, const LinOp* x, - const size_type& num_iterations, const LinOp* residual, - const LinOp* residual_norm, const LinOp* implicit_resnorm_sq, - const array* status, bool stopped) const override; + void on_iteration_complete(const LinOp* solver, + const AbstractMultiVector* right_hand_side, + const AbstractMultiVector* x, + const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_resnorm_sq, + const array* status, + bool stopped) const override; GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") - void on_iteration_complete(const LinOp* solver, - const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm) const override; + void on_iteration_complete( + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm) const override; GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") void on_iteration_complete( const LinOp* solver, const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm) const override; + const AbstractMultiVector* residual, + const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm) const override; /** * Creates a Record logger. This dynamically allocates the memory, @@ -421,10 +352,7 @@ class Record : public Logger { static std::unique_ptr create( std::shared_ptr exec, const mask_type& enabled_events = Logger::all_events_mask, - size_type max_storage = 1) - { - return std::unique_ptr(new Record(enabled_events, max_storage)); - } + size_type max_storage = 1); /** * Creates a Record logger. This dynamically allocates the memory, @@ -442,22 +370,19 @@ class Record : public Logger { */ static std::unique_ptr create( const mask_type& enabled_events = Logger::all_events_mask, - size_type max_storage = 1) - { - return std::unique_ptr(new Record(enabled_events, max_storage)); - } + size_type max_storage = 1); /** * Returns the logged data * * @return the logged data */ - const logged_data& get() const noexcept { return data_; } + const logged_data& get() const noexcept; /** * @copydoc ::get() */ - logged_data& get() noexcept { return data_; } + logged_data& get() noexcept; protected: /** @@ -474,9 +399,7 @@ class Record : public Logger { GKO_DEPRECATED("use two-parameter constructor") explicit Record(std::shared_ptr exec, const mask_type& enabled_events = Logger::all_events_mask, - size_type max_storage = 0) - : Record(enabled_events, max_storage) - {} + size_type max_storage = 0); /** * Creates a Record logger. @@ -489,9 +412,7 @@ class Record : public Logger { * memory overhead of this logger. */ explicit Record(const mask_type& enabled_events = Logger::all_events_mask, - size_type max_storage = 0) - : Logger(enabled_events), max_storage_{max_storage} - {} + size_type max_storage = 0); /** * Helper function which appends an object to a deque diff --git a/include/ginkgo/core/log/stream.hpp b/include/ginkgo/core/log/stream.hpp index 51f5da8fe9c..d7f64a9415d 100644 --- a/include/ginkgo/core/log/stream.hpp +++ b/include/ginkgo/core/log/stream.hpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -90,19 +90,21 @@ class Stream : public Logger { const Executor* exec, const PolymorphicObject* po) const override; /* LinOp events */ - void on_linop_apply_started(const LinOp* A, const LinOp* b, - const LinOp* x) const override; + void on_linop_apply_started(const LinOp* A, const AbstractMultiVector* b, + const AbstractMultiVector* x) const override; - void on_linop_apply_completed(const LinOp* A, const LinOp* b, - const LinOp* x) const override; + void on_linop_apply_completed(const LinOp* A, const AbstractMultiVector* b, + const AbstractMultiVector* x) const override; - void on_linop_advanced_apply_started(const LinOp* A, const LinOp* alpha, - const LinOp* b, const LinOp* beta, - const LinOp* x) const override; + void on_linop_advanced_apply_started( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const override; - void on_linop_advanced_apply_completed(const LinOp* A, const LinOp* alpha, - const LinOp* b, const LinOp* beta, - const LinOp* x) const override; + void on_linop_advanced_apply_completed( + const LinOp* A, const AbstractMultiVector* alpha, + const AbstractMultiVector* b, const AbstractMultiVector* beta, + const AbstractMultiVector* x) const override; /* LinOpFactory events */ void on_linop_factory_generate_started(const LinOpFactory* factory, @@ -115,44 +117,49 @@ class Stream : public Logger { /* Criterion events */ void on_criterion_check_started(const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, - const LinOp* residual_norm, - const LinOp* solution, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, const bool& set_finalized) const override; void on_criterion_check_completed( const stop::Criterion* criterion, const size_type& num_iterations, - const LinOp* residual, const LinOp* residual_norm, - const LinOp* solution, const uint8& stopping_id, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* solution, const uint8& stopping_id, const bool& set_finalized, const array* status, const bool& one_changed, const bool& all_converged) const override; /* Internal solver events */ - void on_iteration_complete(const LinOp* solver, const LinOp* b, - const LinOp* x, const size_type& num_iterations, - const LinOp* residual, - const LinOp* residual_norm, - const LinOp* implicit_resnorm_sq, + void on_iteration_complete(const LinOp* solver, + const AbstractMultiVector* b, + const AbstractMultiVector* x, + const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_resnorm_sq, const array* status, bool stopped) const override; GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") - void on_iteration_complete(const LinOp* solver, - const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm) const override; + void on_iteration_complete( + const LinOp* solver, const size_type& num_iterations, + const AbstractMultiVector* residual, + const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm) const override; GKO_DEPRECATED( "Please use the version with the additional stopping " "information.") void on_iteration_complete( const LinOp* solver, const size_type& num_iterations, - const LinOp* residual, const LinOp* solution, - const LinOp* residual_norm, - const LinOp* implicit_sq_residual_norm) const override; + const AbstractMultiVector* residual, + const AbstractMultiVector* solution, + const AbstractMultiVector* residual_norm, + const AbstractMultiVector* implicit_sq_residual_norm) const override; /** * Creates a Stream logger. This dynamically allocates the memory, From 945f6c282e415b281157ab371e6150e4cca5b3e6 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 16:58:46 +0200 Subject: [PATCH 08/21] fix hierarchy change for stopping criteria --- core/stop/criterion.cpp | 11 ++ core/stop/residual_norm.cpp | 132 ++++++++------------- include/ginkgo/core/stop/criterion.hpp | 25 ++-- include/ginkgo/core/stop/residual_norm.hpp | 30 +---- 4 files changed, 78 insertions(+), 120 deletions(-) diff --git a/core/stop/criterion.cpp b/core/stop/criterion.cpp index df033a0ab2c..160ba07324f 100644 --- a/core/stop/criterion.cpp +++ b/core/stop/criterion.cpp @@ -28,5 +28,16 @@ void Criterion::set_all_statuses(uint8 stoppingId, bool setFinalized, } +CriterionArgs::CriterionArgs(std::shared_ptr system_matrix, + std::shared_ptr b, + const AbstractMultiVector* x, + const AbstractMultiVector* initial_residual) + : system_matrix{system_matrix}, + b{b}, + x{x}, + initial_residual{initial_residual} +{} + + } // namespace stop } // namespace gko diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp index a46b1ba40dd..29ea6a581cb 100644 --- a/core/stop/residual_norm.cpp +++ b/core/stop/residual_norm.cpp @@ -5,7 +5,6 @@ #include "ginkgo/core/stop/residual_norm.hpp" #include -#include #include #include @@ -40,57 +39,6 @@ GKO_REGISTER_OPERATION(implicit_residual_norm, } // namespace implicit_residual_norm -template -bool any_is_complex() -{ - return false; -} - - -template -bool any_is_complex(const LinOp* in, Rest&&... rest) -{ -#if GINKGO_BUILD_MPI - bool is_complex_distributed = dynamic_cast>>*>(in); -#else - bool is_complex_distributed = false; -#endif - - return is_complex() || is_complex_distributed || - dynamic_cast< - const ConvertibleTo>>*>( - in) || - any_is_complex(std::forward(rest)...); -} - - -template -void norm_dispatch(Function&& fn, LinOps*... linops) -{ -#if GINKGO_BUILD_MPI - if (gko::detail::is_distributed(linops...)) { - if (any_is_complex(linops...)) { - experimental::distributed::precision_dispatch< - to_complex>(std::forward(fn), linops...); - } else { - experimental::distributed::precision_dispatch( - std::forward(fn), linops...); - } - } else -#endif - { - if (any_is_complex(linops...)) { - precision_dispatch>( - std::forward(fn), linops...); - } else { - precision_dispatch(std::forward(fn), - linops...); - } - } -} - - template ResidualNormBase::ResidualNormBase( std::shared_ptr exec, const CriterionArgs& args, @@ -114,23 +62,15 @@ ResidualNormBase::ResidualNormBase( } else { this->starting_tau_ = NormVector::create(exec, dim<2>{1, args.b->get_size()[1]}); - auto b_clone = share(as(as(args.b)->clone())); + auto b_clone = share(args.b->clone()); args.system_matrix->apply(neg_one_, args.x, one_, b_clone); - norm_dispatch( - [&](auto dense_r) { - dense_r->compute_norm2(this->starting_tau_, - reduction_tmp_); - }, - b_clone.get()); + b_clone->compute_norm2(this->starting_tau_, reduction_tmp_); } } else { this->starting_tau_ = NormVector::create( exec, dim<2>{1, args.initial_residual->get_size()[1]}); - norm_dispatch( - [&](auto dense_r) { - dense_r->compute_norm2(this->starting_tau_, reduction_tmp_); - }, - args.initial_residual); + args.initial_residual->compute_norm2(this->starting_tau_, + reduction_tmp_); } break; } @@ -140,11 +80,7 @@ ResidualNormBase::ResidualNormBase( } this->starting_tau_ = NormVector::create(exec, dim<2>{1, args.b->get_size()[1]}); - norm_dispatch( - [&](auto dense_r) { - dense_r->compute_norm2(this->starting_tau_, reduction_tmp_); - }, - args.b.get()); + args.b->compute_norm2(this->starting_tau_, reduction_tmp_); break; } case mode::absolute: { @@ -176,22 +112,14 @@ bool ResidualNormBase::check_impl( // Otherwise, we skip the residual check. return false; } else if (updater.residual_ != nullptr) { - norm_dispatch( - [&](auto dense_r) { - dense_r->compute_norm2(u_dense_tau_, reduction_tmp_); - }, - updater.residual_); + updater.residual_->compute_norm2(u_dense_tau_, reduction_tmp_); dense_tau = u_dense_tau_.get(); } else if (updater.solution_ != nullptr && system_matrix_ != nullptr && b_ != nullptr) { auto exec = this->get_executor(); - norm_dispatch( - [&](auto dense_b, auto dense_x) { - auto dense_r = dense_b->clone(); - system_matrix_->apply(neg_one_, dense_x, one_, dense_r); - dense_r->compute_norm2(u_dense_tau_, reduction_tmp_); - }, - b_.get(), updater.solution_); + auto dense_r = b_->clone(); + system_matrix_->apply(neg_one_, updater.solution_, one_, dense_r); + dense_r->compute_norm2(u_dense_tau_, reduction_tmp_); dense_tau = u_dense_tau_.get(); } else { GKO_NOT_SUPPORTED(nullptr); @@ -208,6 +136,29 @@ bool ResidualNormBase::check_impl( } +template +ResidualNormBase::ResidualNormBase( + std::shared_ptr exec) + : Criterion(exec), device_storage_{exec, 2} +{} + + +template +ResidualNorm::ResidualNorm(std::shared_ptr exec) + : ResidualNormBase(exec) +{} + + +template +ResidualNorm::ResidualNorm(const Factory* factory, + const CriterionArgs& args) + : ResidualNormBase(factory->get_executor(), args, + factory->get_parameters().reduction_factor, + factory->get_parameters().baseline), + parameters_{factory->get_parameters()} +{} + + template bool ImplicitResidualNorm::check_impl( uint8 stopping_id, bool set_finalized, array* stop_status, @@ -232,7 +183,24 @@ bool ImplicitResidualNorm::check_impl( } -#define GKO_DECLARE_RESIDUAL_NORM(ValueType) class ResidualNormBase +template +ImplicitResidualNorm::ImplicitResidualNorm( + std::shared_ptr exec) + : ResidualNormBase(exec) +{} + + +template +ImplicitResidualNorm::ImplicitResidualNorm(const Factory* factory, + const CriterionArgs& args) + : ResidualNormBase(factory->get_executor(), args, + factory->get_parameters().reduction_factor, + factory->get_parameters().baseline), + parameters_{factory->get_parameters()} +{} + + +#define GKO_DECLARE_RESIDUAL_NORM(ValueType) class ResidualNorm GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_RESIDUAL_NORM); diff --git a/include/ginkgo/core/stop/criterion.hpp b/include/ginkgo/core/stop/criterion.hpp index da7db75aba4..352bb5c6ff6 100644 --- a/include/ginkgo/core/stop/criterion.hpp +++ b/include/ginkgo/core/stop/criterion.hpp @@ -99,11 +99,12 @@ class Criterion : public PolymorphicObject { GKO_UPDATER_REGISTER_PARAMETER(size_type, num_iterations); // ignore_residual_check default is false GKO_UPDATER_REGISTER_PARAMETER(bool, ignore_residual_check); - GKO_UPDATER_REGISTER_PTR_PARAMETER(const LinOp, residual); - GKO_UPDATER_REGISTER_PTR_PARAMETER(const LinOp, residual_norm); - GKO_UPDATER_REGISTER_PTR_PARAMETER(const LinOp, + GKO_UPDATER_REGISTER_PTR_PARAMETER(const AbstractMultiVector, residual); + GKO_UPDATER_REGISTER_PTR_PARAMETER(const AbstractMultiVector, + residual_norm); + GKO_UPDATER_REGISTER_PTR_PARAMETER(const AbstractMultiVector, implicit_sq_residual_norm); - GKO_UPDATER_REGISTER_PTR_PARAMETER(const LinOp, solution); + GKO_UPDATER_REGISTER_PTR_PARAMETER(const AbstractMultiVector, solution); #undef GKO_UPDATER_REGISTER_PTR_PARAMETER #undef GKO_UPDATER_REGISTER_PARAMETER @@ -204,19 +205,15 @@ class Criterion : public PolymorphicObject { */ struct CriterionArgs { std::shared_ptr system_matrix; - std::shared_ptr b; - const LinOp* x; - const LinOp* initial_residual; + std::shared_ptr b; + const AbstractMultiVector* x; + const AbstractMultiVector* initial_residual; CriterionArgs(std::shared_ptr system_matrix, - std::shared_ptr b, const LinOp* x, - const LinOp* initial_residual = nullptr) - : system_matrix{system_matrix}, - b{b}, - x{x}, - initial_residual{initial_residual} - {} + std::shared_ptr b, + const AbstractMultiVector* x, + const AbstractMultiVector* initial_residual = nullptr); }; diff --git a/include/ginkgo/core/stop/residual_norm.hpp b/include/ginkgo/core/stop/residual_norm.hpp index 946bad16898..88e322961db 100644 --- a/include/ginkgo/core/stop/residual_norm.hpp +++ b/include/ginkgo/core/stop/residual_norm.hpp @@ -59,9 +59,7 @@ class ResidualNormBase : public Criterion { array* stop_status, bool* one_changed, const Criterion::Updater& updater) override; - explicit ResidualNormBase(std::shared_ptr exec) - : Criterion(exec), device_storage_{exec, 2} - {} + explicit ResidualNormBase(std::shared_ptr exec); explicit ResidualNormBase(std::shared_ptr exec, const CriterionArgs& args, @@ -76,7 +74,7 @@ class ResidualNormBase : public Criterion { private: mode baseline_{mode::rhs_norm}; std::shared_ptr system_matrix_{}; - std::shared_ptr b_{}; + std::shared_ptr b_{}; /* one/neg_one for residual computation */ std::shared_ptr one_{}; std::shared_ptr neg_one_{}; @@ -136,17 +134,9 @@ class ResidualNorm : public ResidualNormBase { GKO_ENABLE_BUILD_METHOD(Factory); protected: - explicit ResidualNorm(std::shared_ptr exec) - : ResidualNormBase(exec) - {} + explicit ResidualNorm(std::shared_ptr exec); - explicit ResidualNorm(const Factory* factory, const CriterionArgs& args) - : ResidualNormBase( - factory->get_executor(), args, - factory->get_parameters().reduction_factor, - factory->get_parameters().baseline), - parameters_{factory->get_parameters()} - {} + explicit ResidualNorm(const Factory* factory, const CriterionArgs& args); }; @@ -205,18 +195,10 @@ class ImplicitResidualNorm : public ResidualNormBase { array* stop_status, bool* one_changed, const Criterion::Updater& updater) override; - explicit ImplicitResidualNorm(std::shared_ptr exec) - : ResidualNormBase(exec) - {} + explicit ImplicitResidualNorm(std::shared_ptr exec); explicit ImplicitResidualNorm(const Factory* factory, - const CriterionArgs& args) - : ResidualNormBase( - factory->get_executor(), args, - factory->get_parameters().reduction_factor, - factory->get_parameters().baseline), - parameters_{factory->get_parameters()} - {} + const CriterionArgs& args); }; From ba293b1c5c04ca8ef106922a314b330920b8250d Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:01:05 +0200 Subject: [PATCH 09/21] fix hierarchy change for matrices removed functionality: - implicit Diag * Csr - implicit Csr * Csr --- core/matrix/coo.cpp | 94 ++++----- core/matrix/csr.cpp | 122 +++++------ core/matrix/dense.cpp | 60 +++--- core/matrix/diagonal.cpp | 198 +++++++++++------- core/matrix/ell.cpp | 35 ++-- core/matrix/fbcsr.cpp | 58 ++--- core/matrix/fft.cpp | 135 ++++++------ core/matrix/hybrid.cpp | 38 ++-- core/matrix/identity.cpp | 26 ++- core/matrix/multivector.cpp | 21 +- core/matrix/permutation.cpp | 56 +++-- core/matrix/row_gatherer.cpp | 41 ++-- core/matrix/scaled_permutation.cpp | 38 ++-- core/matrix/sellp.cpp | 28 +-- core/matrix/sparsity_csr.cpp | 36 ++-- include/ginkgo/core/matrix/coo.hpp | 43 ++-- include/ginkgo/core/matrix/csr.hpp | 31 ++- include/ginkgo/core/matrix/dense.hpp | 13 +- include/ginkgo/core/matrix/diagonal.hpp | 48 +++-- include/ginkgo/core/matrix/ell.hpp | 9 +- include/ginkgo/core/matrix/fbcsr.hpp | 9 +- include/ginkgo/core/matrix/fft.hpp | 27 ++- include/ginkgo/core/matrix/hybrid.hpp | 9 +- include/ginkgo/core/matrix/identity.hpp | 9 +- include/ginkgo/core/matrix/multivector.hpp | 8 +- include/ginkgo/core/matrix/permutation.hpp | 8 +- include/ginkgo/core/matrix/row_gatherer.hpp | 9 +- .../ginkgo/core/matrix/scaled_permutation.hpp | 8 +- include/ginkgo/core/matrix/sellp.hpp | 9 +- include/ginkgo/core/matrix/sparsity_csr.hpp | 9 +- 30 files changed, 616 insertions(+), 619 deletions(-) diff --git a/core/matrix/coo.cpp b/core/matrix/coo.cpp index eb8d0656581..8ea5c74e9e1 100644 --- a/core/matrix/coo.cpp +++ b/core/matrix/coo.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -18,6 +17,7 @@ #include #include "core/base/device_matrix_data_kernels.hpp" +#include "core/base/dispatch_helper.hpp" #include "core/components/absolute_array_kernels.hpp" #include "core/components/fill_array_kernels.hpp" #include "core/components/format_conversion_kernels.hpp" @@ -114,8 +114,8 @@ Coo::Coo(std::shared_ptr exec, template -void Coo::apply2(ptr_param b, - ptr_param x) +void Coo::apply2(ptr_param b, + ptr_param x) const { this->validate_application_parameters(b.get(), x.get()); auto exec = this->get_executor(); @@ -123,36 +123,11 @@ void Coo::apply2(ptr_param b, make_temporary_clone(exec, x).get()); } - -template -void Coo::apply2(ptr_param b, - ptr_param x) const -{ - this->validate_application_parameters(b.get(), x.get()); - auto exec = this->get_executor(); - this->apply2_impl(make_temporary_clone(exec, b).get(), - make_temporary_clone(exec, x).get()); -} - - -template -void Coo::apply2(ptr_param alpha, - ptr_param b, - ptr_param x) -{ - this->validate_application_parameters(b.get(), x.get()); - GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1)); - auto exec = this->get_executor(); - this->apply2_impl(make_temporary_clone(exec, alpha).get(), - make_temporary_clone(exec, b).get(), - make_temporary_clone(exec, x).get()); -} - - template -void Coo::apply2(ptr_param alpha, - ptr_param b, - ptr_param x) const +void Coo::apply2( + ptr_param alpha, + ptr_param b, + ptr_param x) const { this->validate_application_parameters(b.get(), x.get()); GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1)); @@ -164,59 +139,62 @@ void Coo::apply2(ptr_param alpha, template -void Coo::apply_impl(const LinOp* b, LinOp* x) const +void Coo::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - this->get_executor()->run(coo::make_spmv( - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_x->get_device_view())); + apply_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { + this->get_executor()->run( + coo::make_spmv(this->get_const_device_view(), view_b, view_x)); }, b, x); } template -void Coo::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Coo::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { + apply_precision_dispatch( + [this](auto dense_alpha, auto view_b, auto dense_beta, auto view_x, + auto...) { this->get_executor()->run(coo::make_advanced_spmv( dense_alpha->get_const_device_view(), - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); + this->get_const_device_view(), view_b, + dense_beta->get_const_device_view(), view_x)); }, alpha, b, beta, x); } template -void Coo::apply2_impl(const LinOp* b, LinOp* x) const +void Coo::apply2_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - this->get_executor()->run(coo::make_spmv2( - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_x->get_device_view())); + apply_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { + this->get_executor()->run( + coo::make_spmv2(this->get_const_device_view(), view_b, view_x)); }, b, x); } template -void Coo::apply2_impl(const LinOp* alpha, const LinOp* b, - LinOp* x) const +void Coo::apply2_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_x) { + auto dense_alpha = as>(alpha->as_precision(this)); + apply_precision_dispatch( + [this, &dense_alpha](auto view_b, auto view_x, auto...) { this->get_executor()->run(coo::make_advanced_spmv2( dense_alpha->get_const_device_view(), - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_x->get_device_view())); + this->get_const_device_view(), view_b, view_x)); }, - alpha, b, x); + b, x); } diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp index f44f43305c4..dbc49af5317 100644 --- a/core/matrix/csr.cpp +++ b/core/matrix/csr.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -27,6 +26,7 @@ #include "core/base/array_access.hpp" #include "core/base/device_matrix_data_kernels.hpp" +#include "core/base/dispatch_helper.hpp" #include "core/components/absolute_array_kernels.hpp" #include "core/components/fill_array_kernels.hpp" #include "core/components/format_conversion_kernels.hpp" @@ -38,7 +38,6 @@ #include "core/matrix/hybrid_kernels.hpp" #include "core/matrix/permutation.hpp" #include "core/matrix/sellp_kernels.hpp" -#include "ginkgo/core/base/multivector.hpp" namespace gko { @@ -186,6 +185,28 @@ Csr::create_const( } +template +void Csr::scale( + ptr_param alpha) + +{ + auto exec = this->get_executor(); + GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1)); + this->scale_impl(make_temporary_clone(exec, alpha).get()); +} + + +template +void Csr::inv_scale( + ptr_param alpha) + +{ + auto exec = this->get_executor(); + GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1)); + this->inv_scale_impl(make_temporary_clone(exec, alpha).get()); +} + + template std::unique_ptr> Csr::create( std::shared_ptr exec, csr::spmv_strategy strategy) @@ -373,25 +394,16 @@ Csr::Csr(Csr&& other) template -void Csr::apply_impl(const LinOp* b, LinOp* x) const +void Csr::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - using ComplexMultiVector = MultiVector>; - using TCsr = Csr; - if (auto b_csr = dynamic_cast(b)) { - // if b is a CSR matrix, we compute a SpGeMM - auto x_csr = as(x); - this->get_executor()->run(csr::make_spgemm( - this, b_csr, make_builder_unique_ptr(x_csr).get())); - } else { - mixed_precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - this->get_executor()->run(csr::make_spmv( - this->get_actual_strategy(), max_nnz_per_row_, this, - dense_b->get_const_device_view(), - dense_x->get_device_view())); - }, - b, x); - } + apply_mixed_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { + this->get_executor()->run( + csr::make_spmv(this->get_actual_strategy(), max_nnz_per_row_, + this, view_b, view_x)); + }, + b, x); } @@ -493,44 +505,20 @@ void Csr::make_srow() template -void Csr::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const -{ - using ComplexMultiVector = MultiVector>; - using RealMultiVector = MultiVector>; - using TCsr = Csr; - if (auto b_csr = dynamic_cast(b)) { - // if b is a CSR matrix, we compute a SpGeMM - auto x_csr = as(x); - auto x_copy = x_csr->clone(); - this->get_executor()->run(csr::make_advanced_spgemm( - as>(alpha)->get_const_device_view(), this, - b_csr, as>(beta)->get_const_device_view(), - x_copy.get(), make_builder_unique_ptr(x_csr).get())); - } else if (dynamic_cast*>(b)) { - // if b is an identity matrix, we compute an SpGEAM - auto x_csr = as(x); - auto x_copy = x_csr->clone(); - this->get_executor()->run(csr::make_spgeam( - as>(alpha)->get_const_device_view(), this, - as>(beta)->get_const_device_view(), - x_copy.get(), make_builder_unique_ptr(x_csr).get())); - } else { - mixed_precision_dispatch_real_complex( - [this, alpha, beta](auto dense_b, auto dense_x) { - auto dense_alpha = make_temporary_conversion(alpha); - auto dense_beta = make_temporary_conversion< - typename std::decay_t::value_type>( - beta); - this->get_executor()->run(csr::make_advanced_spmv( - this->get_actual_strategy(), max_nnz_per_row_, - dense_alpha->get_const_device_view(), this, - dense_b->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); - }, - b, x); - } +void Csr::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const +{ + apply_mixed_precision_dispatch( + [this](auto dense_alpha, auto view_b, auto dense_beta, auto view_x, + auto...) { + this->get_executor()->run(csr::make_advanced_spmv( + this->get_actual_strategy(), max_nnz_per_row_, + dense_alpha->get_const_device_view(), this, view_b, + dense_beta->get_const_device_view(), view_x)); + }, + alpha, b, beta, x); } @@ -1792,21 +1780,23 @@ Csr::compute_absolute() const template -void Csr::scale_impl(const LinOp* alpha) +void Csr::scale_impl(const AbstractMultiVector* alpha) { auto exec = this->get_executor(); - exec->run(csr::make_scale( - make_temporary_conversion(alpha)->get_const_device_view(), - this)); + exec->run( + csr::make_scale(as>(alpha->as_precision(this)) + ->get_const_device_view(), + this)); } template -void Csr::inv_scale_impl(const LinOp* alpha) +void Csr::inv_scale_impl(const AbstractMultiVector* alpha) { auto exec = this->get_executor(); exec->run(csr::make_inv_scale( - make_temporary_conversion(alpha)->get_const_device_view(), + as>(alpha->as_precision(this)) + ->get_const_device_view(), this)); } @@ -1823,9 +1813,9 @@ void Csr::add_scaled_identity_impl( "The matrix has one or more structurally zero diagonal entries!"); } this->get_executor()->run(csr::make_add_scaled_identity( - a->as_precision(this->get_precision()) + a->as_precision(this) ->template get_const_local_device_view(), - b->as_precision(this->get_precision()) + b->as_precision(this) ->template get_const_local_device_view(), this)); } diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp index 9a64a46c469..d223803b661 100644 --- a/core/matrix/dense.cpp +++ b/core/matrix/dense.cpp @@ -2,8 +2,6 @@ // // SPDX-License-Identifier: BSD-3-Clause -#include -#include #include #include #include @@ -709,36 +707,32 @@ void Dense::conj_transpose(ptr_param output) const template -void Dense::add_scaled(ptr_param alpha, +void Dense::add_scaled(ptr_param alpha, ptr_param> diag) { - GKO_ASSERT_EQUAL_ROWS(alpha, dim<2>(1, 1)); - if (alpha->get_size()[1] != 1) { - // different alpha for each column - GKO_ASSERT_EQUAL_COLS(this, alpha); - } - GKO_ASSERT_EQUAL_DIMENSIONS(this, diag); auto exec = this->get_executor(); + GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1)); + auto alpha_clone = make_temporary_clone(exec, alpha); + auto converted_alpha = alpha_clone->as_precision(this->get_precision()); exec->run(dense::make_add_scaled_diag( - make_temporary_conversion(alpha)->get_const_device_view(), - diag.get(), this->get_device_view())); + as>(converted_alpha.get()) + ->get_const_device_view(), + make_temporary_clone(exec, diag).get(), this->get_device_view())); } template -void Dense::sub_scaled(ptr_param alpha, +void Dense::sub_scaled(ptr_param alpha, ptr_param> diag) { - GKO_ASSERT_EQUAL_ROWS(alpha, dim<2>(1, 1)); - if (alpha->get_size()[1] != 1) { - // different alpha for each column - GKO_ASSERT_EQUAL_COLS(this, alpha); - } - GKO_ASSERT_EQUAL_DIMENSIONS(this, diag); auto exec = this->get_executor(); + GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1)); + auto alpha_clone = make_temporary_clone(exec, alpha); + auto converted_alpha = alpha_clone->as_precision(this->get_precision()); exec->run(dense::make_sub_scaled_diag( - make_temporary_conversion(alpha)->get_const_device_view(), - diag.get(), this->get_device_view())); + as>(converted_alpha.get()) + ->get_const_device_view(), + make_temporary_clone(exec, diag).get(), this->get_device_view())); } @@ -958,29 +952,31 @@ Dense::Dense(std::shared_ptr exec, template -void Dense::apply_impl(const LinOp* b, LinOp* x) const +void Dense::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { + apply_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { this->get_executor()->run(dense::make_simple_apply( - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_x->get_device_view())); + this->get_const_device_view(), view_b, view_x)); }, b, x); } template -void Dense::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Dense::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { + apply_precision_dispatch( + [this](auto dense_alpha, auto view_b, auto dense_beta, auto view_x, + auto...) { this->get_executor()->run(dense::make_advanced_apply( dense_alpha->get_const_device_view(), - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); + this->get_const_device_view(), view_b, + dense_beta->get_const_device_view(), view_x)); }, alpha, b, beta, x); } diff --git a/core/matrix/diagonal.cpp b/core/matrix/diagonal.cpp index 7e1cffd224e..ffc36c1bf41 100644 --- a/core/matrix/diagonal.cpp +++ b/core/matrix/diagonal.cpp @@ -5,9 +5,9 @@ #include "ginkgo/core/matrix/diagonal.hpp" #include -#include #include +#include "core/base/dispatch_helper.hpp" #include "core/components/absolute_array_kernels.hpp" #include "core/matrix/diagonal_kernels.hpp" @@ -36,98 +36,52 @@ GKO_REGISTER_OPERATION(outplace_absolute_array, template -void Diagonal::apply_impl(const LinOp* b, LinOp* x) const +void Diagonal::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - auto exec = this->get_executor(); - - if (dynamic_cast*>(b) && - dynamic_cast*>(x)) { - exec->run( - diagonal::make_apply_to_csr(this, as>(b), - as>(x), false)); - } else if (dynamic_cast*>(b) && - dynamic_cast*>(x)) { - exec->run( - diagonal::make_apply_to_csr(this, as>(b), - as>(x), false)); - } else { - precision_dispatch_real_complex( - [this, &exec](auto dense_b, auto dense_x) { - exec->run(diagonal::make_apply_to_dense( - this, dense_b->get_const_device_view(), - dense_x->get_device_view(), false)); - }, - b, x); - } + apply_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { + this->get_executor()->run( + diagonal::make_apply_to_dense(this, view_b, view_x, false)); + }, + b, x); } template -void Diagonal::rapply_impl(const LinOp* b, LinOp* x) const +void Diagonal::apply(ptr_param> b, + ptr_param> x) const { - auto exec = this->get_executor(); - - if (dynamic_cast*>(b) && - dynamic_cast*>(x)) { - exec->run(diagonal::make_right_apply_to_csr( - this, as>(b), as>(x))); - } else if (dynamic_cast*>(b) && - dynamic_cast*>(x)) { - exec->run(diagonal::make_right_apply_to_csr( - this, as>(b), as>(x))); - } else { - // no real-to-complex conversion, as this would require doubling the - // diagonal entries for the complex-to-real columns - precision_dispatch( - [this, &exec](auto dense_b, auto dense_x) { - exec->run(diagonal::make_right_apply_to_dense( - this, dense_b->get_const_device_view(), - dense_x->get_device_view())); - }, - b, x); - } + LinOp::validate_application_parameters(b.get(), x.get()); + this->get_executor()->run( + diagonal::make_apply_to_csr(this, b.get(), x.get(), false)); } template -void Diagonal::inverse_apply_impl(const LinOp* b, LinOp* x) const +void Diagonal::apply(ptr_param> b, + ptr_param> x) const { - auto exec = this->get_executor(); - - if (dynamic_cast*>(b) && - dynamic_cast*>(x)) { - exec->run( - diagonal::make_apply_to_csr(this, as>(b), - as>(x), true)); - } else if (dynamic_cast*>(b) && - dynamic_cast*>(x)) { - exec->run( - diagonal::make_apply_to_csr(this, as>(b), - as>(x), true)); - } else { - precision_dispatch_real_complex( - [this, &exec](auto dense_b, auto dense_x) { - exec->run(diagonal::make_apply_to_dense( - this, dense_b->get_const_device_view(), - dense_x->get_device_view(), true)); - }, - b, x); - } + LinOp::validate_application_parameters(b.get(), x.get()); + this->get_executor()->run( + diagonal::make_apply_to_csr(this, b.get(), x.get(), false)); } template -void Diagonal::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Diagonal::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + auto converted_x = x->as_precision(this); + auto x_clone = converted_x->clone(); + this->apply_impl(b, x_clone.get()); + converted_x->scale( + as>(beta->as_precision(this)).get()); + converted_x->add_scaled( + as>(alpha->as_precision(this)).get(), + x_clone.get()); } @@ -349,6 +303,96 @@ void Diagonal::compute_absolute_inplace() } +template +void validate_reverse_application_parameters(const Diagonal* op, T b, + U x) +{ + GKO_ASSERT_REVERSE_CONFORMANT(op, b); + GKO_ASSERT_EQUAL_ROWS(b, x); + GKO_ASSERT_EQUAL_COLS(op, x); +} + + +template +void Diagonal::rapply(ptr_param b, + ptr_param x) const +{ + validate_reverse_application_parameters(this, b, x); + apply_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { + this->get_executor()->run( + diagonal::make_right_apply_to_dense(this, view_b, view_x)); + }, + b.get(), x.get()); +} + + +template +void Diagonal::rapply(ptr_param> b, + ptr_param> x) const +{ + validate_reverse_application_parameters(this, b, x); + this->get_executor()->run( + diagonal::make_right_apply_to_csr(this, b.get(), x.get())); +} + + +template +void Diagonal::rapply(ptr_param> b, + ptr_param> x) const +{ + validate_reverse_application_parameters(this, b, x); + this->get_executor()->run( + diagonal::make_right_apply_to_csr(this, b.get(), x.get())); +} + + +template +void validate_inverse_application_parameters(const Diagonal* op, T b, + U x) +{ + GKO_ASSERT_CONFORMANT(op, b); + GKO_ASSERT_EQUAL_ROWS(b, x); + GKO_ASSERT_EQUAL_ROWS(op, x); +} + + +template +void Diagonal::inverse_apply(ptr_param b, + ptr_param x) const +{ + validate_inverse_application_parameters(this, b, x); + apply_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { + this->get_executor()->run( + diagonal::make_apply_to_dense(this, view_b, view_x, true)); + }, + b.get(), x.get()); +} + + +template +void Diagonal::inverse_apply( + ptr_param> b, + ptr_param> x) const +{ + validate_inverse_application_parameters(this, b, x); + this->get_executor()->run( + diagonal::make_apply_to_csr(this, b.get(), x.get(), true)); +} + + +template +void Diagonal::inverse_apply( + ptr_param> b, + ptr_param> x) const +{ + validate_inverse_application_parameters(this, b, x); + this->get_executor()->run( + diagonal::make_apply_to_csr(this, b.get(), x.get(), true)); +} + + template std::unique_ptr::absolute_type> Diagonal::compute_absolute() const diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp index 1ca1a014c4e..be9f9bc2bd9 100644 --- a/core/matrix/ell.cpp +++ b/core/matrix/ell.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -20,6 +19,7 @@ #include "core/base/allocator.hpp" #include "core/base/array_access.hpp" #include "core/base/device_matrix_data_kernels.hpp" +#include "core/base/dispatch_helper.hpp" #include "core/components/absolute_array_kernels.hpp" #include "core/components/fill_array_kernels.hpp" #include "core/components/format_conversion_kernels.hpp" @@ -129,34 +129,33 @@ Ell::Ell(Ell&& other) : Ell(other.get_executor()) template -void Ell::apply_impl(const LinOp* b, LinOp* x) const +void Ell::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - mixed_precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - this->get_executor()->run(ell::make_spmv( - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_x->get_device_view())); + apply_mixed_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { + this->get_executor()->run( + ell::make_spmv(this->get_const_device_view(), view_b, view_x)); }, b, x); } template -void Ell::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Ell::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - mixed_precision_dispatch_real_complex( - [this, alpha, beta](auto dense_b, auto dense_x) { - auto dense_alpha = make_temporary_conversion(alpha); - auto dense_beta = make_temporary_conversion< - typename std::decay_t::value_type>(beta); + apply_mixed_precision_dispatch( + [this](auto dense_alpha, auto view_b, auto dense_beta, auto view_x, + auto...) { this->get_executor()->run(ell::make_advanced_spmv( dense_alpha->get_const_device_view(), - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); + this->get_const_device_view(), view_b, + dense_beta->get_const_device_view(), view_x)); }, - b, x); + alpha, b, beta, x); } diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp index a28ef70224c..646cad6d5e8 100644 --- a/core/matrix/fbcsr.cpp +++ b/core/matrix/fbcsr.cpp @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -23,6 +22,7 @@ #include "accessor/block_col_major.hpp" #include "accessor/range.hpp" +#include "core/base/dispatch_helper.hpp" #include "core/components/absolute_array_kernels.hpp" #include "core/components/fill_array_kernels.hpp" #include "core/matrix/fbcsr_kernels.hpp" @@ -104,47 +104,31 @@ Fbcsr::Fbcsr(Fbcsr&& other) : Fbcsr{other.get_executor()} template -void Fbcsr::apply_impl(const LinOp* b, LinOp* x) const +void Fbcsr::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - if (auto b_fbcsr = dynamic_cast*>(b)) { - // if b is a FBCSR matrix, we need an SpGeMM - GKO_NOT_SUPPORTED(b_fbcsr); - } else { - // otherwise we assume that b is dense and compute a SpMV/SpMM - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - this->get_executor()->run( - fbcsr::make_spmv(this, dense_b->get_const_device_view(), - dense_x->get_device_view())); - }, - b, x); - } + apply_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { + this->get_executor()->run(fbcsr::make_spmv(this, view_b, view_x)); + }, + b, x); } template -void Fbcsr::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const -{ - if (auto b_fbcsr = dynamic_cast*>(b)) { - // if b is a FBCSR matrix, we need an SpGeMM - GKO_NOT_SUPPORTED(b_fbcsr); - } else if (auto b_ident = dynamic_cast*>(b)) { - // if b is an identity matrix, we need an SpGEAM - GKO_NOT_SUPPORTED(b_ident); - } else { - // otherwise we assume that b is dense and compute a SpMV/SpMM - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, - auto dense_x) { - this->get_executor()->run(fbcsr::make_advanced_spmv( - dense_alpha->get_const_device_view(), this, - dense_b->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); - }, - alpha, b, beta, x); - } +void Fbcsr::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const +{ + apply_precision_dispatch( + [this](auto dense_alpha, auto view_b, auto dense_beta, auto view_x, + auto...) { + this->get_executor()->run(fbcsr::make_advanced_spmv( + dense_alpha->get_const_device_view(), this, view_b, + dense_beta->get_const_device_view(), view_x)); + }, + alpha, b, beta, x); } diff --git a/core/matrix/fft.cpp b/core/matrix/fft.cpp index beb5a1a12ab..2e121ac23ea 100644 --- a/core/matrix/fft.cpp +++ b/core/matrix/fft.cpp @@ -149,40 +149,45 @@ dim<1> Fft::get_fft_size() const { return dim<1>{this->get_size()[0]}; } bool Fft::is_inverse() const { return inverse_; } +void check_fft_inputs(const AbstractMultiVector* b, + const AbstractMultiVector* x) +{ + if (b->get_precision() != precision::complex_fp32 && + b->get_precision() != precision::complex_fp64 && + b->get_precision() != x->get_precision()) { + GKO_INVALID_STATE( + "Fft/Fft2/Fft3 require that both input and output vectors have the " + "same complex precision"); + } +} + -void Fft::apply_impl(const LinOp* b, LinOp* x) const +void Fft::apply_impl(const AbstractMultiVector* b, AbstractMultiVector* x) const { - if (auto float_b = - dynamic_cast>*>(b)) { - auto dense_x = as>>(x); - get_executor()->run(fft::make_fft(float_b->get_const_device_view(), - dense_x->get_device_view(), inverse_, - buffer_)); + check_fft_inputs(b, x); + if (b->get_precision() == precision::complex_fp32) { + get_executor()->run(fft::make_fft( + b->template get_const_local_device_view>(), + x->template get_local_device_view>(), + inverse_, buffer_)); } else { - auto dense_b = as>>(b); - auto dense_x = as>>(x); - get_executor()->run(fft::make_fft(dense_b->get_const_device_view(), - dense_x->get_device_view(), inverse_, - buffer_)); + get_executor()->run(fft::make_fft( + b->template get_const_local_device_view>(), + x->template get_local_device_view>(), + inverse_, buffer_)); } } -void Fft::apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const +void Fft::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - if (auto float_x = dynamic_cast>*>(x)) { - auto clone_x = as(as(x)->clone()); - this->apply_impl(b, clone_x.get()); - float_x->scale(beta); - float_x->add_scaled(alpha, clone_x); - } else { - auto dense_x = as>>(x); - auto clone_x = as(as(x)->clone()); - this->apply_impl(b, clone_x.get()); - dense_x->scale(beta); - dense_x->add_scaled(alpha, clone_x); - } + auto clone_x = x->clone(); + this->apply_impl(b, clone_x.get()); + x->scale(beta); + x->add_scaled(alpha, clone_x); } @@ -250,39 +255,34 @@ dim<2> Fft2::get_fft_size() const { return fft_size_; } bool Fft2::is_inverse() const { return inverse_; } -void Fft2::apply_impl(const LinOp* b, LinOp* x) const +void Fft2::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - if (auto float_b = - dynamic_cast>*>(b)) { - auto dense_x = as>>(x); + check_fft_inputs(b, x); + + if (b->get_precision() == precision::complex_fp32) { get_executor()->run(fft::make_fft2( - float_b->get_const_device_view(), dense_x->get_device_view(), + b->template get_const_local_device_view>(), + x->template get_local_device_view>(), fft_size_[0], fft_size_[1], inverse_, buffer_)); } else { - auto dense_b = as>>(b); - auto dense_x = as>>(x); get_executor()->run(fft::make_fft2( - dense_b->get_const_device_view(), dense_x->get_device_view(), + b->template get_const_local_device_view>(), + x->template get_local_device_view>(), fft_size_[0], fft_size_[1], inverse_, buffer_)); } } -void Fft2::apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const +void Fft2::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - if (auto float_x = dynamic_cast>*>(x)) { - auto clone_x = as(as(x)->clone()); - this->apply_impl(b, clone_x.get()); - float_x->scale(beta); - float_x->add_scaled(alpha, clone_x); - } else { - auto dense_x = as>>(x); - auto clone_x = as(as(x)->clone()); - this->apply_impl(b, clone_x.get()); - dense_x->scale(beta); - dense_x->add_scaled(alpha, clone_x); - } + auto clone_x = x->clone(); + this->apply_impl(b, clone_x.get()); + x->scale(beta); + x->add_scaled(alpha, clone_x); } @@ -364,39 +364,34 @@ dim<3> Fft3::get_fft_size() const { return fft_size_; } bool Fft3::is_inverse() const { return inverse_; } -void Fft3::apply_impl(const LinOp* b, LinOp* x) const +void Fft3::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - if (auto float_b = - dynamic_cast>*>(b)) { - auto dense_x = as>>(x); + check_fft_inputs(b, x); + + if (b->get_precision() == precision::complex_fp32) { get_executor()->run(fft::make_fft3( - float_b->get_const_device_view(), dense_x->get_device_view(), + b->template get_const_local_device_view>(), + x->template get_local_device_view>(), fft_size_[0], fft_size_[1], fft_size_[2], inverse_, buffer_)); } else { - auto dense_b = as>>(b); - auto dense_x = as>>(x); get_executor()->run(fft::make_fft3( - dense_b->get_const_device_view(), dense_x->get_device_view(), + b->template get_const_local_device_view>(), + x->template get_local_device_view>(), fft_size_[0], fft_size_[1], fft_size_[2], inverse_, buffer_)); } } -void Fft3::apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const +void Fft3::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - if (auto float_x = dynamic_cast>*>(x)) { - auto clone_x = as(as(x)->clone()); - this->apply_impl(b, clone_x.get()); - float_x->scale(beta); - float_x->add_scaled(alpha, clone_x); - } else { - auto dense_x = as>>(x); - auto clone_x = as(as(x)->clone()); - this->apply_impl(b, clone_x.get()); - dense_x->scale(beta); - dense_x->add_scaled(alpha, clone_x); - } + auto clone_x = x->clone(); + this->apply_impl(b, clone_x.get()); + x->scale(beta); + x->add_scaled(alpha, clone_x); } diff --git a/core/matrix/hybrid.cpp b/core/matrix/hybrid.cpp index b3314e425c1..ccac99410bd 100644 --- a/core/matrix/hybrid.cpp +++ b/core/matrix/hybrid.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -17,6 +16,7 @@ #include "core/base/array_access.hpp" #include "core/base/device_matrix_data_kernels.hpp" +#include "core/base/dispatch_helper.hpp" #include "core/components/absolute_array_kernels.hpp" #include "core/components/fill_array_kernels.hpp" #include "core/components/format_conversion_kernels.hpp" @@ -190,32 +190,28 @@ Hybrid::create(std::shared_ptr exec, template -void Hybrid::apply_impl(const LinOp* b, LinOp* x) const +void Hybrid::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - auto ell_mtx = this->get_ell(); - auto coo_mtx = this->get_coo(); - ell_mtx->apply(dense_b, dense_x); - coo_mtx->apply2(dense_b, dense_x); - }, - b, x); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + ell_->apply(converted_b.get(), converted_x.get()); + coo_->apply2(converted_b.get(), converted_x.get()); } template -void Hybrid::apply_impl(const LinOp* alpha, - const LinOp* b, const LinOp* beta, - LinOp* x) const +void Hybrid::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto ell_mtx = this->get_ell(); - auto coo_mtx = this->get_coo(); - ell_mtx->apply(dense_alpha, dense_b, dense_beta, dense_x); - coo_mtx->apply2(dense_alpha, dense_b, dense_x); - }, - alpha, b, beta, x); + auto converted_alpha = alpha->as_precision(this); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + ell_->apply(converted_alpha.get(), converted_b.get(), beta, + converted_x.get()); + coo_->apply2(converted_alpha.get(), converted_b.get(), converted_x.get()); } diff --git a/core/matrix/identity.cpp b/core/matrix/identity.cpp index 85ec47a9e3d..c096d09101c 100644 --- a/core/matrix/identity.cpp +++ b/core/matrix/identity.cpp @@ -5,8 +5,9 @@ #include "ginkgo/core/matrix/identity.hpp" #include -#include -#include +#include + +#include "core/base/dispatch_helper.hpp" namespace gko { @@ -14,22 +15,25 @@ namespace matrix { template -void Identity::apply_impl(const LinOp* b, LinOp* x) const +void Identity::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { as(x)->copy_from(as(b)); } template -void Identity::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Identity::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, dense_b); - }, - alpha, b, beta, x); + auto dense_alpha = as>(alpha->as_precision(this)); + auto dense_beta = as>(beta->as_precision(this)); + auto converted_x = x->as_precision(this); + + converted_x->scale(dense_beta.get()); + converted_x->add_scaled(dense_alpha.get(), b->as_precision(this).get()); } diff --git a/core/matrix/multivector.cpp b/core/matrix/multivector.cpp index 97e465dc155..fc4e4ebdc4c 100644 --- a/core/matrix/multivector.cpp +++ b/core/matrix/multivector.cpp @@ -12,7 +12,6 @@ #include #include #include -#include #include #include #include @@ -928,11 +927,13 @@ void MultiVector::row_gather( template void MultiVector::row_gather( - ptr_param alpha, const array* gather_indices, - ptr_param beta, ptr_param out) const + ptr_param alpha, + const array* gather_indices, + ptr_param beta, + ptr_param out) const { - auto dense_alpha = make_temporary_conversion(alpha); - auto dense_beta = make_temporary_conversion(beta); + auto dense_alpha = as(alpha->as_precision(this)); + auto dense_beta = as(beta->as_precision(this)); GKO_ASSERT_EQUAL_DIMENSIONS(dense_alpha, gko::dim<2>(1, 1)); GKO_ASSERT_EQUAL_DIMENSIONS(dense_beta, gko::dim<2>(1, 1)); gather_mixed_real_complex( @@ -945,11 +946,13 @@ void MultiVector::row_gather( template void MultiVector::row_gather( - ptr_param alpha, const array* gather_indices, - ptr_param beta, ptr_param out) const + ptr_param alpha, + const array* gather_indices, + ptr_param beta, + ptr_param out) const { - auto dense_alpha = make_temporary_conversion(alpha); - auto dense_beta = make_temporary_conversion(beta); + auto dense_alpha = as(alpha->as_precision(this)); + auto dense_beta = as(beta->as_precision(this)); GKO_ASSERT_EQUAL_DIMENSIONS(dense_alpha, gko::dim<2>(1, 1)); GKO_ASSERT_EQUAL_DIMENSIONS(dense_beta, gko::dim<2>(1, 1)); gather_mixed_real_complex( diff --git a/core/matrix/permutation.cpp b/core/matrix/permutation.cpp index d30f08cb7a4..e9f0b3cf7b3 100644 --- a/core/matrix/permutation.cpp +++ b/core/matrix/permutation.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include @@ -262,44 +261,37 @@ void Permutation::write( } -template -void dispatch_dense(const LinOp* op, Functor fn) -{ - using matrix::MultiVector; - using std::complex; - run, -#endif -#if GINKGO_ENABLE_BFLOAT16 - gko::bfloat16, std::complex, -#endif - double, float, std::complex, std::complex>(op, fn); -} - - template -void Permutation::apply_impl(const LinOp* in, LinOp* out) const +void Permutation::apply_impl(const AbstractMultiVector* in, + AbstractMultiVector* out) const { - dispatch_dense(in, [&](auto dense_in) { - auto dense_out = make_temporary_conversion< - typename gko::detail::pointee::value_type>(out); - dense_in->permute(this, dense_out.get(), permute_mode::rows); - }); + std::visit( + [this, in, out](auto p) { + using MultiVectorType = MultiVector>; + as(in)->permute( + this, as(out->as_precision(in)).get(), + permute_mode::rows); + }, + precision_to_variant(in->get_precision())); } template -void Permutation::apply_impl(const LinOp* alpha, const LinOp* in, - const LinOp* beta, LinOp* out) const +void Permutation::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* in, + const AbstractMultiVector* beta, + AbstractMultiVector* out) const { - dispatch_dense(in, [&](auto dense_in) { - auto dense_out = make_temporary_conversion< - typename gko::detail::pointee::value_type>(out); - auto tmp = dense_in->permute(this, permute_mode::rows); - dense_out->scale(beta); - dense_out->add_scaled(alpha, tmp); - }); + std::visit( + [this, in, out, alpha, beta](auto p) { + using MultiVectorType = MultiVector>; + auto converted_out = as(out->as_precision(in)); + auto tmp = + as(in)->permute(this, permute_mode::rows); + converted_out->scale(beta); + converted_out->add_scaled(alpha, tmp); + }, + precision_to_variant(in->get_precision())); } diff --git a/core/matrix/row_gatherer.cpp b/core/matrix/row_gatherer.cpp index 0ea350acde1..e5369d19bc8 100644 --- a/core/matrix/row_gatherer.cpp +++ b/core/matrix/row_gatherer.cpp @@ -63,33 +63,30 @@ RowGatherer::create_const( template -void RowGatherer::apply_impl(const LinOp* in, LinOp* out) const +void RowGatherer::apply_impl(const AbstractMultiVector* in, + AbstractMultiVector* out) const { - run, -#endif -#if GINKGO_ENABLE_BFLOAT16 - gko::bfloat16, std::complex, -#endif - float, double, std::complex, std::complex>( - in, [&](auto gather) { gather->row_gather(&row_idxs_, out); }); + std::visit( + [this, in, out](auto p) { + using value_type = std::decay_t; + as>(in)->row_gather(&row_idxs_, out); + }, + precision_to_variant(in->get_precision())); } template -void RowGatherer::apply_impl(const LinOp* alpha, const LinOp* in, - const LinOp* beta, LinOp* out) const +void RowGatherer::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* in, + const AbstractMultiVector* beta, + AbstractMultiVector* out) const { - run, -#endif -#if GINKGO_ENABLE_BFLOAT16 - gko::bfloat16, std::complex, -#endif - float, double, std::complex, std::complex>( - in, - [&](auto gather) { gather->row_gather(alpha, &row_idxs_, beta, out); }); + std::visit( + [this, in, alpha, beta, out](auto p) { + using value_type = std::decay_t; + as>(in)->row_gather(alpha, &row_idxs_, beta, + out); + }, + precision_to_variant(in->get_precision())); } diff --git a/core/matrix/scaled_permutation.cpp b/core/matrix/scaled_permutation.cpp index a6227f6ddaa..a58fa29396f 100644 --- a/core/matrix/scaled_permutation.cpp +++ b/core/matrix/scaled_permutation.cpp @@ -6,8 +6,8 @@ #include #include -#include +#include "core/base/dispatch_helper.hpp" #include "core/matrix/scaled_permutation_kernels.hpp" @@ -38,7 +38,8 @@ ScaledPermutation::ScaledPermutation( std::shared_ptr exec, array scaling_factors, array permutation_indices) : LinOp(exec, - dim<2>{scaling_factors.get_size(), scaling_factors.get_size()}), + dim<2>{scaling_factors.get_size(), scaling_factors.get_size()}, + precision_v), scale_{exec, std::move(scaling_factors)}, permutation_{exec, std::move(permutation_indices)} { @@ -128,30 +129,27 @@ ScaledPermutation::compose( template -void ScaledPermutation::apply_impl(const LinOp* b, - LinOp* x) const +void ScaledPermutation::apply_impl( + const AbstractMultiVector* b, AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - dense_b->scale_permute(this, dense_x, permute_mode::rows); - }, - b, x); + using dense_type = MultiVector; + as(b->as_precision(this)) + ->scale_permute(this, as(x->as_precision(this).get()), + permute_mode::rows); } template -void ScaledPermutation::apply_impl(const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - LinOp* x) const +void ScaledPermutation::apply_impl( + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto tmp = dense_b->scale_permute(this, permute_mode::rows); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, tmp); - }, - alpha, b, beta, x); + using dense_type = MultiVector; + auto tmp = as(b->as_precision(this)) + ->scale_permute(this, permute_mode::rows); + auto converted_x = as(x->as_precision(this)); + converted_x->scale(beta); + converted_x->add_scaled(alpha, tmp); } diff --git a/core/matrix/sellp.cpp b/core/matrix/sellp.cpp index 7e0d5c3a7b2..67b29221906 100644 --- a/core/matrix/sellp.cpp +++ b/core/matrix/sellp.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -16,6 +15,7 @@ #include "core/base/allocator.hpp" #include "core/base/array_access.hpp" #include "core/base/device_matrix_data_kernels.hpp" +#include "core/base/dispatch_helper.hpp" #include "core/components/absolute_array_kernels.hpp" #include "core/components/fill_array_kernels.hpp" #include "core/components/format_conversion_kernels.hpp" @@ -179,29 +179,31 @@ auto Sellp::get_const_device_view() const template -void Sellp::apply_impl(const LinOp* b, LinOp* x) const +void Sellp::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { + apply_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { this->get_executor()->run(sellp::make_spmv( - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_x->get_device_view())); + this->get_const_device_view(), view_b, view_x)); }, b, x); } template -void Sellp::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Sellp::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { + apply_precision_dispatch( + [this](auto dense_alpha, auto view_b, auto dense_beta, auto view_x, + auto...) { this->get_executor()->run(sellp::make_advanced_spmv( dense_alpha->get_const_device_view(), - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); + this->get_const_device_view(), view_b, + dense_beta->get_const_device_view(), view_x)); }, alpha, b, beta, x); } diff --git a/core/matrix/sparsity_csr.cpp b/core/matrix/sparsity_csr.cpp index f5551e7847c..a1cb2042523 100644 --- a/core/matrix/sparsity_csr.cpp +++ b/core/matrix/sparsity_csr.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -15,6 +14,7 @@ #include "core/base/array_access.hpp" #include "core/base/device_matrix_data_kernels.hpp" +#include "core/base/dispatch_helper.hpp" #include "core/components/format_conversion_kernels.hpp" #include "core/matrix/sparsity_csr_kernels.hpp" @@ -45,37 +45,31 @@ GKO_REGISTER_OPERATION(is_sorted_by_column_index, template -void SparsityCsr::apply_impl(const LinOp* b, - LinOp* x) const +void SparsityCsr::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - mixed_precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { + apply_mixed_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { this->get_executor()->run( - sparsity_csr::make_spmv(this, dense_b->get_const_device_view(), - dense_x->get_device_view())); + sparsity_csr::make_spmv(this, view_b, view_x)); }, b, x); } template -void SparsityCsr::apply_impl(const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - LinOp* x) const +void SparsityCsr::apply_impl( + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const { - mixed_precision_dispatch_real_complex( - [this, alpha, beta](auto dense_b, auto dense_x) { - auto dense_alpha = make_temporary_conversion(alpha); - auto dense_beta = make_temporary_conversion< - typename std::decay_t::value_type>(beta); + apply_mixed_precision_dispatch( + [this](auto dense_alpha, auto view_b, auto dense_beta, auto view_x, + auto...) { this->get_executor()->run(sparsity_csr::make_advanced_spmv( - dense_alpha->get_const_device_view(), this, - dense_b->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); + dense_alpha->get_const_device_view(), this, view_b, + dense_beta->get_const_device_view(), view_x)); }, - b, x); + alpha, b, beta, x); } diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp index 0f61f6c1584..48749964fb0 100644 --- a/include/ginkgo/core/matrix/coo.hpp +++ b/include/ginkgo/core/matrix/coo.hpp @@ -239,33 +239,16 @@ class Coo : public LinOp, * * @param b the input vector(s) on which the operator is applied * @param x the output vector(s) where the result is stored - * - * @return this - */ - void apply2(ptr_param b, ptr_param x); - - /** - * @copydoc apply2(cost LinOp *, LinOp *) - */ - void apply2(ptr_param b, ptr_param x) const; - - /** - * Performs the operation x = alpha * Coo * b + x. - * - * @param alpha scaling of the result of Coo * b - * @param b vector(s) on which the operator is applied - * @param x output vector(s) - * - * @return this */ - void apply2(ptr_param alpha, ptr_param b, - ptr_param x); + void apply2(ptr_param b, + ptr_param x) const; /** * @copydoc apply2(const LinOp *, const LinOp *, LinOp *) */ - void apply2(ptr_param alpha, ptr_param b, - ptr_param x) const; + void apply2(ptr_param alpha, + ptr_param b, + ptr_param x) const; /** * Creates an uninitialized COO matrix of the specified size. @@ -362,14 +345,20 @@ class Coo : public LinOp, */ void resize(dim<2> new_size, size_type nnz); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; - void apply2_impl(const LinOp* b, LinOp* x) const; + void apply2_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const; - void apply2_impl(const LinOp* alpha, const LinOp* b, LinOp* x) const; + void apply2_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + AbstractMultiVector* x) const; private: array values_; diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp index 723b27aaed9..f18a4e26edf 100644 --- a/include/ginkgo/core/matrix/csr.hpp +++ b/include/ginkgo/core/matrix/csr.hpp @@ -921,27 +921,17 @@ class Csr : public LinOp, * Scales the matrix with a scalar. * * @param alpha The entire matrix is scaled by alpha. alpha has to be a 1x1 - * MultiVector. + * MultiVector matrix. */ - void scale(ptr_param alpha) - { - auto exec = this->get_executor(); - GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1)); - this->scale_impl(make_temporary_clone(exec, alpha).get()); - } + void scale(ptr_param alpha); /** * Scales the matrix with the inverse of a scalar. * * @param alpha The entire matrix is scaled by 1 / alpha. alpha has to be a - * 1x1 MultiVector. + * 1x1 MultiVector matrix. */ - void inv_scale(ptr_param alpha) - { - auto exec = this->get_executor(); - GKO_ASSERT_EQUAL_DIMENSIONS(alpha, dim<2>(1, 1)); - this->inv_scale_impl(make_temporary_clone(exec, alpha).get()); - } + void inv_scale(ptr_param alpha); /** * Creates an uninitialized CSR matrix of the specified size. @@ -1138,10 +1128,13 @@ class Csr : public LinOp, array row_ptrs, csr::spmv_strategy strategy = csr::spmv_strategy::automatic); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; /** * Computes srow. It should be run after changing any row_ptrs_ value. @@ -1154,7 +1147,7 @@ class Csr : public LinOp, * @note Other implementations of Csr should override this function * instead of scale(const LinOp *alpha). */ - virtual void scale_impl(const LinOp* alpha); + virtual void scale_impl(const AbstractMultiVector* alpha); /** * @copydoc inv_scale(const LinOp *) @@ -1162,7 +1155,7 @@ class Csr : public LinOp, * @note Other implementations of Csr should override this function * instead of inv_scale(const LinOp *alpha). */ - virtual void inv_scale_impl(const LinOp* alpha); + virtual void inv_scale_impl(const AbstractMultiVector* alpha); /** * Returns the actual strategy. When the strategy is automatic, this diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp index 6c548898fb7..4696f85c0f8 100644 --- a/include/ginkgo/core/matrix/dense.hpp +++ b/include/ginkgo/core/matrix/dense.hpp @@ -287,10 +287,10 @@ class Dense : public LinOp, */ void conj_transpose(ptr_param output) const; - void add_scaled(ptr_param alpha, + void add_scaled(ptr_param alpha, ptr_param> diag); - void sub_scaled(ptr_param alpha, + void sub_scaled(ptr_param alpha, ptr_param> diag); [[nodiscard]] static std::unique_ptr create( @@ -352,10 +352,13 @@ class Dense : public LinOp, Dense(std::shared_ptr exec, const dim<2>& size, array values, size_type stride); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; [[nodiscard]] size_type linearize_index(size_type row, size_type col) const noexcept; diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp index 3faa2c49c80..16a3fb25f57 100644 --- a/include/ginkgo/core/matrix/diagonal.hpp +++ b/include/ginkgo/core/matrix/diagonal.hpp @@ -59,6 +59,7 @@ class Diagonal GKO_ASSERT_SUPPORTED_VALUE_TYPE; public: + using LinOp::apply; using EnableCloneable::convert_to; using EnableCloneable::move_to; using ConvertibleTo>::convert_to; @@ -146,14 +147,14 @@ class Diagonal * @param b the input vector(s) on which the diagonal matrix is applied * @param x the output vector(s) where the result is stored */ - void rapply(ptr_param b, ptr_param x) const - { - GKO_ASSERT_REVERSE_CONFORMANT(this, b); - GKO_ASSERT_EQUAL_ROWS(b, x); - GKO_ASSERT_EQUAL_COLS(this, x); + void rapply(ptr_param b, + ptr_param x) const; - this->rapply_impl(b.get(), x.get()); - } + void rapply(ptr_param> b, + ptr_param> x) const; + + void rapply(ptr_param> b, + ptr_param> x) const; /** * Applies the inverse of the diagonal matrix to a matrix b, @@ -164,14 +165,20 @@ class Diagonal * is applied * @param x the output vector(s) where the result is stored */ - void inverse_apply(ptr_param b, ptr_param x) const - { - GKO_ASSERT_CONFORMANT(this, b); - GKO_ASSERT_EQUAL_ROWS(b, x); - GKO_ASSERT_EQUAL_ROWS(this, x); + void inverse_apply(ptr_param b, + ptr_param x) const; - this->inverse_apply_impl(b.get(), x.get()); - } + void inverse_apply(ptr_param> b, + ptr_param> x) const; + + void inverse_apply(ptr_param> b, + ptr_param> x) const; + + void apply(ptr_param> b, + ptr_param> x) const; + + void apply(ptr_param> b, + ptr_param> x) const; void read(const mat_data& data) override; @@ -253,14 +260,13 @@ class Diagonal Diagonal(std::shared_ptr exec, const size_type size, array values); - void apply_impl(const LinOp* b, LinOp* x) const override; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; - - void rapply_impl(const LinOp* b, LinOp* x) const; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void inverse_apply_impl(const LinOp* b, LinOp* x) const; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: array values_; diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp index f213f8c9946..0d6fc00cf1b 100644 --- a/include/ginkgo/core/matrix/ell.hpp +++ b/include/ginkgo/core/matrix/ell.hpp @@ -388,10 +388,13 @@ class Ell : public LinOp, */ void resize(dim<2> new_size, size_type max_row_nnz); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; size_type linearize_index(size_type row, size_type col) const noexcept { diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp index 83ea205ffb5..173e85ac481 100644 --- a/include/ginkgo/core/matrix/fbcsr.hpp +++ b/include/ginkgo/core/matrix/fbcsr.hpp @@ -470,10 +470,13 @@ class Fbcsr int block_size, array values, array col_idxs, array row_ptrs); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: int bs_; ///< Block size diff --git a/include/ginkgo/core/matrix/fft.hpp b/include/ginkgo/core/matrix/fft.hpp index ab897a04de9..13f2e941f04 100644 --- a/include/ginkgo/core/matrix/fft.hpp +++ b/include/ginkgo/core/matrix/fft.hpp @@ -100,10 +100,13 @@ class Fft : public LinOp, Fft(std::shared_ptr exec, size_type size = 0, bool inverse = false); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: mutable array buffer_; @@ -210,10 +213,13 @@ class Fft2 : public LinOp, Fft2(std::shared_ptr exec, size_type size1 = 0, size_type size2 = 0, bool inverse = false); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: mutable array buffer_; @@ -324,10 +330,13 @@ class Fft3 : public LinOp, Fft3(std::shared_ptr exec, size_type size1 = 0, size_type size2 = 0, size_type size3 = 0, bool inverse = false); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: mutable array buffer_; diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp index e47da1786c9..3ac0dd6e267 100644 --- a/include/ginkgo/core/matrix/hybrid.hpp +++ b/include/ginkgo/core/matrix/hybrid.hpp @@ -790,10 +790,13 @@ class Hybrid */ void resize(dim<2> new_size, size_type ell_row_nnz, size_type coo_nnz); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: std::unique_ptr ell_; diff --git a/include/ginkgo/core/matrix/identity.hpp b/include/ginkgo/core/matrix/identity.hpp index a43405bc827..232267025dc 100644 --- a/include/ginkgo/core/matrix/identity.hpp +++ b/include/ginkgo/core/matrix/identity.hpp @@ -69,10 +69,13 @@ class Identity : public LinOp, protected: explicit Identity(std::shared_ptr exec, size_type size = 0); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; }; diff --git a/include/ginkgo/core/matrix/multivector.hpp b/include/ginkgo/core/matrix/multivector.hpp index a5655e245ad..e190486bd71 100644 --- a/include/ginkgo/core/matrix/multivector.hpp +++ b/include/ginkgo/core/matrix/multivector.hpp @@ -494,18 +494,18 @@ class MultiVector * It must have the same number of columns as this * matrix and `gather_indices->get_size()` rows. */ - void row_gather(ptr_param alpha, + void row_gather(ptr_param alpha, const array* gather_indices, - ptr_param beta, + ptr_param beta, ptr_param row_collection) const; /** * @copydoc row_gather(const LinOp*, const array*, const LinOp*, * LinOp*) const */ - void row_gather(ptr_param alpha, + void row_gather(ptr_param alpha, const array* gather_indices, - ptr_param beta, + ptr_param beta, ptr_param row_collection) const; std::unique_ptr column_permute( diff --git a/include/ginkgo/core/matrix/permutation.hpp b/include/ginkgo/core/matrix/permutation.hpp index 45ebce4957d..91ac30f545a 100644 --- a/include/ginkgo/core/matrix/permutation.hpp +++ b/include/ginkgo/core/matrix/permutation.hpp @@ -265,10 +265,12 @@ class Permutation : public LinOp, Permutation(std::shared_ptr exec, array permutation_indices); - void apply_impl(const LinOp* in, LinOp* out) const override; + void apply_impl(const AbstractMultiVector* in, + AbstractMultiVector* out) const override; - void apply_impl(const LinOp*, const LinOp* in, const LinOp*, - LinOp* out) const override; + void apply_impl(const AbstractMultiVector*, const AbstractMultiVector* in, + const AbstractMultiVector*, + AbstractMultiVector* out) const override; private: array permutation_; diff --git a/include/ginkgo/core/matrix/row_gatherer.hpp b/include/ginkgo/core/matrix/row_gatherer.hpp index bcde2b54496..d21466d6ed5 100644 --- a/include/ginkgo/core/matrix/row_gatherer.hpp +++ b/include/ginkgo/core/matrix/row_gatherer.hpp @@ -115,10 +115,13 @@ class RowGatherer : public LinOp, RowGatherer(std::shared_ptr exec, const dim<2>& size, array row_idxs); - void apply_impl(const LinOp* in, LinOp* out) const override; + void apply_impl(const AbstractMultiVector* in, + AbstractMultiVector* out) const override; - void apply_impl(const LinOp* alpha, const LinOp* in, const LinOp* beta, - LinOp* out) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* in, + const AbstractMultiVector* beta, + AbstractMultiVector* out) const override; private: gko::array row_idxs_; diff --git a/include/ginkgo/core/matrix/scaled_permutation.hpp b/include/ginkgo/core/matrix/scaled_permutation.hpp index 3e68dfab210..31eb8abf6a6 100644 --- a/include/ginkgo/core/matrix/scaled_permutation.hpp +++ b/include/ginkgo/core/matrix/scaled_permutation.hpp @@ -163,10 +163,12 @@ class ScaledPermutation final array scaling_factors, array permutation_indices); - void apply_impl(const LinOp* in, LinOp* out) const override; + void apply_impl(const AbstractMultiVector* in, + AbstractMultiVector* out) const override; - void apply_impl(const LinOp*, const LinOp* in, const LinOp*, - LinOp* out) const override; + void apply_impl(const AbstractMultiVector*, const AbstractMultiVector* in, + const AbstractMultiVector*, + AbstractMultiVector* out) const override; array scale_; array permutation_; diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp index 52dd077caaf..e142ce0d34c 100644 --- a/include/ginkgo/core/matrix/sellp.hpp +++ b/include/ginkgo/core/matrix/sellp.hpp @@ -382,10 +382,13 @@ class Sellp Sellp(std::shared_ptr exec, const dim<2>& size, size_type slice_size, size_type stride_factor, size_type total_cols); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; size_type linearize_index(size_type row, size_type slice_set, size_type col) const noexcept diff --git a/include/ginkgo/core/matrix/sparsity_csr.hpp b/include/ginkgo/core/matrix/sparsity_csr.hpp index 36bfa0cbdb7..674c7057112 100644 --- a/include/ginkgo/core/matrix/sparsity_csr.hpp +++ b/include/ginkgo/core/matrix/sparsity_csr.hpp @@ -311,10 +311,13 @@ class SparsityCsr : public LinOp, SparsityCsr(std::shared_ptr exec, std::shared_ptr matrix); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: array col_idxs_; From d0dbe5ace58b36cb53d9e3ae0f7cac4b24154442 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:02:28 +0200 Subject: [PATCH 10/21] fix hierarchy change for base operators --- core/base/block_operator.cpp | 38 ++------ core/base/combination.cpp | 66 ++++++------- core/base/composition.cpp | 60 ++++++------ core/base/perturbation.cpp | 102 +++++++++++++------- core/test/base/combination.cpp | 2 + include/ginkgo/core/base/block_operator.hpp | 9 +- include/ginkgo/core/base/combination.hpp | 33 ++++--- include/ginkgo/core/base/composition.hpp | 9 +- include/ginkgo/core/base/perturbation.hpp | 53 +++++----- 9 files changed, 197 insertions(+), 175 deletions(-) diff --git a/core/base/block_operator.cpp b/core/base/block_operator.cpp index cde7675f237..b52b842a9f5 100644 --- a/core/base/block_operator.cpp +++ b/core/base/block_operator.cpp @@ -6,7 +6,6 @@ #include -#include #include #include "core/base/dispatch_helper.hpp" @@ -16,33 +15,13 @@ namespace gko { namespace { -template -auto dispatch_dense(Fn&& fn, LinOp* v) -{ - return run, -#endif -#if GINKGO_ENABLE_BFLOAT16 - bfloat16, std::complex, -#endif - std::complex, std::complex>(v, - std::forward(fn)); -} - - -template -auto create_vector_blocks(LinOpType* vector, +template +auto create_vector_blocks(MaybeConstMultiVector* vector, const std::vector& spans) { return [=](size_type i) { - return dispatch_dense( - [&](auto* dense) -> std::unique_ptr { - GKO_ENSURE_IN_BOUNDS(i, spans.size()); - return dense->create_submatrix(spans[i], - {0, dense->get_size()[1]}); - }, - const_cast(vector)); + GKO_ENSURE_IN_BOUNDS(i, spans.size()); + return vector->create_subview(spans[i], {0, vector->get_size()[1]}); }; } @@ -195,7 +174,8 @@ void init_one_cache(std::shared_ptr exec, } -void BlockOperator::apply_impl(const LinOp* b, LinOp* x) const +void BlockOperator::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { auto block_b = create_vector_blocks(b, col_spans_); auto block_x = create_vector_blocks(x, row_spans_); @@ -219,8 +199,10 @@ void BlockOperator::apply_impl(const LinOp* b, LinOp* x) const } -void BlockOperator::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void BlockOperator::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { auto block_b = create_vector_blocks(b, col_spans_); auto block_x = create_vector_blocks(x, row_spans_); diff --git a/core/base/combination.cpp b/core/base/combination.cpp index d9ea8011252..4d587eb41ec 100644 --- a/core/base/combination.cpp +++ b/core/base/combination.cpp @@ -4,7 +4,6 @@ #include "ginkgo/core/base/combination.hpp" -#include #include #include @@ -14,9 +13,10 @@ namespace { template -inline void initialize_scalars(std::shared_ptr exec, - std::unique_ptr& zero, - std::unique_ptr& one) +inline void initialize_scalars( + std::shared_ptr exec, + std::unique_ptr>& zero, + std::unique_ptr>& one) { if (zero == nullptr) { zero = initialize>( @@ -100,8 +100,7 @@ std::unique_ptr Combination::transpose() const transposed->set_size(gko::transpose(this->get_size())); // copy coefficients for (auto& coef : get_coefficients()) { - transposed->coefficients_.push_back( - share(as(as(coef)->clone()))); + transposed->coefficients_.push_back(share(coef->clone())); } // transpose operators for (auto& op : get_operators()) { @@ -121,7 +120,7 @@ std::unique_ptr Combination::conj_transpose() const // conjugate coefficients! for (auto& coef : get_coefficients()) { transposed->coefficients_.push_back( - share(as(coef)->conj_transpose())); + share(as>(coef)->conj_transpose())); } // conjugate-transpose operators for (auto& op : get_operators()) { @@ -134,38 +133,39 @@ std::unique_ptr Combination::conj_transpose() const template -void Combination::apply_impl(const LinOp* b, LinOp* x) const +void Combination::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - initialize_scalars(this->get_executor(), cache_.zero, - cache_.one); - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - operators_[0]->apply(coefficients_[0], dense_b, cache_.zero, - dense_x); - for (size_type i = 1; i < operators_.size(); ++i) { - operators_[i]->apply(coefficients_[i], dense_b, cache_.one, - dense_x); - } - }, - b, x); + initialize_scalars(this->get_executor(), cache_.zero, cache_.one); + + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + operators_[0]->apply(coefficients_[0], dense_b, cache_.zero, dense_x); + for (size_type i = 1; i < operators_.size(); ++i) { + operators_[i]->apply(coefficients_[i], dense_b, cache_.one, dense_x); + } } template -void Combination::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Combination::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - if (cache_.intermediate_x == nullptr || - cache_.intermediate_x->get_size() != dense_x->get_size()) { - cache_.intermediate_x = dense_x->clone(); - } - this->apply_impl(dense_b, cache_.intermediate_x.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, cache_.intermediate_x); - }, - alpha, b, beta, x); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + if (cache_.intermediate_x == nullptr || + cache_.intermediate_x->get_size() != dense_x->get_size()) { + cache_.intermediate_x = dense_x->clone(); + } + this->apply_impl(dense_b, cache_.intermediate_x.get()); + dense_x->scale(beta); + dense_x->add_scaled(alpha, cache_.intermediate_x); } diff --git a/core/base/composition.cpp b/core/base/composition.cpp index 706699d28f6..7e9046cdb5d 100644 --- a/core/base/composition.cpp +++ b/core/base/composition.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include "core/components/fill_array_kernels.hpp" @@ -26,9 +25,9 @@ GKO_REGISTER_OPERATION(fill_array, components::fill_array); template -std::unique_ptr apply_inner_operators( +std::unique_ptr apply_inner_operators( const std::vector>& operators, - array& storage, const LinOp* rhs) + array& storage, const AbstractMultiVector* rhs) { using MultiVector = matrix::MultiVector; // determine amount of necessary storage: @@ -186,38 +185,41 @@ std::unique_ptr Composition::conj_transpose() const template -void Composition::apply_impl(const LinOp* b, LinOp* x) const +void Composition::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - if (operators_.size() > 1) { - operators_[0]->apply( - apply_inner_operators(operators_, storage_, dense_b), - dense_x); - } else { - operators_[0]->apply(dense_b, dense_x); - } - }, - b, x); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + if (operators_.size() > 1) { + operators_[0]->apply( + apply_inner_operators(operators_, storage_, dense_b), dense_x); + } else { + operators_[0]->apply(dense_b, dense_x); + } } template -void Composition::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Composition::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - if (operators_.size() > 1) { - operators_[0]->apply( - dense_alpha, - apply_inner_operators(operators_, storage_, dense_b), - dense_beta, dense_x); - } else { - operators_[0]->apply(dense_alpha, dense_b, dense_beta, dense_x); - } - }, - alpha, b, beta, x); + auto converted_alpha = alpha->as_precision(this); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_alpha = converted_alpha.get(); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + if (operators_.size() > 1) { + operators_[0]->apply( + dense_alpha, apply_inner_operators(operators_, storage_, dense_b), + beta, dense_x); + } else { + operators_[0]->apply(dense_alpha, dense_b, beta, dense_x); + } } diff --git a/core/base/perturbation.cpp b/core/base/perturbation.cpp index 45bfa8f521a..05103d89c7a 100644 --- a/core/base/perturbation.cpp +++ b/core/base/perturbation.cpp @@ -4,8 +4,8 @@ #include "ginkgo/core/base/perturbation.hpp" -#include #include +#include namespace gko { @@ -73,8 +73,9 @@ Perturbation::Perturbation(std::shared_ptr exec) template -Perturbation::Perturbation(std::shared_ptr scalar, - std::shared_ptr basis) +Perturbation::Perturbation( + std::shared_ptr> scalar, + std::shared_ptr basis) : Perturbation(std::move(scalar), // basis can not be std::move(basis). Otherwise, Program // deletes basis before applying conjugate transpose @@ -84,9 +85,9 @@ Perturbation::Perturbation(std::shared_ptr scalar, template -Perturbation::Perturbation(std::shared_ptr scalar, - std::shared_ptr basis, - std::shared_ptr projector) +Perturbation::Perturbation( + std::shared_ptr> scalar, + std::shared_ptr basis, std::shared_ptr projector) : LinOp(basis->get_executor(), gko::dim<2>{basis->get_size()[0]}, precision_v), basis_{std::move(basis)}, @@ -107,7 +108,8 @@ std::unique_ptr> Perturbation::create( template std::unique_ptr> Perturbation::create( - std::shared_ptr scalar, std::shared_ptr basis) + std::shared_ptr> scalar, + std::shared_ptr basis) { return std::unique_ptr{new Perturbation{scalar, basis}}; } @@ -115,8 +117,8 @@ std::unique_ptr> Perturbation::create( template std::unique_ptr> Perturbation::create( - std::shared_ptr scalar, std::shared_ptr basis, - std::shared_ptr projector) + std::shared_ptr> scalar, + std::shared_ptr basis, std::shared_ptr projector) { return std::unique_ptr{ new Perturbation{scalar, basis, projector}}; @@ -133,29 +135,50 @@ void Perturbation::validate_perturbation() template -void Perturbation::apply_impl(const LinOp* b, LinOp* x) const +void Perturbation::cache_struct::allocate( + std::shared_ptr exec, dim<2> size) + +{ + using vec = gko::matrix::MultiVector; + if (one == nullptr) { + one = initialize({gko::one()}, exec); + } + if (alpha_scalar == nullptr) { + alpha_scalar = vec::create(exec, gko::dim<2>(1)); + } + if (intermediate == nullptr || intermediate->get_size() != size) { + intermediate = vec::create(exec, size); + } +} + + +template +void Perturbation::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { // x = (I + scalar * basis * projector) * b // temp = projector * b : projector->apply(b, temp) // x = b : x->copy_from(b) // x = 1 * x + scalar * basis * temp : basis->apply(scalar, temp, 1, x) - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - auto exec = this->get_executor(); - auto intermediate_size = - gko::dim<2>(projector_->get_size()[0], dense_b->get_size()[1]); - cache_.allocate(exec, intermediate_size); - projector_->apply(dense_b, cache_.intermediate); - dense_x->copy_from(dense_b); - basis_->apply(scalar_, cache_.intermediate, cache_.one, dense_x); - }, - b, x); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + auto exec = this->get_executor(); + auto intermediate_size = + gko::dim<2>(projector_->get_size()[0], dense_b->get_size()[1]); + cache_.allocate(exec, intermediate_size); + projector_->apply(dense_b, cache_.intermediate); + dense_x->copy_from(dense_b); + basis_->apply(scalar_, cache_.intermediate, cache_.one, dense_x); } template -void Perturbation::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Perturbation::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { // x = alpha * (I + scalar * basis * projector) b + beta * x // = beta * x + alpha * b + alpha * scalar * basis * projector * b @@ -164,20 +187,25 @@ void Perturbation::apply_impl(const LinOp* alpha, const LinOp* b, // x->add_scaled(alpha, b) // x = x + alpha * scalar * basis * temp // : basis->apply(alpha * scalar, temp, 1, x) - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto exec = this->get_executor(); - auto intermediate_size = - gko::dim<2>(projector_->get_size()[0], dense_b->get_size()[1]); - cache_.allocate(exec, intermediate_size); - projector_->apply(dense_b, cache_.intermediate); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, dense_b); - dense_alpha->apply(scalar_, cache_.alpha_scalar); - basis_->apply(cache_.alpha_scalar, cache_.intermediate, cache_.one, - dense_x); - }, - alpha, b, beta, x); + auto converted_alpha = alpha->as_precision(this); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_alpha = + as>(converted_alpha.get()); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + + auto exec = this->get_executor(); + auto intermediate_size = + gko::dim<2>(projector_->get_size()[0], dense_b->get_size()[1]); + cache_.allocate(exec, intermediate_size); + projector_->apply(dense_b, cache_.intermediate); + dense_x->scale(beta); + dense_x->add_scaled(dense_alpha, dense_b); + cache_.alpha_scalar->copy_from(dense_alpha); + cache_.alpha_scalar->scale(scalar_); + basis_->apply(cache_.alpha_scalar, cache_.intermediate, cache_.one, + dense_x); } diff --git a/core/test/base/combination.cpp b/core/test/base/combination.cpp index c22cdae7e58..29522edc704 100644 --- a/core/test/base/combination.cpp +++ b/core/test/base/combination.cpp @@ -30,6 +30,8 @@ struct DummyOperator : public gko::LinOp { template class Combination : public ::testing::Test { protected: + using MultiVector = gko::matrix::MultiVector; + Combination() : exec{gko::ReferenceExecutor::create()}, operators{std::make_shared(exec), diff --git a/include/ginkgo/core/base/block_operator.hpp b/include/ginkgo/core/base/block_operator.hpp index 5225ddb4646..6b1f02c916f 100644 --- a/include/ginkgo/core/base/block_operator.hpp +++ b/include/ginkgo/core/base/block_operator.hpp @@ -128,10 +128,13 @@ class BlockOperator final : public LinOp, std::shared_ptr exec, std::vector>> blocks); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; dim<2> block_size_; std::vector row_spans_; diff --git a/include/ginkgo/core/base/combination.hpp b/include/ginkgo/core/base/combination.hpp index 32d22487845..01de4d098ad 100644 --- a/include/ginkgo/core/base/combination.hpp +++ b/include/ginkgo/core/base/combination.hpp @@ -10,6 +10,7 @@ #include #include +#include namespace gko { @@ -43,8 +44,8 @@ class Combination : public LinOp, * * @return a list of coefficients */ - const std::vector>& get_coefficients() - const noexcept + const std::vector>>& + get_coefficients() const noexcept { return coefficients_; } @@ -95,8 +96,9 @@ class Combination : public LinOp, void add_operators() {} template - void add_operators(std::shared_ptr coef, - std::shared_ptr oper, Rest&&... rest) + void add_operators( + std::shared_ptr> coef, + std::shared_ptr oper, Rest&&... rest) { GKO_ASSERT_EQUAL_DIMENSIONS(coef, dim<2>(1, 1)); GKO_ASSERT_EQUAL_DIMENSIONS(oper, this->get_size()); @@ -174,8 +176,9 @@ class Combination : public LinOp, * @param rest other coefficient and operators (interleaved) */ template - explicit Combination(std::shared_ptr coef, - std::shared_ptr oper, Rest&&... rest) + explicit Combination( + std::shared_ptr> coef, + std::shared_ptr oper, Rest&&... rest) : Combination(oper->get_executor()) { this->set_size(oper->get_size()); @@ -183,13 +186,17 @@ class Combination : public LinOp, std::forward(rest)...); } - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: - std::vector> coefficients_; + std::vector>> + coefficients_; std::vector> operators_; // TODO: solve race conditions when multithreading @@ -199,9 +206,9 @@ class Combination : public LinOp, cache_struct(const cache_struct& other) {} cache_struct& operator=(const cache_struct& other) { return *this; } - std::unique_ptr zero; - std::unique_ptr one; - std::unique_ptr intermediate_x; + std::unique_ptr> zero; + std::unique_ptr> one; + std::unique_ptr intermediate_x; } cache_; }; diff --git a/include/ginkgo/core/base/composition.hpp b/include/ginkgo/core/base/composition.hpp index 93b46cf8eee..bc5afd24f9b 100644 --- a/include/ginkgo/core/base/composition.hpp +++ b/include/ginkgo/core/base/composition.hpp @@ -156,10 +156,13 @@ class Composition : public LinOp, add_operators(std::move(oper), std::forward(rest)...); } - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: std::vector> operators_; diff --git a/include/ginkgo/core/base/perturbation.hpp b/include/ginkgo/core/base/perturbation.hpp index 87794b90fa2..21b0780b460 100644 --- a/include/ginkgo/core/base/perturbation.hpp +++ b/include/ginkgo/core/base/perturbation.hpp @@ -77,7 +77,8 @@ class Perturbation : public LinOp, * * @return the scalar of the perturbation */ - const std::shared_ptr get_scalar() const noexcept + const std::shared_ptr> get_scalar() + const noexcept { return scalar_; } @@ -111,7 +112,7 @@ class Perturbation : public LinOp, * @return A smart pointer to the newly created perturbation. */ static std::unique_ptr create( - std::shared_ptr scalar, + std::shared_ptr> scalar, std::shared_ptr basis); /** @@ -124,23 +125,29 @@ class Perturbation : public LinOp, * @return A smart pointer to the newly created perturbation. */ static std::unique_ptr create( - std::shared_ptr scalar, std::shared_ptr basis, + std::shared_ptr> scalar, + std::shared_ptr basis, std::shared_ptr projector); protected: explicit Perturbation(std::shared_ptr exec); - explicit Perturbation(std::shared_ptr scalar, - std::shared_ptr basis); + explicit Perturbation( + std::shared_ptr> scalar, + std::shared_ptr basis); - explicit Perturbation(std::shared_ptr scalar, - std::shared_ptr basis, - std::shared_ptr projector); + explicit Perturbation( + std::shared_ptr> scalar, + std::shared_ptr basis, + std::shared_ptr projector); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; /** * Validates the dimensions of the `scalar`, `basis` and `projector` @@ -152,7 +159,7 @@ class Perturbation : public LinOp, private: std::shared_ptr basis_; std::shared_ptr projector_; - std::shared_ptr scalar_; + std::shared_ptr> scalar_; // TODO: solve race conditions when multithreading mutable struct cache_struct { @@ -164,23 +171,11 @@ class Perturbation : public LinOp, // allocate linops of cache. The dimension of `intermediate` is // (the number of rows of projector, the number of columns of b). Others // are 1x1 scalar. - void allocate(std::shared_ptr exec, dim<2> size) - { - using vec = matrix::MultiVector; - if (one == nullptr) { - one = initialize({gko::one()}, exec); - } - if (alpha_scalar == nullptr) { - alpha_scalar = vec::create(exec, gko::dim<2>(1)); - } - if (intermediate == nullptr || intermediate->get_size() != size) { - intermediate = vec::create(exec, size); - } - } - - std::unique_ptr intermediate; - std::unique_ptr one; - std::unique_ptr alpha_scalar; + void allocate(std::shared_ptr exec, dim<2> size); + + std::unique_ptr intermediate; + std::unique_ptr> one; + std::unique_ptr> alpha_scalar; } cache_; }; From f3716b421139a1bd917ffd52c2ac967a84fc7329 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:03:01 +0200 Subject: [PATCH 11/21] fix hierarchy change for preconditioners --- core/preconditioner/ic.cpp | 52 ++++++------ core/preconditioner/ilu.cpp | 81 +++++++++---------- core/preconditioner/isai.cpp | 12 ++- core/preconditioner/jacobi.cpp | 40 +++++---- core/preconditioner/sor.cpp | 4 +- include/ginkgo/core/preconditioner/ic.hpp | 14 ++-- include/ginkgo/core/preconditioner/ilu.hpp | 14 ++-- include/ginkgo/core/preconditioner/isai.hpp | 10 ++- include/ginkgo/core/preconditioner/jacobi.hpp | 9 ++- 9 files changed, 117 insertions(+), 119 deletions(-) diff --git a/core/preconditioner/ic.cpp b/core/preconditioner/ic.cpp index 4c8e3d1897d..445aca72b96 100644 --- a/core/preconditioner/ic.cpp +++ b/core/preconditioner/ic.cpp @@ -9,6 +9,7 @@ #include #include +#include "core/base/dispatch_helper.hpp" #include "core/config/config_helper.hpp" #include "core/config/dispatch.hpp" @@ -125,36 +126,36 @@ Ic::Ic(Ic&& other) : Ic{other.get_executor()} template -void Ic::apply_impl(const LinOp* b, LinOp* x) const - +void Ic::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - // take care of real-to-complex apply - precision_dispatch_real_complex( - [&](auto dense_b, auto dense_x) { - this->set_cache_to(dense_b); - l_solver_->apply(dense_b, cache_.intermediate); - if (lh_solver_->apply_uses_initial_guess()) { - dense_x->copy_from(as(cache_.intermediate.get())); - } - lh_solver_->apply(cache_.intermediate, dense_x); - }, - b, x); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + + this->set_cache_to(dense_b); + l_solver_->apply(dense_b, cache_.intermediate); + if (lh_solver_->apply_uses_initial_guess()) { + dense_x->copy_from(cache_.intermediate.get()); + } + lh_solver_->apply(cache_.intermediate, dense_x); } template -void Ic::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const - +void Ic::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [&](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - this->set_cache_to(dense_b); - l_solver_->apply(dense_b, cache_.intermediate); - lh_solver_->apply(dense_alpha, cache_.intermediate, dense_beta, - dense_x); - }, - alpha, b, beta, x); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + this->set_cache_to(dense_b); + l_solver_->apply(dense_b, cache_.intermediate); + lh_solver_->apply(alpha, cache_.intermediate, beta, dense_x); } @@ -222,8 +223,7 @@ Ic::Ic(const Factory* factory, template -void Ic::set_cache_to(const LinOp* b) const - +void Ic::set_cache_to(const AbstractMultiVector* b) const { if (cache_.intermediate == nullptr) { cache_.intermediate = diff --git a/core/preconditioner/ilu.cpp b/core/preconditioner/ilu.cpp index 8eb0fadfe25..35e31ed0f2e 100644 --- a/core/preconditioner/ilu.cpp +++ b/core/preconditioner/ilu.cpp @@ -153,54 +153,50 @@ Ilu::Ilu(Ilu&& other) template -void Ilu::apply_impl(const LinOp* b, - LinOp* x) const +void Ilu::apply_impl( + const AbstractMultiVector* b, AbstractMultiVector* x) const { - // take care of real-to-complex apply - precision_dispatch_real_complex( - [&](auto dense_b, auto dense_x) { - this->set_cache_to(dense_b); - if (!ReverseApply) { - l_solver_->apply(dense_b, cache_.intermediate); - if (u_solver_->apply_uses_initial_guess()) { - dense_x->copy_from( - as(cache_.intermediate.get())); - } - u_solver_->apply(cache_.intermediate, dense_x); - } else { - u_solver_->apply(dense_b, cache_.intermediate); - if (l_solver_->apply_uses_initial_guess()) { - dense_x->copy_from( - as(cache_.intermediate.get())); - } - l_solver_->apply(cache_.intermediate, dense_x); - } - }, - b, x); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + + this->set_cache_to(dense_b); + if (!ReverseApply) { + l_solver_->apply(dense_b, cache_.intermediate); + if (u_solver_->apply_uses_initial_guess()) { + dense_x->copy_from(cache_.intermediate.get()); + } + u_solver_->apply(cache_.intermediate, dense_x); + } else { + u_solver_->apply(dense_b, cache_.intermediate); + if (l_solver_->apply_uses_initial_guess()) { + dense_x->copy_from(cache_.intermediate.get()); + } + l_solver_->apply(cache_.intermediate, dense_x); + } } template -void Ilu::apply_impl(const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - LinOp* x) const +void Ilu::apply_impl( + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [&](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - this->set_cache_to(dense_b); - if (!ReverseApply) { - l_solver_->apply(dense_b, cache_.intermediate); - u_solver_->apply(dense_alpha, cache_.intermediate, dense_beta, - dense_x); - } else { - u_solver_->apply(dense_b, cache_.intermediate); - l_solver_->apply(dense_alpha, cache_.intermediate, dense_beta, - dense_x); - } - }, - alpha, b, beta, x); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + + this->set_cache_to(dense_b); + if (!ReverseApply) { + l_solver_->apply(dense_b, cache_.intermediate); + u_solver_->apply(alpha, cache_.intermediate, beta, dense_x); + } else { + u_solver_->apply(dense_b, cache_.intermediate); + l_solver_->apply(alpha, cache_.intermediate, beta, dense_x); + } } @@ -266,7 +262,8 @@ Ilu::Ilu( template -void Ilu::set_cache_to(const LinOp* b) const +void Ilu::set_cache_to( + const AbstractMultiVector* b) const { if (cache_.intermediate == nullptr) { diff --git a/core/preconditioner/isai.cpp b/core/preconditioner/isai.cpp index 6a8fa9bd8fa..19cba1861a0 100644 --- a/core/preconditioner/isai.cpp +++ b/core/preconditioner/isai.cpp @@ -124,7 +124,6 @@ Isai::parse( template Isai::Isai( const Factory* factory, std::shared_ptr system_matrix) - : LinOp(factory->get_executor(), system_matrix->get_size(), precision_v), parameters_{factory->get_parameters()} @@ -144,8 +143,8 @@ Isai::Isai( template -void Isai::apply_impl(const LinOp* b, - LinOp* x) const +void Isai::apply_impl( + const AbstractMultiVector* b, AbstractMultiVector* x) const { approximate_inverse_->apply(b, x); @@ -153,10 +152,9 @@ void Isai::apply_impl(const LinOp* b, template -void Isai::apply_impl(const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - LinOp* x) const +void Isai::apply_impl( + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const { approximate_inverse_->apply(alpha, b, beta, x); diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp index c5f8474f8d6..c7d329e7c28 100644 --- a/core/preconditioner/jacobi.cpp +++ b/core/preconditioner/jacobi.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -18,6 +17,7 @@ #include #include +#include "core/base/dispatch_helper.hpp" #include "core/base/extended_float.hpp" #include "core/base/utils.hpp" #include "core/config/config_helper.hpp" @@ -161,21 +161,19 @@ Jacobi::Jacobi(Jacobi&& other) template -void Jacobi::apply_impl(const LinOp* b, LinOp* x) const +void Jacobi::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { + apply_precision_dispatch( + [this](auto view_b, auto view_x, auto...) { if (parameters_.max_block_size == 1) { this->get_executor()->run(jacobi::make_simple_scalar_apply( - this->blocks_, dense_b->get_const_device_view(), - dense_x->get_device_view())); + this->blocks_, view_b, view_x)); } else { this->get_executor()->run(jacobi::make_simple_apply( num_blocks_, parameters_.max_block_size, storage_scheme_, parameters_.storage_optimization.block_wise, - parameters_.block_pointers, blocks_, - dense_b->get_const_device_view(), - dense_x->get_device_view())); + parameters_.block_pointers, blocks_, view_b, view_x)); } }, b, x); @@ -183,27 +181,25 @@ void Jacobi::apply_impl(const LinOp* b, LinOp* x) const template -void Jacobi::apply_impl(const LinOp* alpha, - const LinOp* b, const LinOp* beta, - LinOp* x) const +void Jacobi::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { + apply_precision_dispatch( + [this](auto dense_alpha, auto view_b, auto dense_beta, auto view_x, + auto...) { if (parameters_.max_block_size == 1) { this->get_executor()->run(jacobi::make_scalar_apply( - this->blocks_, dense_alpha->get_const_device_view(), - dense_b->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); + this->blocks_, dense_alpha->get_const_device_view(), view_b, + dense_beta->get_const_device_view(), view_x)); } else { this->get_executor()->run(jacobi::make_apply( num_blocks_, parameters_.max_block_size, storage_scheme_, parameters_.storage_optimization.block_wise, parameters_.block_pointers, blocks_, - dense_alpha->get_const_device_view(), - dense_b->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); + dense_alpha->get_const_device_view(), view_b, + dense_beta->get_const_device_view(), view_x)); } }, alpha, b, beta, x); diff --git a/core/preconditioner/sor.cpp b/core/preconditioner/sor.cpp index 1acfc802c4e..645bfd8f8f9 100644 --- a/core/preconditioner/sor.cpp +++ b/core/preconditioner/sor.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -15,7 +14,6 @@ #include "core/base/utils.hpp" #include "core/config/config_helper.hpp" #include "core/factorization/factorization_kernels.hpp" -#include "core/matrix/csr_builder.hpp" #include "core/preconditioner/sor_kernels.hpp" namespace gko { diff --git a/include/ginkgo/core/preconditioner/ic.hpp b/include/ginkgo/core/preconditioner/ic.hpp index fa5272944ad..cbe9ae62e29 100644 --- a/include/ginkgo/core/preconditioner/ic.hpp +++ b/include/ginkgo/core/preconditioner/ic.hpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -207,10 +206,13 @@ class Ic : public LinOp, public Transposable { Ic(Ic&& other); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit Ic(std::shared_ptr exec); @@ -223,7 +225,7 @@ class Ic : public LinOp, public Transposable { * @param b Right hand side of the first solve. Also acts as the * initial guess, meaning the intermediate value will be a copy of b */ - void set_cache_to(const LinOp* b) const; + void set_cache_to(const AbstractMultiVector* b) const; private: std::shared_ptr l_solver_{}; @@ -245,7 +247,7 @@ class Ic : public LinOp, public Transposable { cache_struct(cache_struct&&) {} cache_struct& operator=(const cache_struct&) { return *this; } cache_struct& operator=(cache_struct&&) { return *this; } - std::unique_ptr intermediate{}; + std::unique_ptr> intermediate{}; } cache_; }; diff --git a/include/ginkgo/core/preconditioner/ilu.hpp b/include/ginkgo/core/preconditioner/ilu.hpp index 82c3d16ffbf..ccf14e716b9 100644 --- a/include/ginkgo/core/preconditioner/ilu.hpp +++ b/include/ginkgo/core/preconditioner/ilu.hpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include #include @@ -222,10 +221,13 @@ class Ilu : public LinOp, public Transposable { Ilu(Ilu&& other); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit Ilu(std::shared_ptr exec); @@ -238,7 +240,7 @@ class Ilu : public LinOp, public Transposable { * @param b Right hand side of the first solve. Also acts as the initial * guess, meaning the intermediate value will be a copy of b */ - void set_cache_to(const LinOp* b) const; + void set_cache_to(const AbstractMultiVector* b) const; private: std::shared_ptr l_solver_{}; @@ -260,7 +262,7 @@ class Ilu : public LinOp, public Transposable { cache_struct(cache_struct&&) {} cache_struct& operator=(const cache_struct&) { return *this; } cache_struct& operator=(cache_struct&&) { return *this; } - std::unique_ptr intermediate{}; + std::unique_ptr> intermediate{}; } cache_; }; diff --git a/include/ginkgo/core/preconditioner/isai.hpp b/include/ginkgo/core/preconditioner/isai.hpp index 54522806450..db91d982bb4 100644 --- a/include/ginkgo/core/preconditioner/isai.hpp +++ b/include/ginkgo/core/preconditioner/isai.hpp @@ -226,10 +226,13 @@ class Isai : public LinOp, public Transposable { explicit Isai(const Factory* factory, std::shared_ptr system_matrix); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: /** @@ -246,7 +249,6 @@ class Isai : public LinOp, public Transposable { bool skip_sorting, int power, index_type excess_limit, remove_complex excess_solver_reduction); -private: std::shared_ptr approximate_inverse_; }; diff --git a/include/ginkgo/core/preconditioner/jacobi.hpp b/include/ginkgo/core/preconditioner/jacobi.hpp index 34ac73508c9..88d4f6b4d1d 100644 --- a/include/ginkgo/core/preconditioner/jacobi.hpp +++ b/include/ginkgo/core/preconditioner/jacobi.hpp @@ -583,10 +583,13 @@ class Jacobi : public LinOp, */ void detect_blocks(const matrix::Csr* system_matrix); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: block_interleaved_storage_scheme storage_scheme_{}; From 6ea1c4c831378fc6030e33a6f8d536ead3852026 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:03:22 +0200 Subject: [PATCH 12/21] fix hierarchy change for reorder --- core/reorder/amd.cpp | 8 +- core/reorder/rcm.cpp | 4 +- core/reorder/scaled_reordered.cpp | 136 ++++++++++-------- .../ginkgo/core/reorder/scaled_reordered.hpp | 30 ++-- 4 files changed, 91 insertions(+), 87 deletions(-) diff --git a/core/reorder/amd.cpp b/core/reorder/amd.cpp index 6f699832920..041e1cff864 100644 --- a/core/reorder/amd.cpp +++ b/core/reorder/amd.cpp @@ -113,9 +113,9 @@ std::unique_ptr Amd::generate_impl( if (!parameters_.skip_symmetrize) { auto scalar = initialize({one>()}, exec); - auto id = complex_identity::create(exec, conv_csr->get_size()[0]); // compute A^T + A - conv_csr->transpose()->apply(scalar, id, scalar, conv_csr); + conv_csr = conv_csr->scale_add( + scalar, scalar, as(conv_csr->transpose())); } d_nnz = conv_csr->get_num_stored_elements(); d_row_ptrs = conv_csr->get_row_ptrs(); @@ -129,9 +129,9 @@ std::unique_ptr Amd::generate_impl( } if (!parameters_.skip_symmetrize) { auto scalar = initialize({one()}, exec); - auto id = real_identity::create(exec, conv_csr->get_size()[0]); // compute A^T + A - conv_csr->transpose()->apply(scalar, id, scalar, conv_csr); + conv_csr = conv_csr->scale_add(scalar, scalar, + as(conv_csr->transpose())); } d_nnz = conv_csr->get_num_stored_elements(); d_row_ptrs = conv_csr->get_row_ptrs(); diff --git a/core/reorder/rcm.cpp b/core/reorder/rcm.cpp index 5c9bf5a11ee..b737df94a52 100644 --- a/core/reorder/rcm.cpp +++ b/core/reorder/rcm.cpp @@ -170,9 +170,9 @@ std::unique_ptr Rcm::generate_impl( as>(op)->convert_to(conv_csr); if (!parameters_.skip_symmetrize) { auto scalar = initialize({one()}, exec); - auto id = Identity::create(exec, conv_csr->get_size()[0]); // compute A^T + A - conv_csr->transpose()->apply(scalar, id, scalar, conv_csr); + conv_csr = conv_csr->scale_add(scalar, scalar, + as(conv_csr->transpose())); } if (exec != work_exec) { conv_csr = gko::clone(work_exec, std::move(conv_csr)); diff --git a/core/reorder/scaled_reordered.cpp b/core/reorder/scaled_reordered.cpp index 773618c052f..71b52e04927 100644 --- a/core/reorder/scaled_reordered.cpp +++ b/core/reorder/scaled_reordered.cpp @@ -6,7 +6,7 @@ #include -#include +#include #include @@ -36,7 +36,8 @@ ScaledReordered::ScaledReordered( auto exec = this->get_executor(); - system_matrix_ = gko::clone(exec, system_matrix); + system_matrix_ = + as>(gko::clone(exec, system_matrix)); // Scale the system matrix if scaling coefficients are provided if (parameters_.row_scaling) { @@ -55,8 +56,8 @@ ScaledReordered::ScaledReordered( if (parameters_.reordering) { auto reordering = parameters_.reordering->generate(system_matrix_); permutation_array_ = reordering->get_permutation_array(); - system_matrix_ = as>(system_matrix_) - ->permute(&permutation_array_); + system_matrix_ = as>( + system_matrix_->permute(&permutation_array_)); } // Generate the inner operator with the scaled and reordered system @@ -71,69 +72,84 @@ ScaledReordered::ScaledReordered( template -void ScaledReordered::apply_impl(const LinOp* b, - LinOp* x) const +void ScaledReordered::apply_impl( + const AbstractMultiVector* b, AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - auto exec = this->get_executor(); - this->set_cache_to(dense_b, dense_x); - - // Preprocess the input vectors before applying the inner operator. - if (row_scaling_) { - row_scaling_->apply(cache_.inner_b, cache_.intermediate); - std::swap(cache_.inner_b, cache_.intermediate); - } - // Col scaling for x is only necessary if the inner operator uses an - // initial guess. Otherwise x is overwritten anyway. - if (col_scaling_ && inner_operator_->apply_uses_initial_guess()) { - col_scaling_->inverse_apply(cache_.inner_x, - cache_.intermediate); - std::swap(cache_.inner_x, cache_.intermediate); - } - if (permutation_array_.get_size() > 0) { - cache_.inner_b->row_permute(&permutation_array_, + auto exec = this->get_executor(); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + this->set_cache_to(converted_b.get(), converted_x.get()); + + // Preprocess the input vectors before applying the inner operator. + if (row_scaling_) { + row_scaling_->apply(cache_.inner_b, cache_.intermediate); + std::swap(cache_.inner_b, cache_.intermediate); + } + // Col scaling for x is only necessary if the inner operator uses an + // initial guess. Otherwise x is overwritten anyway. + if (col_scaling_ && inner_operator_->apply_uses_initial_guess()) { + col_scaling_->inverse_apply(cache_.inner_x, cache_.intermediate); + std::swap(cache_.inner_x, cache_.intermediate); + } + if (permutation_array_.get_size() > 0) { + cache_.inner_b->row_permute(&permutation_array_, cache_.intermediate); + std::swap(cache_.inner_b, cache_.intermediate); + if (inner_operator_->apply_uses_initial_guess()) { + cache_.inner_x->row_permute(&permutation_array_, + cache_.intermediate); + std::swap(cache_.inner_x, cache_.intermediate); + } + } + + inner_operator_->apply(cache_.inner_b, cache_.inner_x); + + // Permute and scale the solution vector back. + if (permutation_array_.get_size() > 0) { + cache_.inner_x->inverse_row_permute(&permutation_array_, cache_.intermediate); - std::swap(cache_.inner_b, cache_.intermediate); - if (inner_operator_->apply_uses_initial_guess()) { - cache_.inner_x->row_permute(&permutation_array_, - cache_.intermediate); - std::swap(cache_.inner_x, cache_.intermediate); - } - } - - inner_operator_->apply(cache_.inner_b, cache_.inner_x); - - // Permute and scale the solution vector back. - if (permutation_array_.get_size() > 0) { - cache_.inner_x->inverse_row_permute(&permutation_array_, - cache_.intermediate); - std::swap(cache_.inner_x, cache_.intermediate); - } - if (col_scaling_) { - col_scaling_->apply(cache_.inner_x, cache_.intermediate); - std::swap(cache_.inner_x, cache_.intermediate); - } - dense_x->copy_from(cache_.inner_x); - }, - b, x); + std::swap(cache_.inner_x, cache_.intermediate); + } + if (col_scaling_) { + col_scaling_->apply(cache_.inner_x, cache_.intermediate); + std::swap(cache_.inner_x, cache_.intermediate); + } + // @todo: this needs two copies in the mixed precision case: + // inner->converted and converted->x + converted_x->copy_from(cache_.inner_x); } template -void ScaledReordered::apply_impl(const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - LinOp* x) const +void ScaledReordered::apply_impl( + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + auto x_clone = x->clone(); + this->apply_impl(b, x_clone.get()); + x->scale(beta); + x->add_scaled(alpha, x_clone); +} + + +template +void ScaledReordered::set_cache_to( + const AbstractMultiVector* b, const AbstractMultiVector* x) const + +{ + if (cache_.inner_b == nullptr || + cache_.inner_b->get_size() != b->get_size()) { + const auto size = b->get_size(); + cache_.inner_b = + matrix::MultiVector::create(this->get_executor(), size); + cache_.inner_x = + matrix::MultiVector::create(this->get_executor(), size); + cache_.intermediate = + matrix::MultiVector::create(this->get_executor(), size); + } + cache_.inner_b->copy_from(as>(b)); + if (inner_operator_->apply_uses_initial_guess()) { + cache_.inner_x->copy_from(as>(x)); + } } diff --git a/include/ginkgo/core/reorder/scaled_reordered.hpp b/include/ginkgo/core/reorder/scaled_reordered.hpp index 8bc7831fe8b..22ce02d506b 100644 --- a/include/ginkgo/core/reorder/scaled_reordered.hpp +++ b/include/ginkgo/core/reorder/scaled_reordered.hpp @@ -102,10 +102,13 @@ class ScaledReordered : public LinOp { explicit ScaledReordered(const Factory* factory, std::shared_ptr system_matrix); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; /** * Prepares the intermediate right hand side, solution and intermediate @@ -118,26 +121,11 @@ class ScaledReordered : public LinOp { * case the inner operator uses an initial guess, will be scaled and * permuted accordingly. */ - void set_cache_to(const LinOp* b, const LinOp* x) const - { - if (cache_.inner_b == nullptr || - cache_.inner_b->get_size() != b->get_size()) { - const auto size = b->get_size(); - cache_.inner_b = matrix::MultiVector::create( - this->get_executor(), size); - cache_.inner_x = matrix::MultiVector::create( - this->get_executor(), size); - cache_.intermediate = matrix::MultiVector::create( - this->get_executor(), size); - } - cache_.inner_b->copy_from(as(b)); - if (inner_operator_->apply_uses_initial_guess()) { - cache_.inner_x->copy_from(as(x)); - } - } + void set_cache_to(const AbstractMultiVector* b, + const AbstractMultiVector* x) const; private: - std::shared_ptr system_matrix_{}; + std::shared_ptr> system_matrix_{}; std::shared_ptr inner_operator_{}; std::shared_ptr> row_scaling_{}; std::shared_ptr> col_scaling_{}; From 3d7794afb913254d917fd25119798b2436e7073a Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:03:49 +0200 Subject: [PATCH 13/21] fix hierarchy change for distributed --- core/distributed/helpers.hpp | 95 ++--------- core/distributed/matrix.cpp | 71 +++++---- .../distributed/neighborhood_communicator.cpp | 3 - core/distributed/preconditioner/schwarz.cpp | 137 ++++++++-------- core/distributed/row_gatherer.cpp | 149 ++++++++---------- include/ginkgo/core/distributed/matrix.hpp | 13 +- .../distributed/preconditioner/schwarz.hpp | 12 +- .../ginkgo/core/distributed/row_gatherer.hpp | 47 +++--- 8 files changed, 228 insertions(+), 299 deletions(-) diff --git a/core/distributed/helpers.hpp b/core/distributed/helpers.hpp index 3b90b3bc17e..3c85209bcb9 100644 --- a/core/distributed/helpers.hpp +++ b/core/distributed/helpers.hpp @@ -17,58 +17,21 @@ namespace gko { +namespace experimental { +namespace distributed { namespace detail { - - -template -std::unique_ptr> create_with_config_of( - const matrix::MultiVector* mtx) -{ - return matrix::MultiVector::create( - mtx->get_executor(), mtx->get_size(), mtx->get_stride()); -} - - -template -const matrix::MultiVector* get_local( - const matrix::MultiVector* mtx) -{ - return mtx; -} - - -template -matrix::MultiVector* get_local(matrix::MultiVector* mtx) -{ - return mtx; -} - - #if GINKGO_BUILD_MPI template -std::unique_ptr> -create_with_config_of(const experimental::distributed::Vector* mtx) -{ - return experimental::distributed::Vector::create( - mtx->get_executor(), mtx->get_communicator(), mtx->get_size(), - mtx->get_local_vector()->get_size(), - mtx->get_local_vector()->get_stride()); -} - - -template -matrix::MultiVector* get_local( - experimental::distributed::Vector* mtx) +matrix::MultiVector* get_local_mutable(Vector* mtx) { return const_cast*>(mtx->get_local_vector()); } template -const matrix::MultiVector* get_local( - const experimental::distributed::Vector* mtx) +const matrix::MultiVector* get_local(const Vector* mtx) { return mtx->get_local_vector(); } @@ -81,8 +44,7 @@ template bool is_distributed(Arg* linop) { #if GINKGO_BUILD_MPI - return dynamic_cast( - linop); + return dynamic_cast(linop); #else return false; #endif @@ -93,8 +55,7 @@ template bool is_distributed(Arg* linop, Rest*... rest) { #if GINKGO_BUILD_MPI - bool is_distributed_value = - dynamic_cast(linop); + bool is_distributed_value = dynamic_cast(linop); GKO_ASSERT(is_distributed_value == is_distributed(rest...)); return is_distributed_value; #else @@ -103,46 +64,6 @@ bool is_distributed(Arg* linop, Rest*... rest) } -/** - * Cast an input linop to the correct underlying vector type (dense/distributed) - * and passes it to the given function. - * - * @tparam ValueType The value type of the underlying dense or distributed - * vector. - * @tparam T The linop type, either LinOp, or const LinOp. - * @tparam F The function type. - * @tparam Args The types for the additional arguments of f. - * - * @param linop The linop to be casted into either a dense or distributed - * vector. - * @param f The function that is to be called with the correctly casted linop. - * @param args The additional arguments of f. - */ -template -void vector_dispatch(T* linop, F&& f, Args&&... args) -{ -#if GINKGO_BUILD_MPI - if (is_distributed(linop)) { - using type = std::conditional_t< - std::is_const::value, - const experimental::distributed::Vector, - experimental::distributed::Vector>; - f(dynamic_cast(linop), std::forward(args)...); - } else -#endif - { - using type = std::conditional_t::value, - const matrix::MultiVector, - matrix::MultiVector>; - if (auto concrete_linop = dynamic_cast(linop)) { - f(concrete_linop, std::forward(args)...); - } else { - GKO_NOT_SUPPORTED(linop); - } - } -} - - #if GINKGO_BUILD_MPI @@ -152,7 +73,7 @@ void vector_dispatch(T* linop, F&& f, Args&&... args) template auto run_matrix(T* linop, F&& f, Args&&... args) { - using namespace gko::experimental::distributed; + using namespace gko::detail; return run< with_same_constness_t, T>, with_same_constness_t, T>, @@ -205,6 +126,8 @@ inline const LinOp* get_local(const LinOp* mtx) } // namespace detail +} // namespace distributed +} // namespace experimental } // namespace gko diff --git a/core/distributed/matrix.cpp b/core/distributed/matrix.cpp index fa191cf13b0..1f016c9a60b 100644 --- a/core/distributed/matrix.cpp +++ b/core/distributed/matrix.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -16,7 +15,9 @@ #include #include +#include "core/base/dispatch_helper.hpp" #include "core/distributed/matrix_kernels.hpp" +#include "ginkgo/core/matrix/batch_csr.hpp" namespace gko { @@ -527,14 +528,14 @@ init_recv_buffers(std::shared_ptr exec, template void Matrix::apply_impl( - const LinOp* b, LinOp* x) const + const AbstractMultiVector* b, AbstractMultiVector* x) const { - distributed::mixed_precision_dispatch_real_complex( - [this](const auto dense_b, auto dense_x) { - using x_value_type = - typename std::decay_t::value_type; - using b_value_type = - typename std::decay_t::value_type; + mixed_precision_dispatch( + [this](const auto b_, auto x_, auto p_b, auto p_x) { + using x_value_type = std::decay_t; + using b_value_type = std::decay_t; + auto dense_x = as>(x_); + auto dense_b = as>(b_); auto x_exec = dense_x->get_executor(); auto local_x = gko::matrix::MultiVector::create( x_exec, dense_x->get_local_vector()->get_size(), @@ -590,18 +591,20 @@ void Matrix::apply_impl( template void Matrix::apply_impl( - const LinOp* alpha, const LinOp* b, const LinOp* beta, LinOp* x) const + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const { - distributed::mixed_precision_dispatch_real_complex( - [this, alpha, beta](const auto dense_b, auto dense_x) { - using x_value_type = - typename std::decay_t::value_type; - using b_value_type = - typename std::decay_t::value_type; + mixed_precision_dispatch( + [&](const auto b_, auto x_, auto p_b, auto p_x) { + using x_value_type = std::decay_t; + using b_value_type = std::decay_t; + auto dense_x = as>(x_); + auto dense_b = as>(b_); + auto converted_alpha = alpha->as_precision(this); + auto converted_beta = beta->as_precision(dense_x); + auto dense_alpha = converted_alpha.get(); + auto dense_beta = converted_beta.get(); const auto x_exec = dense_x->get_executor(); - auto local_alpha = gko::make_temporary_conversion(alpha); - auto local_beta = - gko::make_temporary_conversion(beta); auto local_x = gko::matrix::MultiVector::create( x_exec, dense_x->get_local_vector()->get_size(), gko::make_array_view( @@ -624,15 +627,15 @@ void Matrix::apply_impl( // reference and omp executor does not have event, so we still // submit the mpi first. auto req = this->row_gatherer_->apply_async(dense_b, recv_ptr); - diag_mtx_->apply(local_alpha.get(), dense_b->get_local_vector(), - local_beta.get(), local_x); + diag_mtx_->apply(dense_alpha, dense_b->get_local_vector(), + dense_beta, local_x); req.wait(); } else { // we use event here such that we can submit spmv job first // without waiting for synchronization from the row gatherer. auto ev = this->row_gatherer_->apply_prepare(dense_b); - diag_mtx_->apply(local_alpha.get(), dense_b->get_local_vector(), - local_beta.get(), local_x); + diag_mtx_->apply(dense_alpha, dense_b->get_local_vector(), + dense_beta, local_x); auto req = this->row_gatherer_->apply_finalize(dense_b, recv_ptr, ev); req.wait(); @@ -644,11 +647,11 @@ void Matrix::apply_impl( if (auto coo = std::dynamic_pointer_cast< const ::gko::matrix::Coo>( off_diag_mtx_)) { - coo->apply2(local_alpha.get(), recv_vector->get_local_vector(), + coo->apply2(dense_alpha, recv_vector->get_local_vector(), local_x); } else { off_diag_mtx_->apply( - local_alpha.get(), recv_vector->get_local_vector(), + dense_alpha, recv_vector->get_local_vector(), one_scalar_.template get().get(), local_x); } }, @@ -688,19 +691,24 @@ void Matrix::col_scale( ? host_recv_vector.get() : recv_vector.get(); + auto diag_mtx_csr = + as>(diag_mtx_); + auto off_diag_mtx_csr = + as>(off_diag_mtx_); + if (scaling_factors->get_executor() == scaling_factors->get_executor()->get_master()) { // reference and omp executor does not have event, so we still // submit the mpi first. auto req = this->row_gatherer_->apply_async(scaling_factors_ptr, recv_ptr); - scale_diag->rapply(diag_mtx_, diag_mtx_); + scale_diag->rapply(diag_mtx_csr, diag_mtx_csr); req.wait(); } else { // we use event here such that we can submit diag matrix scaling job // first without waiting for synchronization from the row gatherer. auto ev = this->row_gatherer_->apply_prepare(scaling_factors_ptr); - scale_diag->rapply(diag_mtx_, diag_mtx_); + scale_diag->rapply(diag_mtx_csr, diag_mtx_csr); auto req = this->row_gatherer_->apply_finalize(scaling_factors_ptr, recv_ptr, ev); req.wait(); @@ -714,7 +722,7 @@ void Matrix::col_scale( exec, n_off_diag_cols, make_const_array_view(exec, n_off_diag_cols, recv_vector->get_const_local_values())); - off_diag_scale_diag->rapply(off_diag_mtx_, off_diag_mtx_); + off_diag_scale_diag->rapply(off_diag_mtx_csr, off_diag_mtx_csr); } } @@ -741,8 +749,13 @@ void Matrix::row_scale( exec, n_local_rows, make_const_array_view(exec, n_local_rows, scale_values)); - scale_diag->apply(diag_mtx_, diag_mtx_); - scale_diag->apply(off_diag_mtx_, off_diag_mtx_); + auto diag_mtx_csr = + as>(diag_mtx_); + auto off_diag_mtx_csr = + as>(off_diag_mtx_); + + scale_diag->apply(diag_mtx_csr, diag_mtx_csr); + scale_diag->apply(off_diag_mtx_csr, off_diag_mtx_csr); } diff --git a/core/distributed/neighborhood_communicator.cpp b/core/distributed/neighborhood_communicator.cpp index 4c97a256562..aed29b930e0 100644 --- a/core/distributed/neighborhood_communicator.cpp +++ b/core/distributed/neighborhood_communicator.cpp @@ -4,9 +4,6 @@ #include "ginkgo/core/distributed/neighborhood_communicator.hpp" -#include -#include - #include "core/base/allocator.hpp" diff --git a/core/distributed/preconditioner/schwarz.cpp b/core/distributed/preconditioner/schwarz.cpp index a9b7e702f44..ad4ce8960e6 100644 --- a/core/distributed/preconditioner/schwarz.cpp +++ b/core/distributed/preconditioner/schwarz.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -87,83 +86,75 @@ bool Schwarz void Schwarz::apply_impl( - const LinOp* b, LinOp* x) const + const AbstractMultiVector* b, AbstractMultiVector* x) const { - precision_dispatch_real_complex_distributed( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); + precision_dispatch( + [&](auto converted_b, auto converted_x) { + auto exec = this->get_executor(); + auto dense_b = as>(converted_b); + auto dense_x = as>(converted_x); + + // Two-level + if (this->coarse_solver_ != nullptr && + this->coarse_level_ != nullptr) { + if (this->local_solver_) { + this->local_solver_->apply( + dense_b->get_local_vector(), + detail::get_local_mutable(dense_x)); + } + auto coarse_level = + as(this->coarse_level_); + auto restrict_op = coarse_level->get_restrict_op(); + auto prolong_op = coarse_level->get_prolong_op(); + auto coarse_op = as>( + coarse_level->get_coarse_op()); + + // Coarse solve vector cache init + // Should allocate only in the first apply call if the number of + // rhs is unchanged. + auto cs_ncols = dense_x->get_size()[1]; + auto cs_local_nrows = + coarse_op->get_diag_matrix()->get_size()[0]; + auto cs_global_nrows = coarse_op->get_size()[0]; + auto cs_local_size = dim<2>(cs_local_nrows, cs_ncols); + auto cs_global_size = dim<2>(cs_global_nrows, cs_ncols); + auto comm = coarse_op->get_communicator(); + csol_cache_.init(exec, comm, cs_global_size, cs_local_size); + crhs_cache_.init(exec, comm, cs_global_size, cs_local_size); + + // Additive apply of coarse correction + restrict_op->apply(dense_b, crhs_cache_.get()); + // TODO: Does it make sense to restrict dense_x (to csol_cache) + // to provide a good initial guess for the coarse solver ? + if (this->coarse_solver_->apply_uses_initial_guess()) { + csol_cache_->copy_from(crhs_cache_.get()); + } + this->coarse_solver_->apply(crhs_cache_.get(), + csol_cache_.get()); + prolong_op->apply(this->coarse_weight_, csol_cache_.get(), + this->local_weight_, dense_x); + } else if (this->local_solver_ != nullptr) { + this->local_solver_->apply(dense_b->get_local_vector(), + detail::get_local_mutable(dense_x)); + } }, b, x); } -template -template -void Schwarz::apply_dense_impl( - const VectorType* dense_b, VectorType* dense_x) const -{ - using Vector = matrix::MultiVector; - using dist_vec = experimental::distributed::Vector; - auto exec = this->get_executor(); - - // Two-level - if (this->coarse_solver_ != nullptr && this->coarse_level_ != nullptr) { - if (this->local_solver_) { - this->local_solver_->apply(gko::detail::get_local(dense_b), - gko::detail::get_local(dense_x)); - } - auto coarse_level = - as(this->coarse_level_); - auto restrict_op = coarse_level->get_restrict_op(); - auto prolong_op = coarse_level->get_prolong_op(); - auto coarse_op = - as>( - coarse_level->get_coarse_op()); - - // Coarse solve vector cache init - // Should allocate only in the first apply call if the number of rhs is - // unchanged. - auto cs_ncols = dense_x->get_size()[1]; - auto cs_local_nrows = coarse_op->get_diag_matrix()->get_size()[0]; - auto cs_global_nrows = coarse_op->get_size()[0]; - auto cs_local_size = dim<2>(cs_local_nrows, cs_ncols); - auto cs_global_size = dim<2>(cs_global_nrows, cs_ncols); - auto comm = coarse_op->get_communicator(); - csol_cache_.init(exec, comm, cs_global_size, cs_local_size); - crhs_cache_.init(exec, comm, cs_global_size, cs_local_size); - - // Additive apply of coarse correction - restrict_op->apply(dense_b, crhs_cache_.get()); - // TODO: Does it make sense to restrict dense_x (to csol_cache) to - // provide a good initial guess for the coarse solver ? - if (this->coarse_solver_->apply_uses_initial_guess()) { - csol_cache_->copy_from(crhs_cache_.get()); - } - this->coarse_solver_->apply(crhs_cache_.get(), csol_cache_.get()); - prolong_op->apply(this->coarse_weight_, csol_cache_.get(), - this->local_weight_, dense_x); - } else if (this->local_solver_ != nullptr) { - this->local_solver_->apply(gko::detail::get_local(dense_b), - gko::detail::get_local(dense_x)); - } -} - - template void Schwarz::apply_impl( - const LinOp* alpha, const LinOp* b, const LinOp* beta, LinOp* x) const + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const { // only dispatch distributed case - experimental::distributed::precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - cache_.init_from(dense_x); - cache_->copy_from(dense_x); - this->apply_impl(dense_b, cache_.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, cache_.get()); - }, - alpha, b, beta, x); + auto dense_x = as>(x->as_precision(this)); + cache_.init_from(dense_x.get()); + cache_->copy_from(dense_x.get()); + this->apply_impl(b, cache_.get()); + dense_x->scale(beta); + dense_x->add_scaled(alpha, cache_.get()); } @@ -213,8 +204,9 @@ void Schwarz::generate( auto exec = this->get_executor(); using Csr = matrix::Csr; - auto diag_matrix_copy = share(Csr::create(exec)); - as>(diag_matrix)->convert_to(diag_matrix_copy); + auto diag_matrix_csr = + gko::detail::temporary_conversion::create( + diag_matrix.get()); auto off_diag_matrix = copy_and_convert_to( exec, as>( @@ -235,10 +227,9 @@ void Schwarz::generate( exec, diag_matrix->get_size()[0]); auto one = initialize>( {::gko::one()}, exec); - l1_diag_csr->apply(one, id, one, diag_matrix_copy); - this->set_solver( - gko::share(parameters_.local_solver->generate(diag_matrix_copy))); + this->set_solver(gko::share(parameters_.local_solver->generate( + l1_diag_csr->scale_add(one, one, diag_matrix_csr.get())))); } else { this->set_solver( gko::share(parameters_.local_solver->generate(diag_matrix))); diff --git a/core/distributed/row_gatherer.cpp b/core/distributed/row_gatherer.cpp index 22ba220db51..0dbaa3b7e32 100644 --- a/core/distributed/row_gatherer.cpp +++ b/core/distributed/row_gatherer.cpp @@ -6,9 +6,8 @@ #include #include -#include #include -#include +#include #include "core/base/dispatch_helper.hpp" #include "core/base/event_kernels.hpp" @@ -26,8 +25,9 @@ GKO_REGISTER_OPERATION(record_event, event::record_event); template -mpi::request RowGatherer::apply_async(ptr_param b, - ptr_param x) const +mpi::request RowGatherer::apply_async( + ptr_param b, + ptr_param x) const { return apply_async(b, x, send_cache_); } @@ -35,7 +35,7 @@ mpi::request RowGatherer::apply_async(ptr_param b, template mpi::request RowGatherer::apply_async( - ptr_param b, ptr_param x, + ptr_param b, ptr_param x, gko::detail::GenericDenseCache& workspace) const { auto ev = this->apply_prepare(b, workspace); @@ -44,7 +44,8 @@ mpi::request RowGatherer::apply_async( template std::shared_ptr -RowGatherer::apply_prepare(ptr_param b) const +RowGatherer::apply_prepare( + ptr_param b) const { return apply_prepare(b, send_cache_); } @@ -52,46 +53,37 @@ RowGatherer::apply_prepare(ptr_param b) const template std::shared_ptr RowGatherer::apply_prepare( - ptr_param b, gko::detail::GenericDenseCache& workspace) const + ptr_param b, + gko::detail::GenericDenseCache& workspace) const { std::shared_ptr ev = nullptr; auto exec = this->get_executor(); auto use_host_buffer = mpi::requires_host_buffer(exec, coll_comm_->get_base_communicator()); auto mpi_exec = use_host_buffer ? exec->get_master() : exec; - - // dispatch global vector - run, -#endif -#if GINKGO_ENABLE_BFLOAT16 - bfloat16, std::complex, -#endif - double, float, std::complex, std::complex>( - make_temporary_clone(exec, b).get(), [&](const auto* b_global) { - using ValueType = - typename std::decay_t::value_type; - // dispatch local vector with the same precision as the global - // vector - distributed::precision_dispatch([&]() { - auto b_local = b_global->get_local_vector(); - - dim<2> send_size(coll_comm_->get_send_size(), - b_local->get_size()[1]); - auto send_buffer = - workspace.get(mpi_exec, send_size); - b_local->row_gather(&send_idxs_, send_buffer); - b_local->get_executor()->run(event::make_record_event(ev)); - }); - }); + auto tmp_b = make_temporary_clone(exec, b); + + std::visit( + [&](auto p) { + using ValueType = std::decay_t; + + auto b_local = + as>(tmp_b.get())->get_local_vector(); + + dim<2> send_size(coll_comm_->get_send_size(), + b_local->get_size()[1]); + auto send_buffer = workspace.get(mpi_exec, send_size); + b_local->row_gather(&send_idxs_, send_buffer); + b_local->get_executor()->run(event::make_record_event(ev)); + }, + precision_to_variant(b->get_precision())); return ev; } template mpi::request RowGatherer::apply_finalize( - ptr_param b, ptr_param x, + ptr_param b, ptr_param x, std::shared_ptr ev) const { auto req = apply_finalize(b, x, ev, send_cache_); @@ -100,7 +92,7 @@ mpi::request RowGatherer::apply_finalize( template mpi::request RowGatherer::apply_finalize( - ptr_param b, ptr_param x, + ptr_param b, ptr_param x, std::shared_ptr ev, gko::detail::GenericDenseCache& workspace) const { @@ -118,40 +110,29 @@ mpi::request RowGatherer::apply_finalize( "Please provide a host buffer or enable MPI support for device " "memory."); - // dispatch global vector - run, -#endif -#if GINKGO_ENABLE_BFLOAT16 - bfloat16, std::complex, -#endif - double, float, std::complex, std::complex>( - make_temporary_clone(exec, b).get(), [&](const auto* b_global) { - using ValueType = - typename std::decay_t::value_type; - // dispatch local vector with the same precision as the global - // vector - distributed::precision_dispatch( - [&](auto* x_global) { - auto b_local = b_global->get_local_vector(); - - dim<2> send_size(coll_comm_->get_send_size(), - b_local->get_size()[1]); - auto send_buffer = - workspace.get(mpi_exec, send_size); - - auto recv_ptr = x_global->get_local_values(); - auto send_ptr = send_buffer->get_values(); - ev->synchronize(); - mpi::contiguous_type type( - b_local->get_size()[1], - mpi::type_impl::get_type()); - req = coll_comm_->i_all_to_all_v( - mpi_exec, send_ptr, type.get(), recv_ptr, type.get()); - }, - x.get()); - }); + auto tmp_b = make_temporary_clone(exec, b); + std::visit( + [&](auto p) { + using ValueType = std::decay_t; + + auto b_local = + as>(tmp_b.get())->get_local_vector(); + + dim<2> send_size(coll_comm_->get_send_size(), + b_local->get_size()[1]); + auto send_buffer = workspace.get(mpi_exec, send_size); + + auto x_global = as>(x->as_precision(b_local)); + auto recv_ptr = x_global->get_local_values(); + auto send_ptr = send_buffer->get_values(); + ev->synchronize(); + mpi::contiguous_type type(b_local->get_size()[1], + mpi::type_impl::get_type()); + req = coll_comm_->i_all_to_all_v(mpi_exec, send_ptr, type.get(), + recv_ptr, type.get()); + }, + precision_to_variant(b->get_precision())); + return req; } @@ -161,7 +142,8 @@ namespace detail { template std::shared_ptr apply_prepare( - const RowGatherer* rg, ptr_param b) + const RowGatherer* rg, + ptr_param b) { return rg->apply_prepare(b); } @@ -169,7 +151,8 @@ std::shared_ptr apply_prepare( template std::shared_ptr apply_prepare( - const RowGatherer* rg, ptr_param b, + const RowGatherer* rg, + ptr_param b, gko::detail::GenericDenseCache& workspace) { return rg->apply_prepare(b, workspace); @@ -178,7 +161,8 @@ std::shared_ptr apply_prepare( template mpi::request apply_finalize(const RowGatherer* rg, - ptr_param b, ptr_param x, + ptr_param b, + ptr_param x, std::shared_ptr ev) { return rg->apply_finalize(b, x, ev); @@ -187,7 +171,8 @@ mpi::request apply_finalize(const RowGatherer* rg, template mpi::request apply_finalize(const RowGatherer* rg, - ptr_param b, ptr_param x, + ptr_param b, + ptr_param x, std::shared_ptr ev, gko::detail::GenericDenseCache& workspace) { @@ -197,21 +182,23 @@ mpi::request apply_finalize(const RowGatherer* rg, #define GKO_DECLARE_TEST_APPLY_PREPARE(IndexType) \ std::shared_ptr apply_prepare( \ - const RowGatherer*, ptr_param) + const RowGatherer*, ptr_param) -#define GKO_DECLARE_TEST_APPLY_PREPARE_WORKSPACE(IndexType) \ - std::shared_ptr apply_prepare( \ - const RowGatherer*, ptr_param, \ +#define GKO_DECLARE_TEST_APPLY_PREPARE_WORKSPACE(IndexType) \ + std::shared_ptr apply_prepare( \ + const RowGatherer*, ptr_param, \ gko::detail::GenericDenseCache&) -#define GKO_DECLARE_TEST_APPLY_FINALIZE(IndexType) \ - mpi::request apply_finalize(const RowGatherer* rg, \ - ptr_param b, ptr_param x, \ +#define GKO_DECLARE_TEST_APPLY_FINALIZE(IndexType) \ + mpi::request apply_finalize(const RowGatherer* rg, \ + ptr_param b, \ + ptr_param x, \ std::shared_ptr ev) #define GKO_DECLARE_TEST_APPLY_FINALIZE_WORKSPACE(IndexType) \ mpi::request apply_finalize(const RowGatherer* rg, \ - ptr_param b, ptr_param x, \ + ptr_param b, \ + ptr_param x, \ std::shared_ptr ev, \ gko::detail::GenericDenseCache&) diff --git a/include/ginkgo/core/distributed/matrix.hpp b/include/ginkgo/core/distributed/matrix.hpp index fb214883937..ddc6ff8c40b 100644 --- a/include/ginkgo/core/distributed/matrix.hpp +++ b/include/ginkgo/core/distributed/matrix.hpp @@ -716,6 +716,8 @@ class Matrix * The vector's row partition has to be the same as the matrix's column * partition. The scaling is done in-place. * + * @note: The local and non-local matrices must be in the CSR format. + * * @param scaling_factors The vector containing the scaling factors. */ void col_scale(ptr_param scaling_factors); @@ -725,6 +727,8 @@ class Matrix * The vector and the matrix have to have the same row partition. * The scaling is done in-place. * + * @note: The local and non-local matrices must be in the CSR format. + * * @param scaling_factors The vector containing the scaling factors. */ void row_scale(ptr_param scaling_factors); @@ -749,10 +753,13 @@ class Matrix std::shared_ptr diag_linop, std::shared_ptr off_diag_linop); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: std::shared_ptr> row_gatherer_; diff --git a/include/ginkgo/core/distributed/preconditioner/schwarz.hpp b/include/ginkgo/core/distributed/preconditioner/schwarz.hpp index d2b447460d6..e40a2309c27 100644 --- a/include/ginkgo/core/distributed/preconditioner/schwarz.hpp +++ b/include/ginkgo/core/distributed/preconditioner/schwarz.hpp @@ -194,13 +194,13 @@ class Schwarz : public LinOp { */ void generate(std::shared_ptr system_matrix); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - template - void apply_dense_impl(const VectorType* b, VectorType* x) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: /** diff --git a/include/ginkgo/core/distributed/row_gatherer.hpp b/include/ginkgo/core/distributed/row_gatherer.hpp index a37b73f062a..41cfb62e46f 100644 --- a/include/ginkgo/core/distributed/row_gatherer.hpp +++ b/include/ginkgo/core/distributed/row_gatherer.hpp @@ -36,24 +36,28 @@ namespace detail { // give access to test function on protected function template std::shared_ptr apply_prepare( - const RowGatherer* rg, ptr_param b); + const RowGatherer* rg, + ptr_param b); // give access to test function on protected function template std::shared_ptr apply_prepare( - const RowGatherer* rg, ptr_param b, + const RowGatherer* rg, + ptr_param b, gko::detail::GenericDenseCache& workspace); // give access to test function on protected function template mpi::request apply_finalize(const RowGatherer* rg, - ptr_param b, ptr_param x, + ptr_param b, + ptr_param x, std::shared_ptr); // give access to test function on protected function template mpi::request apply_finalize(const RowGatherer* rg, - ptr_param b, ptr_param x, + ptr_param b, + ptr_param x, std::shared_ptr, gko::detail::GenericDenseCache& workspace); @@ -96,16 +100,19 @@ class RowGatherer final : public PolymorphicObject, friend class Matrix; // for test purpose friend std::shared_ptr - detail::apply_prepare(const RowGatherer* rg, - ptr_param b); - friend std::shared_ptr detail::apply_prepare< - LocalIndexType>(const RowGatherer* rg, ptr_param b, - gko::detail::GenericDenseCache& workspace); + detail::apply_prepare( + const RowGatherer* rg, ptr_param b); + friend std::shared_ptr + detail::apply_prepare( + const RowGatherer* rg, ptr_param b, + gko::detail::GenericDenseCache& workspace); friend mpi::request detail::apply_finalize( - const RowGatherer* rg, ptr_param b, ptr_param x, + const RowGatherer* rg, ptr_param b, + ptr_param x, std::shared_ptr); friend mpi::request detail::apply_finalize( - const RowGatherer* rg, ptr_param b, ptr_param x, + const RowGatherer* rg, ptr_param b, + ptr_param x, std::shared_ptr, gko::detail::GenericDenseCache& workspace); @@ -125,8 +132,9 @@ class RowGatherer final : public PolymorphicObject, * @return a mpi::request for this task. The task is guaranteed to * be completed only after `.wait()` has been called on it. */ - [[nodiscard]] mpi::request apply_async(ptr_param b, - ptr_param x) const; + [[nodiscard]] mpi::request apply_async( + ptr_param b, + ptr_param x) const; /** * Asynchronous version of LinOp::apply. @@ -147,7 +155,8 @@ class RowGatherer final : public PolymorphicObject, * be completed only after `.wait()` has been called on it. */ [[nodiscard]] mpi::request apply_async( - ptr_param b, ptr_param x, + ptr_param b, + ptr_param x, gko::detail::GenericDenseCache& workspace) const; /** @@ -239,18 +248,20 @@ class RowGatherer final : public PolymorphicObject, protected: std::shared_ptr apply_prepare( - ptr_param b) const; + ptr_param b) const; std::shared_ptr apply_prepare( - ptr_param b, + ptr_param b, gko::detail::GenericDenseCache& workspace) const; mpi::request apply_finalize( - ptr_param b, ptr_param x, + ptr_param b, + ptr_param x, std::shared_ptr) const; mpi::request apply_finalize( - ptr_param b, ptr_param x, + ptr_param b, + ptr_param x, std::shared_ptr, gko::detail::GenericDenseCache& workspace) const; From bdc716641bfcf5337508464d8283d7f061aef905 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:04:15 +0200 Subject: [PATCH 14/21] fix hierarchy change for solvers --- core/factorization/factorization.cpp | 11 +- core/factorization/symbolic.cpp | 11 +- core/solver/bicg.cpp | 297 +++++---- core/solver/bicgstab.cpp | 347 +++++----- core/solver/cb_gmres.cpp | 623 +++++++++--------- core/solver/cg.cpp | 87 ++- core/solver/cgs.cpp | 275 ++++---- core/solver/chebyshev.cpp | 131 ++-- core/solver/direct.cpp | 56 +- core/solver/fcg.cpp | 234 ++++--- core/solver/gcr.cpp | 442 ++++++------- core/solver/gmres.cpp | 222 +++---- core/solver/idr.cpp | 138 ++-- core/solver/ir.cpp | 80 +-- core/solver/lower_trs.cpp | 31 +- core/solver/minres.cpp | 431 ++++++------ core/solver/multigrid.cpp | 117 ++-- core/solver/pipe_cg.cpp | 457 +++++++------ core/solver/solver_base.hpp | 10 +- core/solver/solver_boilerplate.hpp | 6 +- core/solver/update_residual.hpp | 10 +- core/solver/upper_trs.cpp | 31 +- .../core/factorization/factorization.hpp | 9 +- include/ginkgo/core/solver/bicg.hpp | 12 +- include/ginkgo/core/solver/bicgstab.hpp | 12 +- include/ginkgo/core/solver/cb_gmres.hpp | 12 +- include/ginkgo/core/solver/cg.hpp | 14 +- include/ginkgo/core/solver/cgs.hpp | 12 +- include/ginkgo/core/solver/chebyshev.hpp | 33 +- include/ginkgo/core/solver/direct.hpp | 9 +- include/ginkgo/core/solver/fcg.hpp | 12 +- include/ginkgo/core/solver/gcr.hpp | 9 +- include/ginkgo/core/solver/gmres.hpp | 12 +- include/ginkgo/core/solver/idr.hpp | 9 +- include/ginkgo/core/solver/ir.hpp | 33 +- include/ginkgo/core/solver/minres.hpp | 12 +- include/ginkgo/core/solver/multigrid.hpp | 22 +- include/ginkgo/core/solver/pipe_cg.hpp | 12 +- include/ginkgo/core/solver/solver_base.hpp | 125 ++-- include/ginkgo/core/solver/triangular.hpp | 18 +- include/ginkgo/core/solver/workspace.hpp | 38 +- 41 files changed, 2228 insertions(+), 2234 deletions(-) diff --git a/core/factorization/factorization.cpp b/core/factorization/factorization.cpp index d330f9cdfc2..6d936ecf017 100644 --- a/core/factorization/factorization.cpp +++ b/core/factorization/factorization.cpp @@ -330,8 +330,8 @@ Factorization::create_from_combined_ldl( template -void Factorization::apply_impl(const LinOp* b, - LinOp* x) const +void Factorization::apply_impl( + const AbstractMultiVector* b, AbstractMultiVector* x) const { switch (storage_type_) { case storage_type::composition: @@ -350,10 +350,9 @@ void Factorization::apply_impl(const LinOp* b, template -void Factorization::apply_impl(const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - LinOp* x) const +void Factorization::apply_impl( + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const { switch (storage_type_) { case storage_type::composition: diff --git a/core/factorization/symbolic.cpp b/core/factorization/symbolic.cpp index b8c0256b53c..2736a59a4dc 100644 --- a/core/factorization/symbolic.cpp +++ b/core/factorization/symbolic.cpp @@ -69,8 +69,7 @@ void symbolic_cholesky( auto lt_factor = as(factors->transpose()); const auto scalar = initialize>( {one()}, exec); - const auto id = matrix::Identity::create(exec, num_rows); - lt_factor->apply(scalar, id, scalar, factors); + factors = factors->scale_add(scalar, scalar, lt_factor); } } @@ -117,8 +116,7 @@ void symbolic_cholesky_device( auto lt_factor = as(factors->transpose()); const auto scalar = initialize>( {one()}, exec); - const auto id = matrix::Identity::create(exec, num_rows); - lt_factor->apply(scalar, id, scalar, factors); + factors = factors->scale_add(scalar, scalar, lt_factor); } } @@ -157,9 +155,8 @@ void symbolic_lu_near_symm( mtx->get_const_row_ptrs())); // compute A + A^T symbolically const auto scalar = gko::initialize({one()}, exec); - const auto symm_mtx = as(float_mtx->transpose()); - const auto id = id_type::create(exec, size[0]); - float_mtx->apply(scalar, id, scalar, symm_mtx); + const auto symm_mtx = float_mtx->scale_add( + scalar, scalar, as(float_mtx->transpose())); // compute Cholesky factorization std::unique_ptr> forest; symbolic_cholesky(symm_mtx.get(), true, symm_factors, forest); diff --git a/core/solver/bicg.cpp b/core/solver/bicg.cpp index 4d0912c5889..60cca0a1649 100644 --- a/core/solver/bicg.cpp +++ b/core/solver/bicg.cpp @@ -11,8 +11,8 @@ #include #include #include -#include +#include "core/base/dispatch_helper.hpp" #include "core/config/config_helper.hpp" #include "core/config/solver_config.hpp" #include "core/solver/bicg_kernels.hpp" @@ -95,154 +95,156 @@ std::unique_ptr conj_transpose_with_csr(const LinOp* mtx) template -void Bicg::apply_impl(const LinOp* b, LinOp* x) const +void Bicg::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); + + precision_dispatch( + [this](auto converted_b, auto converted_x) { + auto dense_b = as>(converted_b); + auto dense_x = as>(converted_x); + + using std::swap; + using Vector = matrix::MultiVector; + constexpr uint8 RelativeStoppingId{1}; + + auto exec = this->get_executor(); + this->setup_workspace(); + + GKO_SOLVER_VECTOR(r, dense_b); + GKO_SOLVER_VECTOR(z, dense_b); + GKO_SOLVER_VECTOR(p, dense_b); + GKO_SOLVER_VECTOR(q, dense_b); + GKO_SOLVER_VECTOR(r2, dense_b); + GKO_SOLVER_VECTOR(z2, dense_b); + GKO_SOLVER_VECTOR(p2, dense_b); + GKO_SOLVER_VECTOR(q2, dense_b); + + GKO_SOLVER_SCALAR(beta, dense_b); + GKO_SOLVER_SCALAR(prev_rho, dense_b); + GKO_SOLVER_SCALAR(rho, dense_b); + + GKO_SOLVER_ONE_MINUS_ONE(); + + bool one_changed{}; + GKO_SOLVER_STOP_REDUCTION_ARRAYS(converted_b->get_size()[1]); + + // rho = 0.0 + // prev_rho = 1.0 + // z = p = q = 0 + // r = r2 = dense_b + // z2 = p2 = q2 = 0 + exec->run(bicg::make_initialize( + dense_b->get_const_device_view(), r->get_device_view(), + z->get_device_view(), p->get_device_view(), + q->get_device_view(), prev_rho->get_device_view(), + rho->get_device_view(), r2->get_device_view(), + z2->get_device_view(), p2->get_device_view(), + q2->get_device_view(), stop_status)); + + std::unique_ptr conj_trans_A; + auto conj_transposable_system_matrix = + dynamic_cast( + this->get_system_matrix().get()); + + if (conj_transposable_system_matrix) { + conj_trans_A = + conj_transposable_system_matrix->conj_transpose(); + } else { + // TODO Extend when adding more IndexTypes + // Try to figure out the IndexType that can be used for the CSR + // matrix + using Csr32 = matrix::Csr; + using Csr64 = matrix::Csr; + auto supports_int64 = dynamic_cast*>( + this->get_system_matrix().get()); + if (supports_int64) { + conj_trans_A = conj_transpose_with_csr( + this->get_system_matrix().get()); + } else { + conj_trans_A = conj_transpose_with_csr( + this->get_system_matrix().get()); + } + } + + auto conj_trans_preconditioner = + as(this->get_preconditioner()) + ->conj_transpose(); + + // r = r - Ax + this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, r); + // r2 = r + r2->copy_from(r); + auto stop_criterion = this->get_stop_criterion_factory()->generate( + this->get_system_matrix(), + std::shared_ptr(dense_b, [](const Vector*) {}), + dense_x, r); + + int iter = -1; + + /* Memory movement summary: + * 28n * values + matrix/preconditioner storage + conj storage + * 2x SpMV: 4n * values + storage + conj storage + * 2x Preconditioner: 4n * values + storage + conj storage + * 2x dot 4n + * 1x step 1 (axpys) 6n + * 1x step 2 (axpys) 9n + * 1x norm2 residual n + */ + while (true) { + this->get_preconditioner()->apply(r, z); + conj_trans_preconditioner->apply(r2, z2); + z->compute_conj_dot(r2, rho, reduction_tmp); + + ++iter; + bool all_stopped = stop_criterion->update() + .num_iterations(iter) + .residual(r) + .implicit_sq_residual_norm(rho) + .solution(dense_x) + .check(RelativeStoppingId, true, + &stop_status, &one_changed); + this->template log( + this, dense_b, dense_x, iter, r, nullptr, rho, &stop_status, + all_stopped); + if (all_stopped) { + break; + } + + // tmp = rho / prev_rho + // p = z + tmp * p + // p2 = z2 + tmp * p2 + exec->run(bicg::make_step_1( + p->get_device_view(), z->get_const_device_view(), + p2->get_device_view(), z2->get_const_device_view(), + rho->get_const_device_view(), + prev_rho->get_const_device_view(), stop_status)); + // q = A * p + this->get_system_matrix()->apply(p, q); + // q2 = A^T * p2 + conj_trans_A->apply(p2, q2); + // beta = dot(p2, q) + p2->compute_conj_dot(q, beta, reduction_tmp); + // tmp = rho / beta + // x = x + tmp * p + // r = r - tmp * q + // r2 = r2 - tmp * q2 + exec->run(bicg::make_step_2( + dense_x->get_device_view(), r->get_device_view(), + r2->get_device_view(), p->get_const_device_view(), + q->get_const_device_view(), q2->get_const_device_view(), + beta->get_const_device_view(), rho->get_const_device_view(), + stop_status)); + swap(prev_rho, rho); + } }, b, x); } -template -void Bicg::apply_dense_impl( - const matrix::MultiVector* dense_b, - matrix::MultiVector* dense_x) const -{ - using std::swap; - using Vector = matrix::MultiVector; - constexpr uint8 RelativeStoppingId{1}; - - auto exec = this->get_executor(); - this->setup_workspace(); - - GKO_SOLVER_VECTOR(r, dense_b); - GKO_SOLVER_VECTOR(z, dense_b); - GKO_SOLVER_VECTOR(p, dense_b); - GKO_SOLVER_VECTOR(q, dense_b); - GKO_SOLVER_VECTOR(r2, dense_b); - GKO_SOLVER_VECTOR(z2, dense_b); - GKO_SOLVER_VECTOR(p2, dense_b); - GKO_SOLVER_VECTOR(q2, dense_b); - - GKO_SOLVER_SCALAR(beta, dense_b); - GKO_SOLVER_SCALAR(prev_rho, dense_b); - GKO_SOLVER_SCALAR(rho, dense_b); - - GKO_SOLVER_ONE_MINUS_ONE(); - - bool one_changed{}; - GKO_SOLVER_STOP_REDUCTION_ARRAYS(); - - // rho = 0.0 - // prev_rho = 1.0 - // z = p = q = 0 - // r = r2 = dense_b - // z2 = p2 = q2 = 0 - exec->run(bicg::make_initialize( - dense_b->get_const_device_view(), r->get_device_view(), - z->get_device_view(), p->get_device_view(), q->get_device_view(), - prev_rho->get_device_view(), rho->get_device_view(), - r2->get_device_view(), z2->get_device_view(), p2->get_device_view(), - q2->get_device_view(), stop_status)); - - std::unique_ptr conj_trans_A; - auto conj_transposable_system_matrix = - dynamic_cast(this->get_system_matrix().get()); - - if (conj_transposable_system_matrix) { - conj_trans_A = conj_transposable_system_matrix->conj_transpose(); - } else { - // TODO Extend when adding more IndexTypes - // Try to figure out the IndexType that can be used for the CSR matrix - using Csr32 = matrix::Csr; - using Csr64 = matrix::Csr; - auto supports_int64 = dynamic_cast*>( - this->get_system_matrix().get()); - if (supports_int64) { - conj_trans_A = - conj_transpose_with_csr(this->get_system_matrix().get()); - } else { - conj_trans_A = - conj_transpose_with_csr(this->get_system_matrix().get()); - } - } - - auto conj_trans_preconditioner = - as(this->get_preconditioner())->conj_transpose(); - - // r = r - Ax - this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, r); - // r2 = r - r2->copy_from(r); - auto stop_criterion = this->get_stop_criterion_factory()->generate( - this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, r); - - int iter = -1; - - /* Memory movement summary: - * 28n * values + matrix/preconditioner storage + conj storage - * 2x SpMV: 4n * values + storage + conj storage - * 2x Preconditioner: 4n * values + storage + conj storage - * 2x dot 4n - * 1x step 1 (axpys) 6n - * 1x step 2 (axpys) 9n - * 1x norm2 residual n - */ - while (true) { - this->get_preconditioner()->apply(r, z); - conj_trans_preconditioner->apply(r2, z2); - z->compute_conj_dot(r2, rho, reduction_tmp); - - ++iter; - bool all_stopped = - stop_criterion->update() - .num_iterations(iter) - .residual(r) - .implicit_sq_residual_norm(rho) - .solution(dense_x) - .check(RelativeStoppingId, true, &stop_status, &one_changed); - this->template log( - this, dense_b, dense_x, iter, r, nullptr, rho, &stop_status, - all_stopped); - if (all_stopped) { - break; - } - - // tmp = rho / prev_rho - // p = z + tmp * p - // p2 = z2 + tmp * p2 - exec->run(bicg::make_step_1( - p->get_device_view(), z->get_const_device_view(), - p2->get_device_view(), z2->get_const_device_view(), - rho->get_const_device_view(), prev_rho->get_const_device_view(), - stop_status)); - // q = A * p - this->get_system_matrix()->apply(p, q); - // q2 = A^T * p2 - conj_trans_A->apply(p2, q2); - // beta = dot(p2, q) - p2->compute_conj_dot(q, beta, reduction_tmp); - // tmp = rho / beta - // x = x + tmp * p - // r = r - tmp * q - // r2 = r2 - tmp * q2 - exec->run(bicg::make_step_2( - dense_x->get_device_view(), r->get_device_view(), - r2->get_device_view(), p->get_const_device_view(), - q->get_const_device_view(), q2->get_const_device_view(), - beta->get_const_device_view(), rho->get_const_device_view(), - stop_status)); - swap(prev_rho, rho); - } -} - - template Bicg::Bicg(std::shared_ptr exec) : LinOp(std::move(exec), dim<2>{}, precision_v) @@ -261,20 +263,15 @@ Bicg::Bicg(const Factory* factory, template -void Bicg::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Bicg::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/core/solver/bicgstab.cpp b/core/solver/bicgstab.cpp index 18c63ec396c..fe414ca8b63 100644 --- a/core/solver/bicgstab.cpp +++ b/core/solver/bicgstab.cpp @@ -10,10 +10,10 @@ #include #include #include -#include #include #include +#include "core/base/dispatch_helper.hpp" #include "core/config/config_helper.hpp" #include "core/config/solver_config.hpp" #include "core/distributed/helpers.hpp" @@ -77,195 +77,190 @@ std::unique_ptr Bicgstab::conj_transpose() const template -void Bicgstab::apply_impl(const LinOp* b, LinOp* x) const +void Bicgstab::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); + precision_dispatch( + [this](auto converted_b, auto converted_x) { + using std::swap; + + constexpr uint8 RelativeStoppingId{1}; + + auto exec = this->get_executor(); + this->setup_workspace(); + + GKO_SOLVER_VECTOR(r, converted_b); + GKO_SOLVER_VECTOR(z, converted_b); + GKO_SOLVER_VECTOR(y, converted_b); + GKO_SOLVER_VECTOR(v, converted_b); + GKO_SOLVER_VECTOR(s, converted_b); + GKO_SOLVER_VECTOR(t, converted_b); + GKO_SOLVER_VECTOR(p, converted_b); + GKO_SOLVER_VECTOR(rr, converted_b); + + GKO_SOLVER_SCALAR(alpha, converted_b); + GKO_SOLVER_SCALAR(beta, converted_b); + GKO_SOLVER_SCALAR(gamma, converted_b); + GKO_SOLVER_SCALAR(prev_rho, converted_b); + GKO_SOLVER_SCALAR(rho, converted_b); + GKO_SOLVER_SCALAR(omega, converted_b); + + GKO_SOLVER_ONE_MINUS_ONE(); + + bool one_changed{}; + GKO_SOLVER_STOP_REDUCTION_ARRAYS(converted_b->get_size()[1]); + + // r = converted_b + // prev_rho = rho = omega = alpha = beta = gamma = 1.0 + // rr = v = s = t = z = y = p = 0 + // stop_status = 0x00 + exec->run(bicgstab::make_initialize( + converted_b->template get_const_local_device_view(), + r->template get_local_device_view(), + rr->template get_local_device_view(), + y->template get_local_device_view(), + s->template get_local_device_view(), + t->template get_local_device_view(), + z->template get_local_device_view(), + v->template get_local_device_view(), + p->template get_local_device_view(), + prev_rho->get_device_view(), rho->get_device_view(), + alpha->get_device_view(), beta->get_device_view(), + gamma->get_device_view(), omega->get_device_view(), + stop_status)); + + // r = b - Ax + this->get_system_matrix()->apply(neg_one_op, converted_x, one_op, + r); + auto stop_criterion = this->get_stop_criterion_factory()->generate( + this->get_system_matrix(), + std::shared_ptr( + converted_b, [](const AbstractMultiVector*) {}), + converted_x, r); + // rr = r + rr->copy_from(r); + + int iter = -1; + + /* Memory movement summary: + * 31n * values + 2 * matrix/preconditioner storage + * 2x SpMV: 4n * values + 2 * storage + * 2x Preconditioner: 4n * values + 2 * storage + * 3x dot 6n + * 1x norm2 n + * 1x step 1 (fused axpys) 4n + * 1x step 2 (axpy) 3n + * 1x step 3 (fused axpys) 7n + * 2x norm2 residual 2n + */ + while (true) { + ++iter; + rr->compute_conj_dot(r, rho, reduction_tmp); + + bool all_stopped = stop_criterion->update() + .num_iterations(iter) + .residual(r) + .implicit_sq_residual_norm(rho) + .solution(converted_x) + .check(RelativeStoppingId, true, + &stop_status, &one_changed); + this->template log( + this, converted_b, converted_x, iter, r, nullptr, rho, + &stop_status, all_stopped); + if (all_stopped) { + break; + } + + // tmp = rho / prev_rho * alpha / omega + // p = r + tmp * (p - omega * v) + exec->run(bicgstab::make_step_1( + r->template get_const_local_device_view(), + p->template get_local_device_view(), + v->template get_const_local_device_view(), + rho->get_const_device_view(), + prev_rho->get_const_device_view(), + alpha->get_const_device_view(), + omega->get_const_device_view(), stop_status)); + + // y = preconditioner * p + this->get_preconditioner()->apply(p, y); + // v = A * y + this->get_system_matrix()->apply(y, v); + // beta = dot(rr, v) + rr->compute_conj_dot(v, beta, reduction_tmp); + // alpha = rho / beta + // s = r - alpha * v + exec->run(bicgstab::make_step_2( + r->template get_const_local_device_view(), + s->template get_local_device_view(), + v->template get_const_local_device_view(), + rho->get_const_device_view(), alpha->get_device_view(), + beta->get_const_device_view(), stop_status)); + + all_stopped = + stop_criterion->update() + .num_iterations(iter) + .residual(s) + .implicit_sq_residual_norm(rho) + // .solution(converted_x) // outdated at this point + .check(RelativeStoppingId, false, &stop_status, + &one_changed); + if (one_changed) { + exec->run(bicgstab::make_finalize( + converted_x + ->template get_local_device_view(), + y->template get_const_local_device_view(), + alpha->get_const_device_view(), stop_status)); + } + this->template log( + this, converted_b, converted_x, iter, r, nullptr, rho, + &stop_status, all_stopped); + if (all_stopped) { + break; + } + + // z = preconditioner * s + this->get_preconditioner()->apply(s, z); + // t = A * z + this->get_system_matrix()->apply(z, t); + // gamma = dot(s, t) + s->compute_conj_dot(t, gamma, reduction_tmp); + // beta = dot(t, t) + t->compute_conj_dot(t, beta, reduction_tmp); + // omega = gamma / beta + // x = x + alpha * y + omega * z + // r = s - omega * t + exec->run(bicgstab::make_step_3( + converted_x->template get_local_device_view(), + r->template get_local_device_view(), + s->template get_const_local_device_view(), + t->template get_const_local_device_view(), + y->template get_const_local_device_view(), + z->template get_const_local_device_view(), + alpha->get_const_device_view(), + beta->get_const_device_view(), + gamma->get_const_device_view(), omega->get_device_view(), + stop_status)); + swap(prev_rho, rho); + } }, b, x); } template -template -void Bicgstab::apply_dense_impl(const VectorType* dense_b, - VectorType* dense_x) const -{ - using std::swap; - - constexpr uint8 RelativeStoppingId{1}; - - auto exec = this->get_executor(); - this->setup_workspace(); - - GKO_SOLVER_VECTOR(r, dense_b); - GKO_SOLVER_VECTOR(z, dense_b); - GKO_SOLVER_VECTOR(y, dense_b); - GKO_SOLVER_VECTOR(v, dense_b); - GKO_SOLVER_VECTOR(s, dense_b); - GKO_SOLVER_VECTOR(t, dense_b); - GKO_SOLVER_VECTOR(p, dense_b); - GKO_SOLVER_VECTOR(rr, dense_b); - - GKO_SOLVER_SCALAR(alpha, dense_b); - GKO_SOLVER_SCALAR(beta, dense_b); - GKO_SOLVER_SCALAR(gamma, dense_b); - GKO_SOLVER_SCALAR(prev_rho, dense_b); - GKO_SOLVER_SCALAR(rho, dense_b); - GKO_SOLVER_SCALAR(omega, dense_b); - - GKO_SOLVER_ONE_MINUS_ONE(); - - bool one_changed{}; - GKO_SOLVER_STOP_REDUCTION_ARRAYS(); - - // r = dense_b - // prev_rho = rho = omega = alpha = beta = gamma = 1.0 - // rr = v = s = t = z = y = p = 0 - // stop_status = 0x00 - exec->run(bicgstab::make_initialize( - gko::detail::get_local(dense_b)->get_const_device_view(), - gko::detail::get_local(r)->get_device_view(), - gko::detail::get_local(rr)->get_device_view(), - gko::detail::get_local(y)->get_device_view(), - gko::detail::get_local(s)->get_device_view(), - gko::detail::get_local(t)->get_device_view(), - gko::detail::get_local(z)->get_device_view(), - gko::detail::get_local(v)->get_device_view(), - gko::detail::get_local(p)->get_device_view(), - prev_rho->get_device_view(), rho->get_device_view(), - alpha->get_device_view(), beta->get_device_view(), - gamma->get_device_view(), omega->get_device_view(), stop_status)); - - // r = b - Ax - this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, r); - auto stop_criterion = this->get_stop_criterion_factory()->generate( - this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, r); - // rr = r - rr->copy_from(r); - - int iter = -1; - - /* Memory movement summary: - * 31n * values + 2 * matrix/preconditioner storage - * 2x SpMV: 4n * values + 2 * storage - * 2x Preconditioner: 4n * values + 2 * storage - * 3x dot 6n - * 1x norm2 n - * 1x step 1 (fused axpys) 4n - * 1x step 2 (axpy) 3n - * 1x step 3 (fused axpys) 7n - * 2x norm2 residual 2n - */ - while (true) { - ++iter; - rr->compute_conj_dot(r, rho, reduction_tmp); - - bool all_stopped = - stop_criterion->update() - .num_iterations(iter) - .residual(r) - .implicit_sq_residual_norm(rho) - .solution(dense_x) - .check(RelativeStoppingId, true, &stop_status, &one_changed); - this->template log( - this, dense_b, dense_x, iter, r, nullptr, rho, &stop_status, - all_stopped); - if (all_stopped) { - break; - } - - // tmp = rho / prev_rho * alpha / omega - // p = r + tmp * (p - omega * v) - exec->run(bicgstab::make_step_1( - gko::detail::get_local(r)->get_const_device_view(), - gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(v)->get_const_device_view(), - rho->get_const_device_view(), prev_rho->get_const_device_view(), - alpha->get_const_device_view(), omega->get_const_device_view(), - stop_status)); - - // y = preconditioner * p - this->get_preconditioner()->apply(p, y); - // v = A * y - this->get_system_matrix()->apply(y, v); - // beta = dot(rr, v) - rr->compute_conj_dot(v, beta, reduction_tmp); - // alpha = rho / beta - // s = r - alpha * v - exec->run(bicgstab::make_step_2( - gko::detail::get_local(r)->get_const_device_view(), - gko::detail::get_local(s)->get_device_view(), - gko::detail::get_local(v)->get_const_device_view(), - rho->get_const_device_view(), alpha->get_device_view(), - beta->get_const_device_view(), stop_status)); - - all_stopped = - stop_criterion->update() - .num_iterations(iter) - .residual(s) - .implicit_sq_residual_norm(rho) - // .solution(dense_x) // outdated at this point - .check(RelativeStoppingId, false, &stop_status, &one_changed); - if (one_changed) { - exec->run(bicgstab::make_finalize( - gko::detail::get_local(dense_x)->get_device_view(), - gko::detail::get_local(y)->get_const_device_view(), - alpha->get_const_device_view(), stop_status)); - } - this->template log( - this, dense_b, dense_x, iter, r, nullptr, rho, &stop_status, - all_stopped); - if (all_stopped) { - break; - } - - // z = preconditioner * s - this->get_preconditioner()->apply(s, z); - // t = A * z - this->get_system_matrix()->apply(z, t); - // gamma = dot(s, t) - s->compute_conj_dot(t, gamma, reduction_tmp); - // beta = dot(t, t) - t->compute_conj_dot(t, beta, reduction_tmp); - // omega = gamma / beta - // x = x + alpha * y + omega * z - // r = s - omega * t - exec->run(bicgstab::make_step_3( - gko::detail::get_local(dense_x)->get_device_view(), - gko::detail::get_local(r)->get_device_view(), - gko::detail::get_local(s)->get_const_device_view(), - gko::detail::get_local(t)->get_const_device_view(), - gko::detail::get_local(y)->get_const_device_view(), - gko::detail::get_local(z)->get_const_device_view(), - alpha->get_const_device_view(), beta->get_const_device_view(), - gamma->get_const_device_view(), omega->get_device_view(), - stop_status)); - swap(prev_rho, rho); - } -} - - -template -void Bicgstab::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Bicgstab::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp index 570bf883bca..6135836fb95 100644 --- a/core/solver/cb_gmres.cpp +++ b/core/solver/cb_gmres.cpp @@ -12,12 +12,12 @@ #include #include #include -#include #include #include #include #include +#include "core/base/dispatch_helper.hpp" #include "core/base/extended_float.hpp" #include "core/config/config_helper.hpp" #include "core/config/solver_config.hpp" @@ -213,337 +213,366 @@ CbGmres::CbGmres(const Factory* factory, template -void CbGmres::apply_impl(const LinOp* b, LinOp* x) const +void CbGmres::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); - }, - b, x); -} + precision_dispatch( + [this](auto converted_b, auto converted_x) { + using Vector = matrix::MultiVector; + + auto dense_b = as(converted_b); + auto dense_x = as(converted_x); + + // Current workaround to get a lambda with a template argument (only + // the type of `value` matters, the content does not) + auto apply_templated = [&](auto value) { + using storage_type = decltype(value); + + using VectorNorms = + matrix::MultiVector>; + using Range3dHelper = + gko::cb_gmres::Range3dHelper; + + constexpr uint8 RelativeStoppingId{1}; + + auto exec = this->get_executor(); + + auto one_op = initialize({one()}, exec); + auto neg_one_op = initialize({-one()}, exec); + + const auto num_rows = this->get_size()[0]; + const auto num_rhs = dense_b->get_size()[1]; + const auto krylov_dim = this->get_krylov_dim(); + auto residual = Vector::create_with_config_of(dense_b); + /* The dimensions {x, y, z} explained for the krylov_bases: + * - x: selects the krylov vector (which has krylov_dim + 1 + * vectors) + * - y: selects the (row-)element of said krylov vector + * - z: selects which column-element of said krylov vector + * should be used + */ + const dim<3> krylov_bases_dim{krylov_dim + 1, num_rows, + num_rhs}; + Range3dHelper helper(exec, krylov_bases_dim); + auto krylov_bases_range = helper.get_range(); + + auto next_krylov_basis = Vector::create_with_config_of(dense_b); + std::shared_ptr> + preconditioned_vector = + Vector::create_with_config_of(dense_b); + auto hessenberg = Vector::create( + exec, dim<2>{krylov_dim + 1, krylov_dim * num_rhs}); + auto buffer = + Vector::create(exec, dim<2>{krylov_dim + 1, num_rhs}); + auto givens_sin = + Vector::create(exec, dim<2>{krylov_dim, num_rhs}); + auto givens_cos = + Vector::create(exec, dim<2>{krylov_dim, num_rhs}); + auto residual_norm_collection = + Vector::create(exec, dim<2>{krylov_dim + 1, num_rhs}); + auto residual_norm = + VectorNorms::create(exec, dim<2>{1, num_rhs}); + // 1st row of arnoldi_norm: == eta * + // norm2(old_next_krylov_basis) + // with eta == 1 / sqrt(2) + // (computed right before updating + // next_krylov_basis) + // 2nd row of arnoldi_norm: The actual arnoldi norm + // == norm2(next_krylov_basis) + // 3rd row of arnoldi_norm: the infinity norm of + // next_krylov_basis + // (ONLY when using a scalar accessor) + auto arnoldi_norm = + VectorNorms::create(exec, dim<2>{3, num_rhs}); + array final_iter_nums(this->get_executor(), num_rhs); + auto y = Vector::create(exec, dim<2>{krylov_dim, num_rhs}); + + bool one_changed{}; + array reduction_tmp{this->get_executor()}; + array stop_status(this->get_executor(), + num_rhs); + // reorth_status and num_reorth are both helper variables for + // GPU implementations at the moment. num_reorth := Number of + // vectors which require a re-orthogonalization reorth_status := + // stopping status for the re-orthogonalization, + // marking which RHS requires one, and which + // does not + array reorth_status(this->get_executor(), + num_rhs); + array num_reorth(this->get_executor(), 1); + + // Initialization + exec->run(cb_gmres::make_initialize( + dense_b->get_const_device_view(), + residual->get_device_view(), givens_sin->get_device_view(), + givens_cos->get_device_view(), stop_status, krylov_dim)); + // residual = dense_b + // givens_sin = givens_cos = 0 + this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, + residual); + // residual = residual - Ax + exec->run(cb_gmres::make_restart( + residual->get_const_device_view(), + residual_norm->get_device_view(), + residual_norm_collection->get_device_view(), + arnoldi_norm->get_device_view(), krylov_bases_range, + next_krylov_basis->get_device_view(), final_iter_nums, + reduction_tmp, krylov_dim)); + // residual_norm = norm(residual) + // residual_norm_collection = {residual_norm, 0, ..., 0} + // krylov_bases(:, 1) = residual / residual_norm + // next_krylov_basis = residual / residual_norm + // final_iter_nums = {0, ..., 0} -template -void CbGmres::apply_dense_impl( - const matrix::MultiVector* dense_b, - matrix::MultiVector* dense_x) const -{ - // Current workaround to get a lambda with a template argument (only - // the type of `value` matters, the content does not) - auto apply_templated = [&](auto value) { - using storage_type = decltype(value); - - using Vector = matrix::MultiVector; - using VectorNorms = matrix::MultiVector>; - using Range3dHelper = - gko::cb_gmres::Range3dHelper; - - - constexpr uint8 RelativeStoppingId{1}; - - auto exec = this->get_executor(); - - auto one_op = initialize({one()}, exec); - auto neg_one_op = initialize({-one()}, exec); - - const auto num_rows = this->get_size()[0]; - const auto num_rhs = dense_b->get_size()[1]; - const auto krylov_dim = this->get_krylov_dim(); - auto residual = Vector::create_with_config_of(dense_b); - /* The dimensions {x, y, z} explained for the krylov_bases: - * - x: selects the krylov vector (which has krylov_dim + 1 vectors) - * - y: selects the (row-)element of said krylov vector - * - z: selects which column-element of said krylov vector should be - * used - */ - const dim<3> krylov_bases_dim{krylov_dim + 1, num_rows, num_rhs}; - Range3dHelper helper(exec, krylov_bases_dim); - auto krylov_bases_range = helper.get_range(); - - auto next_krylov_basis = Vector::create_with_config_of(dense_b); - std::shared_ptr> preconditioned_vector = - Vector::create_with_config_of(dense_b); - auto hessenberg = - Vector::create(exec, dim<2>{krylov_dim + 1, krylov_dim * num_rhs}); - auto buffer = Vector::create(exec, dim<2>{krylov_dim + 1, num_rhs}); - auto givens_sin = Vector::create(exec, dim<2>{krylov_dim, num_rhs}); - auto givens_cos = Vector::create(exec, dim<2>{krylov_dim, num_rhs}); - auto residual_norm_collection = - Vector::create(exec, dim<2>{krylov_dim + 1, num_rhs}); - auto residual_norm = VectorNorms::create(exec, dim<2>{1, num_rhs}); - // 1st row of arnoldi_norm: == eta * norm2(old_next_krylov_basis) - // with eta == 1 / sqrt(2) - // (computed right before updating - // next_krylov_basis) - // 2nd row of arnoldi_norm: The actual arnoldi norm - // == norm2(next_krylov_basis) - // 3rd row of arnoldi_norm: the infinity norm of next_krylov_basis - // (ONLY when using a scalar accessor) - auto arnoldi_norm = VectorNorms::create(exec, dim<2>{3, num_rhs}); - array final_iter_nums(this->get_executor(), num_rhs); - auto y = Vector::create(exec, dim<2>{krylov_dim, num_rhs}); - - bool one_changed{}; - array reduction_tmp{this->get_executor()}; - array stop_status(this->get_executor(), num_rhs); - // reorth_status and num_reorth are both helper variables for GPU - // implementations at the moment. - // num_reorth := Number of vectors which require a re-orthogonalization - // reorth_status := stopping status for the re-orthogonalization, - // marking which RHS requires one, and which does not - array reorth_status(this->get_executor(), num_rhs); - array num_reorth(this->get_executor(), 1); - - // Initialization - exec->run(cb_gmres::make_initialize( - dense_b->get_const_device_view(), residual->get_device_view(), - givens_sin->get_device_view(), givens_cos->get_device_view(), - stop_status, krylov_dim)); - // residual = dense_b - // givens_sin = givens_cos = 0 - this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, residual); - // residual = residual - Ax - - exec->run(cb_gmres::make_restart( - residual->get_const_device_view(), residual_norm->get_device_view(), - residual_norm_collection->get_device_view(), - arnoldi_norm->get_device_view(), krylov_bases_range, - next_krylov_basis->get_device_view(), final_iter_nums, - reduction_tmp, krylov_dim)); - // residual_norm = norm(residual) - // residual_norm_collection = {residual_norm, 0, ..., 0} - // krylov_bases(:, 1) = residual / residual_norm - // next_krylov_basis = residual / residual_norm - // final_iter_nums = {0, ..., 0} - - auto stop_criterion = this->get_stop_criterion_factory()->generate( - this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, - residual.get()); - - int total_iter = -1; - size_type restart_iter = 0; - - auto before_preconditioner = - matrix::MultiVector::create_with_config_of(dense_x); - auto after_preconditioner = - matrix::MultiVector::create_with_config_of(dense_x); - - array stop_encountered_rhs(exec->get_master(), num_rhs); - array fully_converged_rhs(exec->get_master(), num_rhs); - array host_stop_status( - this->get_executor()->get_master(), stop_status); - for (size_type i = 0; i < stop_encountered_rhs.get_size(); ++i) { - stop_encountered_rhs.get_data()[i] = false; - fully_converged_rhs.get_data()[i] = false; - } - // Start only after this value with performing forced iterations after - // convergence detection - constexpr int start_force_reset{10}; - bool perform_reset{false}; - // Fraction of the krylov_dim (or total_iter if it is lower), - // determining the number of forced iteration to perform - constexpr size_type forced_iteration_fraction{10}; - const size_type forced_limit{krylov_dim / forced_iteration_fraction}; - // Counter for the forced iterations. Start at max in order to properly - // test convergence at the beginning - size_type forced_iterations{forced_limit}; - - while (true) { - ++total_iter; - // In the beginning, only force a fraction of the total iterations - if (forced_iterations < forced_limit && - forced_iterations < total_iter / forced_iteration_fraction) { - this->template log( - this, dense_b, dense_x, total_iter, residual.get(), - residual_norm.get(), nullptr, &stop_status, false); - ++forced_iterations; - } else { - bool all_changed = stop_criterion->update() - .num_iterations(total_iter) - .residual(residual) - .residual_norm(residual_norm) - .solution(dense_x) - .check(RelativeStoppingId, true, - &stop_status, &one_changed); - this->template log( - this, dense_b, dense_x, total_iter, residual.get(), - residual_norm.get(), nullptr, &stop_status, all_changed); - if (one_changed || all_changed) { - host_stop_status = stop_status; - bool host_array_changed{false}; - for (size_type i = 0; i < host_stop_status.get_size(); - ++i) { - auto local_status = host_stop_status.get_data() + i; - // Ignore all actually converged ones! - if (fully_converged_rhs.get_data()[i]) { - continue; - } - if (local_status->has_converged()) { - // If convergence was detected earlier, or - // at the very beginning: - if (stop_encountered_rhs.get_data()[i] || - total_iter < start_force_reset) { - fully_converged_rhs.get_data()[i] = true; + auto stop_criterion = + this->get_stop_criterion_factory()->generate( + this->get_system_matrix(), + std::shared_ptr(dense_b, + [](const Vector*) {}), + dense_x, residual.get()); + + int total_iter = -1; + size_type restart_iter = 0; + + auto before_preconditioner = + matrix::MultiVector::create_with_config_of( + dense_x); + auto after_preconditioner = + matrix::MultiVector::create_with_config_of( + dense_x); + + array stop_encountered_rhs(exec->get_master(), num_rhs); + array fully_converged_rhs(exec->get_master(), num_rhs); + array host_stop_status( + this->get_executor()->get_master(), stop_status); + for (size_type i = 0; i < stop_encountered_rhs.get_size(); + ++i) { + stop_encountered_rhs.get_data()[i] = false; + fully_converged_rhs.get_data()[i] = false; + } + // Start only after this value with performing forced iterations + // after convergence detection + constexpr int start_force_reset{10}; + bool perform_reset{false}; + // Fraction of the krylov_dim (or total_iter if it is lower), + // determining the number of forced iteration to perform + constexpr size_type forced_iteration_fraction{10}; + const size_type forced_limit{krylov_dim / + forced_iteration_fraction}; + // Counter for the forced iterations. Start at max in order to + // properly test convergence at the beginning + size_type forced_iterations{forced_limit}; + + while (true) { + ++total_iter; + // In the beginning, only force a fraction of the total + // iterations + if (forced_iterations < forced_limit && + forced_iterations < + total_iter / forced_iteration_fraction) { + this->template log( + this, dense_b, dense_x, total_iter, residual.get(), + residual_norm.get(), nullptr, &stop_status, false); + ++forced_iterations; + } else { + bool all_changed = + stop_criterion->update() + .num_iterations(total_iter) + .residual(residual) + .residual_norm(residual_norm) + .solution(dense_x) + .check(RelativeStoppingId, true, &stop_status, + &one_changed); + this->template log( + this, dense_b, dense_x, total_iter, residual.get(), + residual_norm.get(), nullptr, &stop_status, + all_changed); + if (one_changed || all_changed) { + host_stop_status = stop_status; + bool host_array_changed{false}; + for (size_type i = 0; + i < host_stop_status.get_size(); ++i) { + auto local_status = + host_stop_status.get_data() + i; + // Ignore all actually converged ones! + if (fully_converged_rhs.get_data()[i]) { + continue; + } + if (local_status->has_converged()) { + // If convergence was detected earlier, or + // at the very beginning: + if (stop_encountered_rhs.get_data()[i] || + total_iter < start_force_reset) { + fully_converged_rhs.get_data()[i] = + true; + } else { + stop_encountered_rhs.get_data()[i] = + true; + local_status->reset(); + host_array_changed = true; + } + } + } + if (host_array_changed) { + perform_reset = true; + stop_status = host_stop_status; } else { - stop_encountered_rhs.get_data()[i] = true; - local_status->reset(); - host_array_changed = true; + // Stop here can happen if all RHS are + // "fully_converged" or if it was stopped for + // non-convergence reason (like time or + // iteration) + break; + } + forced_iterations = 0; + + } else { + for (size_type i = 0; + i < stop_encountered_rhs.get_size(); ++i) { + stop_encountered_rhs.get_data()[i] = false; } } } - if (host_array_changed) { - perform_reset = true; - stop_status = host_stop_status; - } else { - // Stop here can happen if all RHS are "fully_converged" - // or if it was stopped for non-convergence reason - // (like time or iteration) - break; - } - forced_iterations = 0; - } else { - for (size_type i = 0; i < stop_encountered_rhs.get_size(); - ++i) { - stop_encountered_rhs.get_data()[i] = false; + if (perform_reset || restart_iter == krylov_dim) { + perform_reset = false; + // Restart + // use a view in case this is called earlier + auto hessenberg_view = hessenberg->create_subview( + local_span{0, restart_iter}, + local_span{0, num_rhs * (restart_iter)}); + + exec->run(cb_gmres::make_solve_krylov( + residual_norm_collection->get_const_device_view(), + krylov_bases_range.get_accessor().to_const(), + hessenberg_view->get_const_device_view(), + y->get_device_view(), + before_preconditioner->get_device_view(), + final_iter_nums)); + // Solve upper triangular. + // y = hessenberg \ residual_norm_collection + + this->get_preconditioner()->apply(before_preconditioner, + after_preconditioner); + dense_x->add_scaled(one_op, after_preconditioner); + // Solve x + // x = x + get_preconditioner() * krylov_bases * y + residual->copy_from(dense_b); + // residual = dense_b + this->get_system_matrix()->apply(neg_one_op, dense_x, + one_op, residual); + // residual = residual - Ax + exec->run(cb_gmres::make_restart( + residual->get_const_device_view(), + residual_norm->get_device_view(), + residual_norm_collection->get_device_view(), + arnoldi_norm->get_device_view(), krylov_bases_range, + next_krylov_basis->get_device_view(), + final_iter_nums, reduction_tmp, krylov_dim)); + // residual_norm = norm(residual) + // residual_norm_collection = {residual_norm, 0, ..., 0} + // krylov_bases(:, 1) = residual / residual_norm + // next_krylov_basis = residual / residual_norm + // final_iter_nums = {0, ..., 0} + restart_iter = 0; } - } - } - if (perform_reset || restart_iter == krylov_dim) { - perform_reset = false; - // Restart - // use a view in case this is called earlier - auto hessenberg_view = hessenberg->create_submatrix( - span{0, restart_iter}, span{0, num_rhs * (restart_iter)}); + this->get_preconditioner()->apply(next_krylov_basis, + preconditioned_vector); + // preconditioned_vector = get_preconditioner() * + // next_krylov_basis + + // Do Arnoldi and givens rotation + auto hessenberg_iter = hessenberg->create_subview( + local_span{0, restart_iter + 2}, + local_span{num_rhs * restart_iter, + num_rhs * (restart_iter + 1)}); + auto buffer_iter = + buffer->create_subview(local_span{0, restart_iter + 2}, + local_span{0, num_rhs}); + + // Start of arnoldi + this->get_system_matrix()->apply(preconditioned_vector, + next_krylov_basis); + // next_krylov_basis = A * preconditioned_vector + exec->run(cb_gmres::make_arnoldi( + next_krylov_basis->get_device_view(), + givens_sin->get_device_view(), + givens_cos->get_device_view(), + residual_norm->get_device_view(), + residual_norm_collection->get_device_view(), + krylov_bases_range, hessenberg_iter->get_device_view(), + buffer_iter->get_device_view(), + arnoldi_norm->get_device_view(), restart_iter, + final_iter_nums, stop_status, reorth_status, + num_reorth)); + // for i in 0:restart_iter + // hessenberg(restart_iter, i) = next_krylov_basis' * + // krylov_bases(:, i) next_krylov_basis -= + // hessenberg(restart_iter, i) * krylov_bases(:, i) + // end + // hessenberg(restart_iter, restart_iter + 1) = + // norm(next_krylov_basis) next_krylov_basis /= + // hessenberg(restart_iter, restart_iter + 1) End of arnoldi + // Start apply givens rotation for j in 0:restart_iter + // temp = cos(j)*hessenberg(j) + + // sin(j)*hessenberg(j+1) + // hessenberg(j+1) = -sin(j)*hessenberg(j) + + // cos(j)*hessenberg(j+1) + // hessenberg(j) = temp; + // end + // Calculate sin and cos + // hessenberg(restart_iter) = + // cos(restart_iter)*hessenberg(restart_iter) + + // sin(restart_iter)*hessenberg(restart_iter) + // hessenberg(restart_iter+1) = 0 + // End apply givens rotation + // Calculate residual norm + + restart_iter++; + } // closes while(true) + // Solve x + + auto hessenberg_small = hessenberg->create_subview( + local_span{0, restart_iter}, + local_span{0, num_rhs * restart_iter}); exec->run(cb_gmres::make_solve_krylov( residual_norm_collection->get_const_device_view(), krylov_bases_range.get_accessor().to_const(), - hessenberg_view->get_const_device_view(), + hessenberg_small->get_const_device_view(), y->get_device_view(), before_preconditioner->get_device_view(), final_iter_nums)); // Solve upper triangular. // y = hessenberg \ residual_norm_collection - this->get_preconditioner()->apply(before_preconditioner, after_preconditioner); dense_x->add_scaled(one_op, after_preconditioner); // Solve x // x = x + get_preconditioner() * krylov_bases * y - residual->copy_from(dense_b); - // residual = dense_b - this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, - residual); - // residual = residual - Ax - exec->run(cb_gmres::make_restart( - residual->get_const_device_view(), - residual_norm->get_device_view(), - residual_norm_collection->get_device_view(), - arnoldi_norm->get_device_view(), krylov_bases_range, - next_krylov_basis->get_device_view(), final_iter_nums, - reduction_tmp, krylov_dim)); - // residual_norm = norm(residual) - // residual_norm_collection = {residual_norm, 0, ..., 0} - // krylov_bases(:, 1) = residual / residual_norm - // next_krylov_basis = residual / residual_norm - // final_iter_nums = {0, ..., 0} - restart_iter = 0; - } + }; // End of apply_lambda - this->get_preconditioner()->apply(next_krylov_basis, - preconditioned_vector); - // preconditioned_vector = get_preconditioner() * - // next_krylov_basis - - // Do Arnoldi and givens rotation - auto hessenberg_iter = hessenberg->create_submatrix( - span{0, restart_iter + 2}, - span{num_rhs * restart_iter, num_rhs * (restart_iter + 1)}); - auto buffer_iter = buffer->create_submatrix( - span{0, restart_iter + 2}, span{0, num_rhs}); - - // Start of arnoldi - this->get_system_matrix()->apply(preconditioned_vector, - next_krylov_basis); - // next_krylov_basis = A * preconditioned_vector - exec->run(cb_gmres::make_arnoldi( - next_krylov_basis->get_device_view(), - givens_sin->get_device_view(), givens_cos->get_device_view(), - residual_norm->get_device_view(), - residual_norm_collection->get_device_view(), krylov_bases_range, - hessenberg_iter->get_device_view(), - buffer_iter->get_device_view(), arnoldi_norm->get_device_view(), - restart_iter, final_iter_nums, stop_status, reorth_status, - num_reorth)); - // for i in 0:restart_iter - // hessenberg(restart_iter, i) = next_krylov_basis' * - // krylov_bases(:, i) next_krylov_basis -= - // hessenberg(restart_iter, i) * krylov_bases(:, i) - // end - // hessenberg(restart_iter, restart_iter + 1) = - // norm(next_krylov_basis) next_krylov_basis /= - // hessenberg(restart_iter, restart_iter + 1) End of arnoldi - // Start apply givens rotation for j in 0:restart_iter - // temp = cos(j)*hessenberg(j) + - // sin(j)*hessenberg(j+1) - // hessenberg(j+1) = -sin(j)*hessenberg(j) + - // cos(j)*hessenberg(j+1) - // hessenberg(j) = temp; - // end - // Calculate sin and cos - // hessenberg(restart_iter) = - // cos(restart_iter)*hessenberg(restart_iter) + - // sin(restart_iter)*hessenberg(restart_iter) - // hessenberg(restart_iter+1) = 0 - // End apply givens rotation - // Calculate residual norm - - restart_iter++; - } // closes while(true) - // Solve x - - auto hessenberg_small = hessenberg->create_submatrix( - span{0, restart_iter}, span{0, num_rhs * restart_iter}); - - exec->run(cb_gmres::make_solve_krylov( - residual_norm_collection->get_const_device_view(), - krylov_bases_range.get_accessor().to_const(), - hessenberg_small->get_const_device_view(), y->get_device_view(), - before_preconditioner->get_device_view(), final_iter_nums)); - // Solve upper triangular. - // y = hessenberg \ residual_norm_collection - this->get_preconditioner()->apply(before_preconditioner, - after_preconditioner); - dense_x->add_scaled(one_op, after_preconditioner); - // Solve x - // x = x + get_preconditioner() * krylov_bases * y - }; // End of apply_lambda - - // Look which precision to use as the storage type - helper::call(apply_templated, this->get_storage_precision()); + // Look which precision to use as the storage type + helper::call(apply_templated, + this->get_storage_precision()); + }, + b, x); } template -void CbGmres::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void CbGmres::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } #define GKO_DECLARE_CB_GMRES(_type1) class CbGmres<_type1> diff --git a/core/solver/cg.cpp b/core/solver/cg.cpp index 743da453dc3..c01ff3ff5da 100644 --- a/core/solver/cg.cpp +++ b/core/solver/cg.cpp @@ -10,13 +10,9 @@ #include #include #include -#include -#include -#include #include "core/config/config_helper.hpp" #include "core/config/solver_config.hpp" -#include "core/distributed/helpers.hpp" #include "core/solver/cg_kernels.hpp" #include "core/solver/solver_boilerplate.hpp" @@ -75,26 +71,26 @@ std::unique_ptr Cg::conj_transpose() const template -void Cg::apply_impl(const LinOp* b, LinOp* x) const +bool Cg::apply_uses_initial_guess() const { - if (!this->get_system_matrix()) { - return; - } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); - }, - b, x); + return true; } template -template -void Cg::apply_dense_impl(const VectorType* dense_b, - VectorType* dense_x) const +void Cg::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { + if (!this->get_system_matrix()) { + return; + } + using std::swap; - using LocalVector = matrix::MultiVector; + + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); constexpr uint8 RelativeStoppingId{1}; @@ -113,24 +109,26 @@ void Cg::apply_dense_impl(const VectorType* dense_b, GKO_SOLVER_ONE_MINUS_ONE(); bool one_changed{}; - GKO_SOLVER_STOP_REDUCTION_ARRAYS(); + GKO_SOLVER_STOP_REDUCTION_ARRAYS(converted_b->get_size()[1]); // r = dense_b // rho = 0.0 // prev_rho = 1.0 // z = p = q = 0 exec->run(cg::make_initialize( - gko::detail::get_local(dense_b)->get_const_device_view(), - gko::detail::get_local(r)->get_device_view(), - gko::detail::get_local(z)->get_device_view(), - gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(q)->get_device_view(), + dense_b->template get_const_local_device_view(), + r->template get_local_device_view(), + z->template get_local_device_view(), + p->template get_local_device_view(), + q->template get_local_device_view(), prev_rho->get_device_view(), rho->get_device_view(), stop_status)); this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, r); auto stop_criterion = this->get_stop_criterion_factory()->generate( this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, r); + std::shared_ptr( + dense_b, [](const AbstractMultiVector*) {}), + dense_x, r); int iter = -1; /* Memory movement summary: @@ -165,11 +163,11 @@ void Cg::apply_dense_impl(const VectorType* dense_b, // tmp = rho / prev_rho // p = z + tmp * p - exec->run( - cg::make_step_1(gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(z)->get_const_device_view(), - rho->get_const_device_view(), - prev_rho->get_const_device_view(), stop_status)); + exec->run(cg::make_step_1( + p->template get_local_device_view(), + z->template get_const_local_device_view(), + rho->get_const_device_view(), prev_rho->get_const_device_view(), + stop_status)); // q = A * p this->get_system_matrix()->apply(p, q); // beta = dot(p, q) @@ -177,39 +175,33 @@ void Cg::apply_dense_impl(const VectorType* dense_b, // tmp = rho / beta // x = x + tmp * p // r = r - tmp * q - exec->run( - cg::make_step_2(gko::detail::get_local(dense_x)->get_device_view(), - gko::detail::get_local(r)->get_device_view(), - gko::detail::get_local(p)->get_const_device_view(), - gko::detail::get_local(q)->get_const_device_view(), - beta->get_const_device_view(), - rho->get_const_device_view(), stop_status)); + exec->run(cg::make_step_2( + dense_x->template get_local_device_view(), + r->template get_local_device_view(), + p->template get_const_local_device_view(), + q->template get_const_local_device_view(), + beta->get_const_device_view(), rho->get_const_device_view(), + stop_status)); swap(prev_rho, rho); } } template -void Cg::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Cg::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } template Cg::Cg(std::shared_ptr exec) - : LinOp(std::move(exec), dim<2>{}, precision_v) {} @@ -217,7 +209,6 @@ Cg::Cg(std::shared_ptr exec) template Cg::Cg(const Factory* factory, std::shared_ptr system_matrix) - : LinOp(factory->get_executor(), gko::transpose(system_matrix->get_size()), precision_v), EnablePreconditionedIterativeSolver>{ diff --git a/core/solver/cgs.cpp b/core/solver/cgs.cpp index 01887f72d5a..8b693b2ec39 100644 --- a/core/solver/cgs.cpp +++ b/core/solver/cgs.cpp @@ -10,15 +10,17 @@ #include #include #include -#include #include #include +#include "core/base/dispatch_helper.hpp" #include "core/config/config_helper.hpp" #include "core/config/solver_config.hpp" #include "core/distributed/helpers.hpp" #include "core/solver/cgs_kernels.hpp" #include "core/solver/solver_boilerplate.hpp" + + namespace gko { namespace solver { namespace cgs { @@ -75,160 +77,151 @@ std::unique_ptr Cgs::conj_transpose() const template -void Cgs::apply_impl(const LinOp* b, LinOp* x) const +void Cgs::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); + + precision_dispatch( + [this](auto converted_b, auto converted_x) { + using std::swap; + + constexpr uint8 RelativeStoppingId{1}; + + auto exec = this->get_executor(); + this->setup_workspace(); + + GKO_SOLVER_VECTOR(r, converted_b); + GKO_SOLVER_VECTOR(r_tld, converted_b); + GKO_SOLVER_VECTOR(p, converted_b); + GKO_SOLVER_VECTOR(q, converted_b); + GKO_SOLVER_VECTOR(u, converted_b); + GKO_SOLVER_VECTOR(u_hat, converted_b); + GKO_SOLVER_VECTOR(v_hat, converted_b); + GKO_SOLVER_VECTOR(t, converted_b); + + GKO_SOLVER_SCALAR(alpha, converted_b); + GKO_SOLVER_SCALAR(beta, converted_b); + GKO_SOLVER_SCALAR(gamma, converted_b); + GKO_SOLVER_SCALAR(prev_rho, converted_b); + GKO_SOLVER_SCALAR(rho, converted_b); + + GKO_SOLVER_ONE_MINUS_ONE(); + + bool one_changed{}; + GKO_SOLVER_STOP_REDUCTION_ARRAYS(converted_b->get_size()[1]); + + // r = converted_b + // r_tld = r + // rho = 0.0 + // prev_rho = alpha = beta = gamma = 1.0 + // p = q = u = u_hat = v_hat = t = 0 + exec->run(cgs::make_initialize( + converted_b->template get_const_local_device_view(), + r->template get_local_device_view(), + r_tld->template get_local_device_view(), + p->template get_local_device_view(), + q->template get_local_device_view(), + u->template get_local_device_view(), + u_hat->template get_local_device_view(), + v_hat->template get_local_device_view(), + t->template get_local_device_view(), + alpha->get_device_view(), beta->get_device_view(), + gamma->get_device_view(), prev_rho->get_device_view(), + rho->get_device_view(), stop_status)); + + this->get_system_matrix()->apply(neg_one_op, converted_x, one_op, + r); + auto stop_criterion = this->get_stop_criterion_factory()->generate( + this->get_system_matrix(), + std::shared_ptr( + converted_b, [](const AbstractMultiVector*) {}), + converted_x, r); + r_tld->copy_from(r); + + int iter = -1; + /* Memory movement summary: + * 28n * values + 2 * matrix/preconditioner storage + * 2x SpMV: 4n * values + 2 * storage + * 2x Preconditioner: 4n * values + 2 * storage + * 2x dot 4n + * 1x step 1 (fused axpys) 5n + * 1x step 2 (fused axpys) 4n + * 1x step 3 (axpys) 6n + * 1x norm2 residual n + */ + while (true) { + r->compute_conj_dot(r_tld, rho, reduction_tmp); + + ++iter; + bool all_stopped = stop_criterion->update() + .num_iterations(iter) + .residual(r) + .implicit_sq_residual_norm(rho) + .solution(converted_x) + .check(RelativeStoppingId, true, + &stop_status, &one_changed); + this->template log( + this, converted_b, converted_x, iter, r, nullptr, rho, + &stop_status, all_stopped); + if (all_stopped) { + break; + } + + // beta = rho / prev_rho + // u = r + beta * q + // p = u + beta * ( q + beta * p ) + exec->run(cgs::make_step_1( + r->template get_const_local_device_view(), + u->template get_local_device_view(), + p->template get_local_device_view(), + q->template get_const_local_device_view(), + beta->get_device_view(), rho->get_const_device_view(), + prev_rho->get_const_device_view(), stop_status)); + this->get_preconditioner()->apply(p, t); + this->get_system_matrix()->apply(t, v_hat); + r_tld->compute_conj_dot(v_hat, gamma, reduction_tmp); + // alpha = rho / gamma + // q = u - alpha * v_hat + // t = u + q + exec->run(cgs::make_step_2( + u->template get_const_local_device_view(), + v_hat->template get_const_local_device_view(), + q->template get_local_device_view(), + t->template get_local_device_view(), + alpha->get_device_view(), rho->get_const_device_view(), + gamma->get_const_device_view(), stop_status)); + + this->get_preconditioner()->apply(t, u_hat); + this->get_system_matrix()->apply(u_hat, t); + // r = r - alpha * t + // x = x + alpha * u_hat + exec->run(cgs::make_step_3( + t->template get_const_local_device_view(), + u_hat->template get_const_local_device_view(), + r->template get_local_device_view(), + converted_x->template get_local_device_view(), + alpha->get_const_device_view(), stop_status)); + + swap(prev_rho, rho); + } }, b, x); } template -template -void Cgs::apply_dense_impl(const VectorType* dense_b, - VectorType* dense_x) const -{ - using std::swap; - using LocalVector = matrix::MultiVector; - - constexpr uint8 RelativeStoppingId{1}; - - auto exec = this->get_executor(); - this->setup_workspace(); - - GKO_SOLVER_VECTOR(r, dense_b); - GKO_SOLVER_VECTOR(r_tld, dense_b); - GKO_SOLVER_VECTOR(p, dense_b); - GKO_SOLVER_VECTOR(q, dense_b); - GKO_SOLVER_VECTOR(u, dense_b); - GKO_SOLVER_VECTOR(u_hat, dense_b); - GKO_SOLVER_VECTOR(v_hat, dense_b); - GKO_SOLVER_VECTOR(t, dense_b); - - GKO_SOLVER_SCALAR(alpha, dense_b); - GKO_SOLVER_SCALAR(beta, dense_b); - GKO_SOLVER_SCALAR(gamma, dense_b); - GKO_SOLVER_SCALAR(prev_rho, dense_b); - GKO_SOLVER_SCALAR(rho, dense_b); - - GKO_SOLVER_ONE_MINUS_ONE(); - - bool one_changed{}; - GKO_SOLVER_STOP_REDUCTION_ARRAYS(); - - // r = dense_b - // r_tld = r - // rho = 0.0 - // prev_rho = alpha = beta = gamma = 1.0 - // p = q = u = u_hat = v_hat = t = 0 - exec->run(cgs::make_initialize( - gko::detail::get_local(dense_b)->get_const_device_view(), - gko::detail::get_local(r)->get_device_view(), - gko::detail::get_local(r_tld)->get_device_view(), - gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(q)->get_device_view(), - gko::detail::get_local(u)->get_device_view(), - gko::detail::get_local(u_hat)->get_device_view(), - gko::detail::get_local(v_hat)->get_device_view(), - gko::detail::get_local(t)->get_device_view(), alpha->get_device_view(), - beta->get_device_view(), gamma->get_device_view(), - prev_rho->get_device_view(), rho->get_device_view(), stop_status)); - - this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, r); - auto stop_criterion = this->get_stop_criterion_factory()->generate( - this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, r); - r_tld->copy_from(r); - - int iter = -1; - /* Memory movement summary: - * 28n * values + 2 * matrix/preconditioner storage - * 2x SpMV: 4n * values + 2 * storage - * 2x Preconditioner: 4n * values + 2 * storage - * 2x dot 4n - * 1x step 1 (fused axpys) 5n - * 1x step 2 (fused axpys) 4n - * 1x step 3 (axpys) 6n - * 1x norm2 residual n - */ - while (true) { - r->compute_conj_dot(r_tld, rho, reduction_tmp); - - ++iter; - bool all_stopped = - stop_criterion->update() - .num_iterations(iter) - .residual(r) - .implicit_sq_residual_norm(rho) - .solution(dense_x) - .check(RelativeStoppingId, true, &stop_status, &one_changed); - this->template log( - this, dense_b, dense_x, iter, r, nullptr, rho, &stop_status, - all_stopped); - if (all_stopped) { - break; - } - - // beta = rho / prev_rho - // u = r + beta * q - // p = u + beta * ( q + beta * p ) - exec->run(cgs::make_step_1( - gko::detail::get_local(r)->get_const_device_view(), - gko::detail::get_local(u)->get_device_view(), - gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(q)->get_const_device_view(), - beta->get_device_view(), rho->get_const_device_view(), - prev_rho->get_const_device_view(), stop_status)); - this->get_preconditioner()->apply(p, t); - this->get_system_matrix()->apply(t, v_hat); - r_tld->compute_conj_dot(v_hat, gamma, reduction_tmp); - // alpha = rho / gamma - // q = u - alpha * v_hat - // t = u + q - exec->run(cgs::make_step_2( - gko::detail::get_local(u)->get_const_device_view(), - gko::detail::get_local(v_hat)->get_const_device_view(), - gko::detail::get_local(q)->get_device_view(), - gko::detail::get_local(t)->get_device_view(), - alpha->get_device_view(), rho->get_const_device_view(), - gamma->get_const_device_view(), stop_status)); - - this->get_preconditioner()->apply(t, u_hat); - this->get_system_matrix()->apply(u_hat, t); - // r = r - alpha * t - // x = x + alpha * u_hat - exec->run(cgs::make_step_3( - gko::detail::get_local(t)->get_const_device_view(), - gko::detail::get_local(u_hat)->get_const_device_view(), - gko::detail::get_local(r)->get_device_view(), - gko::detail::get_local(dense_x)->get_device_view(), - alpha->get_const_device_view(), stop_status)); - - swap(prev_rho, rho); - } -} - - -template -void Cgs::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Cgs::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/core/solver/chebyshev.cpp b/core/solver/chebyshev.cpp index 49fd676c58d..f51b3701657 100644 --- a/core/solver/chebyshev.cpp +++ b/core/solver/chebyshev.cpp @@ -6,10 +6,10 @@ #include -#include #include #include +#include "core/base/dispatch_helper.hpp" #include "core/config/solver_config.hpp" #include "core/distributed/helpers.hpp" #include "core/solver/chebyshev_kernels.hpp" @@ -177,33 +177,17 @@ std::unique_ptr Chebyshev::conj_transpose() const template -void Chebyshev::apply_impl(const LinOp* b, LinOp* x) const +void Chebyshev::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { this->apply_with_initial_guess(b, x, this->get_default_initial_guess()); } template -void Chebyshev::apply_with_initial_guess_impl( - const LinOp* b, LinOp* x, initial_guess_mode guess) const -{ - if (!this->get_system_matrix()) { - return; - } - experimental::precision_dispatch_real_complex_distributed( - [this, guess](auto dense_b, auto dense_x) { - prepare_initial_guess(dense_b, dense_x, guess); - this->apply_dense_impl(dense_b, dense_x, guess); - }, - b, x); -} - - -template -template -void Chebyshev::apply_dense_impl(const VectorType* dense_b, - VectorType* dense_x, - initial_guess_mode guess) const +void Chebyshev::apply_with_initial_guess_prepared_impl( + const AbstractMultiVector* b, AbstractMultiVector* x, + initial_guess_mode guess) const { using Vector = matrix::MultiVector; using ws = workspace_traits; @@ -212,9 +196,9 @@ void Chebyshev::apply_dense_impl(const VectorType* dense_b, auto exec = this->get_executor(); this->setup_workspace(); - GKO_SOLVER_VECTOR(residual, dense_b); - GKO_SOLVER_VECTOR(inner_solution, dense_b); - GKO_SOLVER_VECTOR(update_solution, dense_b); + GKO_SOLVER_VECTOR(residual, b); + GKO_SOLVER_VECTOR(inner_solution, b); + GKO_SOLVER_VECTOR(update_solution, b); GKO_SOLVER_ONE_MINUS_ONE(); @@ -223,35 +207,35 @@ void Chebyshev::apply_dense_impl(const VectorType* dense_b, (foci_direction_ * alpha_host); auto& stop_status = this->template create_workspace_array( - ws::stop, dense_b->get_size()[1]); + ws::stop, b->get_size()[1]); exec->run(ir::make_initialize(stop_status)); if (guess != initial_guess_mode::zero) { - residual->copy_from(dense_b); - this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, residual); + residual->copy_from(b); + this->get_system_matrix()->apply(neg_one_op, x, one_op, residual); } - // zero input the residual is dense_b - const VectorType* residual_ptr = - guess == initial_guess_mode::zero ? dense_b : residual; + // zero input the residual is b + const AbstractMultiVector* residual_ptr = + guess == initial_guess_mode::zero ? b : residual; auto stop_criterion = this->get_stop_criterion_factory()->generate( this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, - residual_ptr); + std::shared_ptr( + b, [](const AbstractMultiVector*) {}), + x, residual_ptr); int iter = -1; while (true) { ++iter; - auto log_func = [this](auto solver, auto dense_b, auto dense_x, - auto iter, auto residual_ptr, - array& stop_status, - bool all_stopped) { - this->template log( - solver, dense_b, dense_x, iter, residual_ptr, nullptr, nullptr, - &stop_status, all_stopped); - }; - bool all_stopped = update_residual( - this, iter, dense_b, dense_x, residual, residual_ptr, - stop_criterion, stop_status, log_func); + auto log_func = + [this](auto solver, auto b, auto x, auto iter, auto residual_ptr, + array& stop_status, bool all_stopped) { + this->template log( + solver, b, x, iter, residual_ptr, nullptr, nullptr, + &stop_status, all_stopped); + }; + bool all_stopped = + update_residual(this, iter, b, x, residual, residual_ptr, + stop_criterion, stop_status, log_func); if (all_stopped) { break; } @@ -268,9 +252,10 @@ void Chebyshev::apply_dense_impl(const VectorType* dense_b, // update_solution = inner_solution exec->run(chebyshev::make_init_update( alpha_host, - gko::detail::get_local(inner_solution)->get_const_device_view(), - gko::detail::get_local(update_solution)->get_device_view(), - gko::detail::get_local(dense_x)->get_device_view())); + inner_solution + ->template get_const_local_device_view(), + update_solution->template get_local_device_view(), + x->template get_local_device_view())); continue; } // beta_host for iter == 1 is initialized in the beginning @@ -284,16 +269,36 @@ void Chebyshev::apply_dense_impl(const VectorType* dense_b, // x += alpha * p exec->run(chebyshev::make_update( alpha_host, beta_host, - gko::detail::get_local(inner_solution)->get_device_view(), - gko::detail::get_local(update_solution)->get_device_view(), - gko::detail::get_local(dense_x)->get_device_view())); + inner_solution->template get_local_device_view(), + update_solution->template get_local_device_view(), + x->template get_local_device_view())); } } template -void Chebyshev::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Chebyshev::apply_with_initial_guess_impl( + const AbstractMultiVector* b, AbstractMultiVector* x, + initial_guess_mode guess) const +{ + if (!this->get_system_matrix()) { + return; + } + + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + prepare_initial_guess(dense_b, dense_x, guess); + this->apply_with_initial_guess_prepared_impl(dense_b, dense_x, guess); +} + + +template +void Chebyshev::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { this->apply_with_initial_guess(alpha, b, beta, x, this->get_default_initial_guess()); @@ -301,22 +306,22 @@ void Chebyshev::apply_impl(const LinOp* alpha, const LinOp* b, template void Chebyshev::apply_with_initial_guess_impl( - const LinOp* alpha, const LinOp* b, const LinOp* beta, LinOp* x, + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x, initial_guess_mode guess) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this, guess](auto dense_alpha, auto dense_b, auto dense_beta, - auto dense_x) { - prepare_initial_guess(dense_b, dense_x, guess); - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get(), guess); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone.get()); - }, - alpha, b, beta, x); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + prepare_initial_guess(dense_b, dense_x, guess); + auto x_clone = dense_x->clone(); + this->apply_with_initial_guess_prepared_impl(dense_b, x_clone.get(), guess); + dense_x->scale(beta); + dense_x->add_scaled(alpha, x_clone); } diff --git a/core/solver/direct.cpp b/core/solver/direct.cpp index a01cf8672ce..fef73dff5fa 100644 --- a/core/solver/direct.cpp +++ b/core/solver/direct.cpp @@ -7,7 +7,6 @@ #include #include -#include #include #include @@ -180,47 +179,48 @@ Direct::Direct(const Factory* factory, template -void Direct::apply_impl(const LinOp* b, LinOp* x) const +void Direct::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix() || !this->lower_solver_ || !this->upper_solver_) { return; } - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - using Vector = matrix::MultiVector; - using ws = gko::solver::workspace_traits; - this->setup_workspace(); - auto intermediate = this->create_workspace_op_with_config_of( - ws::intermediate, dense_b); - lower_solver_->apply(dense_b, intermediate); - upper_solver_->apply(intermediate, dense_x); - }, - b, x); + using ws = gko::solver::workspace_traits; + this->setup_workspace(); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto intermediate = this->create_workspace_op_with_config_of( + ws::intermediate, converted_b.get()); + lower_solver_->apply(converted_b.get(), intermediate); + upper_solver_->apply(intermediate, converted_x.get()); } template -void Direct::apply_impl(const LinOp* alpha, - const LinOp* b, const LinOp* beta, - LinOp* x) const +void Direct::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix() || !this->lower_solver_ || !this->upper_solver_) { return; } - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - using Vector = matrix::MultiVector; - using ws = gko::solver::workspace_traits; - this->setup_workspace(); - auto intermediate = this->create_workspace_op_with_config_of( - ws::intermediate, dense_b); - lower_solver_->apply(dense_b, intermediate); - upper_solver_->apply(dense_alpha, intermediate, dense_beta, - dense_x); - }, - alpha, b, beta, x); + + using ws = gko::solver::workspace_traits; + this->setup_workspace(); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_alpha = + as>(alpha->as_precision(this)); + auto dense_beta = + as>(beta->as_precision(this)); + auto intermediate = this->create_workspace_op_with_config_of( + ws::intermediate, converted_b.get()); + lower_solver_->apply(converted_b.get(), intermediate); + upper_solver_->apply(dense_alpha.get(), intermediate, dense_beta.get(), + converted_x.get()); } diff --git a/core/solver/fcg.cpp b/core/solver/fcg.cpp index 801877f8f01..8ada06325db 100644 --- a/core/solver/fcg.cpp +++ b/core/solver/fcg.cpp @@ -10,9 +10,8 @@ #include #include #include -#include -#include +#include "core/base/dispatch_helper.hpp" #include "core/config/config_helper.hpp" #include "core/config/solver_config.hpp" #include "core/distributed/helpers.hpp" @@ -73,141 +72,130 @@ std::unique_ptr Fcg::conj_transpose() const template -void Fcg::apply_impl(const LinOp* b, LinOp* x) const +void Fcg::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); + precision_dispatch( + [this](auto converted_b, auto converted_x) { + using std::swap; + + constexpr uint8 RelativeStoppingId{1}; + + auto exec = this->get_executor(); + this->setup_workspace(); + + GKO_SOLVER_VECTOR(r, converted_b); + GKO_SOLVER_VECTOR(z, converted_b); + GKO_SOLVER_VECTOR(p, converted_b); + GKO_SOLVER_VECTOR(q, converted_b); + GKO_SOLVER_VECTOR(t, converted_b); + + GKO_SOLVER_SCALAR(beta, converted_b); + GKO_SOLVER_SCALAR(prev_rho, converted_b); + GKO_SOLVER_SCALAR(rho, converted_b); + GKO_SOLVER_SCALAR(rho_t, converted_b); + + GKO_SOLVER_ONE_MINUS_ONE(); + + bool one_changed{}; + GKO_SOLVER_STOP_REDUCTION_ARRAYS(converted_b->get_size()[1]); + + // r = converted_b + // t = r + // rho = 0.0 + // prev_rho = 1.0 + // rho_t = 1.0 + // z = p = q = 0 + exec->run(fcg::make_initialize( + converted_b->template get_const_local_device_view(), + r->template get_local_device_view(), + z->template get_local_device_view(), + p->template get_local_device_view(), + q->template get_local_device_view(), + t->template get_local_device_view(), + prev_rho->get_device_view(), rho->get_device_view(), + rho_t->get_device_view(), stop_status)); + + this->get_system_matrix()->apply(neg_one_op, converted_x, one_op, + r); + auto stop_criterion = this->get_stop_criterion_factory()->generate( + this->get_system_matrix(), + std::shared_ptr( + converted_b, [](const AbstractMultiVector*) {}), + converted_x, r); + + int iter = -1; + /* Memory movement summary: + * 21n * values + matrix/preconditioner storage + * 1x SpMV: 2n * values + storage + * 1x Preconditioner: 2n * values + storage + * 3x dot 6n + * 1x step 1 (axpy) 3n + * 1x step 2 (fused axpys) 7n + * 1x norm2 residual n + */ + while (true) { + this->get_preconditioner()->apply(r, z); + r->compute_conj_dot(z, rho, reduction_tmp); + t->compute_conj_dot(z, rho_t, reduction_tmp); + + ++iter; + bool all_stopped = stop_criterion->update() + .num_iterations(iter) + .residual(r) + .implicit_sq_residual_norm(rho) + .solution(converted_x) + .check(RelativeStoppingId, true, + &stop_status, &one_changed); + this->template log( + this, converted_b, converted_x, iter, r, nullptr, rho, + &stop_status, all_stopped); + if (all_stopped) { + break; + } + + // tmp = rho_t / prev_rho + // p = z + tmp * p + exec->run(fcg::make_step_1( + p->template get_local_device_view(), + z->template get_const_local_device_view(), + rho_t->get_const_device_view(), + prev_rho->get_const_device_view(), stop_status)); + this->get_system_matrix()->apply(p, q); + p->compute_conj_dot(q, beta, reduction_tmp); + // tmp = rho / beta + // [prev_r = r] in registers + // x = x + tmp * p + // r = r - tmp * q + // t = r - [prev_r] + exec->run(fcg::make_step_2( + converted_x->template get_local_device_view(), + r->template get_local_device_view(), + t->template get_local_device_view(), + p->template get_const_local_device_view(), + q->template get_const_local_device_view(), + beta->get_const_device_view(), rho->get_const_device_view(), + stop_status)); + swap(prev_rho, rho); + } }, b, x); } template -template -void Fcg::apply_dense_impl(const VectorType* dense_b, - VectorType* dense_x) const -{ - using std::swap; - using LocalVector = matrix::MultiVector; - - constexpr uint8 RelativeStoppingId{1}; - - auto exec = this->get_executor(); - this->setup_workspace(); - - GKO_SOLVER_VECTOR(r, dense_b); - GKO_SOLVER_VECTOR(z, dense_b); - GKO_SOLVER_VECTOR(p, dense_b); - GKO_SOLVER_VECTOR(q, dense_b); - GKO_SOLVER_VECTOR(t, dense_b); - - GKO_SOLVER_SCALAR(beta, dense_b); - GKO_SOLVER_SCALAR(prev_rho, dense_b); - GKO_SOLVER_SCALAR(rho, dense_b); - GKO_SOLVER_SCALAR(rho_t, dense_b); - - GKO_SOLVER_ONE_MINUS_ONE(); - - bool one_changed{}; - GKO_SOLVER_STOP_REDUCTION_ARRAYS(); - - // r = dense_b - // t = r - // rho = 0.0 - // prev_rho = 1.0 - // rho_t = 1.0 - // z = p = q = 0 - exec->run(fcg::make_initialize( - gko::detail::get_local(dense_b)->get_const_device_view(), - gko::detail::get_local(r)->get_device_view(), - gko::detail::get_local(z)->get_device_view(), - gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(q)->get_device_view(), - gko::detail::get_local(t)->get_device_view(), - prev_rho->get_device_view(), rho->get_device_view(), - rho_t->get_device_view(), stop_status)); - - this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, r); - auto stop_criterion = this->get_stop_criterion_factory()->generate( - this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, r); - - int iter = -1; - /* Memory movement summary: - * 21n * values + matrix/preconditioner storage - * 1x SpMV: 2n * values + storage - * 1x Preconditioner: 2n * values + storage - * 3x dot 6n - * 1x step 1 (axpy) 3n - * 1x step 2 (fused axpys) 7n - * 1x norm2 residual n - */ - while (true) { - this->get_preconditioner()->apply(r, z); - r->compute_conj_dot(z, rho, reduction_tmp); - t->compute_conj_dot(z, rho_t, reduction_tmp); - - ++iter; - bool all_stopped = - stop_criterion->update() - .num_iterations(iter) - .residual(r) - .implicit_sq_residual_norm(rho) - .solution(dense_x) - .check(RelativeStoppingId, true, &stop_status, &one_changed); - this->template log( - this, dense_b, dense_x, iter, r, nullptr, rho, &stop_status, - all_stopped); - if (all_stopped) { - break; - } - - // tmp = rho_t / prev_rho - // p = z + tmp * p - exec->run(fcg::make_step_1( - gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(z)->get_const_device_view(), - gko::detail::get_local(rho_t)->get_const_device_view(), - prev_rho->get_const_device_view(), stop_status)); - this->get_system_matrix()->apply(p, q); - p->compute_conj_dot(q, beta, reduction_tmp); - // tmp = rho / beta - // [prev_r = r] in registers - // x = x + tmp * p - // r = r - tmp * q - // t = r - [prev_r] - exec->run( - fcg::make_step_2(gko::detail::get_local(dense_x)->get_device_view(), - gko::detail::get_local(r)->get_device_view(), - gko::detail::get_local(t)->get_device_view(), - gko::detail::get_local(p)->get_const_device_view(), - gko::detail::get_local(q)->get_const_device_view(), - beta->get_const_device_view(), - rho->get_const_device_view(), stop_status)); - swap(prev_rho, rho); - } -} - - -template -void Fcg::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Fcg::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/core/solver/gcr.cpp b/core/solver/gcr.cpp index 15d7d9db8ca..03c9408f2e0 100644 --- a/core/solver/gcr.cpp +++ b/core/solver/gcr.cpp @@ -11,16 +11,16 @@ #include #include #include -#include -#include -#include #include +#include "core/base/dispatch_helper.hpp" #include "core/config/config_helper.hpp" #include "core/config/solver_config.hpp" #include "core/distributed/helpers.hpp" #include "core/solver/gcr_kernels.hpp" #include "core/solver/solver_boilerplate.hpp" + + namespace gko { namespace solver { namespace gcr { @@ -81,220 +81,227 @@ std::unique_ptr Gcr::conj_transpose() const template -void Gcr::apply_impl(const LinOp* b, LinOp* x) const +void Gcr::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); - }, - b, x); -} - - -template -template -void Gcr::apply_dense_impl(const VectorType* dense_b, - VectorType* dense_x) const -{ - using Vector = VectorType; - using LocalVector = matrix::MultiVector; - using NormVector = typename LocalVector::absolute_type; - using ws = workspace_traits; - - constexpr uint8 RelativeStoppingId{1}; - - auto exec = this->get_executor(); - this->setup_workspace(); - - const auto num_rows = this->get_size()[0]; - const auto num_rhs = dense_b->get_size()[1]; - const auto local_num_rows = - ::gko::detail::get_local(dense_b)->get_size()[0]; - const auto krylov_dim = this->get_krylov_dim(); - GKO_SOLVER_VECTOR(residual, dense_b); - GKO_SOLVER_VECTOR(precon_residual, dense_b); - GKO_SOLVER_VECTOR(A_precon_residual, dense_b); - auto krylov_bases_p = this->create_workspace_op_with_type_of( - ws::krylov_bases_p, dense_b, - dim<2>{num_rows * (krylov_dim + 1), num_rhs}, - dim<2>{local_num_rows * (krylov_dim + 1), num_rhs}); - auto mapped_krylov_bases_Ap = this->create_workspace_op_with_type_of( - ws::mapped_krylov_bases_Ap, dense_b, - dim<2>{num_rows * (krylov_dim + 1), num_rhs}, - dim<2>{local_num_rows * (krylov_dim + 1), num_rhs}); - auto tmp_rAp = this->template create_workspace_op( - ws::tmp_rAp, dim<2>{1, num_rhs}); - auto tmp_minus_beta = this->template create_workspace_op( - ws::tmp_minus_beta, dim<2>{1, num_rhs}); - auto residual_norm = this->template create_workspace_op( - ws::residual_norm, dim<2>{1, num_rhs}); - auto Ap_norms = this->template create_workspace_op( - ws::Ap_norms, dim<2>{krylov_dim + 1, num_rhs}); - auto& final_iter_nums = this->template create_workspace_array( - ws::final_iter_nums, num_rhs); - - // indicates if the status of a vector has changed - bool one_changed{}; - GKO_SOLVER_ONE_MINUS_ONE(); - GKO_SOLVER_STOP_REDUCTION_ARRAYS(); - - // Initialization - // residual = dense_b - // reset stop status - exec->run(gcr::make_initialize( - ::gko::detail::get_local(dense_b)->get_const_device_view(), - ::gko::detail::get_local(residual)->get_device_view(), - stop_status.get_data())); - // residual = residual - Ax - // Note: x is passed in with initial guess - this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, residual); - // apply preconditioner to residual - this->get_preconditioner()->apply(residual, precon_residual); - // A_precon_residual = A*precon_residual - this->get_system_matrix()->apply(precon_residual, A_precon_residual); - - // p(:, 1) = precon_residual(:, 1) - // Ap(:, 1) = A_precon_residual(:, 1) - // final_iter_nums = {0, ..., 0} - exec->run(gcr::make_restart( - ::gko::detail::get_local(precon_residual)->get_const_device_view(), - ::gko::detail::get_local(A_precon_residual)->get_const_device_view(), - ::gko::detail::get_local(krylov_bases_p)->get_device_view(), - ::gko::detail::get_local(mapped_krylov_bases_Ap)->get_device_view(), - final_iter_nums.get_data())); - - auto stop_criterion = this->get_stop_criterion_factory()->generate( - this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, - residual); - - int total_iter = -1; - size_type restart_iter = 0; - - /* Memory movement summary for average iteration with krylov_dim d: - * (4d+22+4/d)n+(d+1+1/d) * values + matrix/preconditioner storage - * 1x SpMV: 2n * values + storage - * 1x Preconditioner: 2n * values + storage - * 1x step 1 (scal, axpys) 6n - * 1x dot 2n - * MGS: (4d+10)n+(d+1) - * = sum k=0 to d-1 of (8k+8)n+(2k+2) /d + 6n - * 1x dots 2(k+1)n in iteration k (0-based) - * 2x axpys 6(k+1)n in iteration k (0-based) - * 1x scals 2(k+1) in iteration k (0-based) - * 1x norm2 n - * 1x sq_norm2 n - * 2x copy 4n - * Restart: (4/d)n+1/d (every dth iteration) - * (2+1)x copy 4n+1 - */ - while (true) { - ++total_iter; - // compute residual norm - residual->compute_norm2(residual_norm, reduction_tmp); - - // Should the iteration stop? - auto all_stopped = - stop_criterion->update() - .num_iterations(total_iter) - .residual(residual) - .residual_norm(residual_norm) - .solution(dense_x) - .check(RelativeStoppingId, true, &stop_status, &one_changed); - - // Log current iteration - this->template log( - this, dense_b, dense_x, total_iter, residual, residual_norm, - nullptr, &stop_status, all_stopped); - // Check stopping criterion - if (all_stopped) { - break; - } - - // If krylov_dim reached, restart with new initial guess - if (restart_iter == krylov_dim) { - // Restart - // p(:, 1) = precon_residual(:) - // Ap(:, 1) = A_precon_residual(:) + precision_dispatch( + [this](auto converted_b, auto converted_x) { + using LocalVector = matrix::MultiVector; + using NormVector = typename LocalVector::absolute_type; + using ws = workspace_traits; + + constexpr uint8 RelativeStoppingId{1}; + + auto exec = this->get_executor(); + this->setup_workspace(); + + const auto num_rows = this->get_size()[0]; + const auto num_rhs = converted_b->get_size()[1]; + const auto local_num_rows = + converted_b->template get_const_local_device_view() + .size[0]; + const auto krylov_dim = this->get_krylov_dim(); + GKO_SOLVER_VECTOR(residual, converted_b); + GKO_SOLVER_VECTOR(precon_residual, converted_b); + GKO_SOLVER_VECTOR(A_precon_residual, converted_b); + auto krylov_bases_p = this->create_workspace_op_with_type_of( + ws::krylov_bases_p, converted_b, + dim<2>{num_rows * (krylov_dim + 1), num_rhs}, + dim<2>{local_num_rows * (krylov_dim + 1), num_rhs}); + auto mapped_krylov_bases_Ap = + this->create_workspace_op_with_type_of( + ws::mapped_krylov_bases_Ap, converted_b, + dim<2>{num_rows * (krylov_dim + 1), num_rhs}, + dim<2>{local_num_rows * (krylov_dim + 1), num_rhs}); + auto tmp_rAp = this->template create_workspace_op( + ws::tmp_rAp, dim<2>{1, num_rhs}); + auto tmp_minus_beta = + this->template create_workspace_op( + ws::tmp_minus_beta, dim<2>{1, num_rhs}); + auto residual_norm = this->template create_workspace_op( + ws::residual_norm, dim<2>{1, num_rhs}); + auto Ap_norms = this->template create_workspace_op( + ws::Ap_norms, dim<2>{krylov_dim + 1, num_rhs}); + auto& final_iter_nums = + this->template create_workspace_array( + ws::final_iter_nums, num_rhs); + + // indicates if the status of a vector has changed + bool one_changed{}; + GKO_SOLVER_ONE_MINUS_ONE(); + GKO_SOLVER_STOP_REDUCTION_ARRAYS(converted_b->get_size()[1]); + + // Initialization + // residual = converted_b + // reset stop status + exec->run(gcr::make_initialize( + converted_b->template get_const_local_device_view(), + residual->template get_local_device_view(), + stop_status.get_data())); + // residual = residual - Ax + // Note: x is passed in with initial guess + this->get_system_matrix()->apply(neg_one_op, converted_x, one_op, + residual); + // apply preconditioner to residual + this->get_preconditioner()->apply(residual, precon_residual); + // A_precon_residual = A*precon_residual + this->get_system_matrix()->apply(precon_residual, + A_precon_residual); + + // p(:, 1) = precon_residual(:, 1) + // Ap(:, 1) = A_precon_residual(:, 1) // final_iter_nums = {0, ..., 0} exec->run(gcr::make_restart( - ::gko::detail::get_local(precon_residual) - ->get_const_device_view(), - ::gko::detail::get_local(A_precon_residual) - ->get_const_device_view(), - ::gko::detail::get_local(krylov_bases_p)->get_device_view(), - ::gko::detail::get_local(mapped_krylov_bases_Ap) - ->get_device_view(), + precon_residual + ->template get_const_local_device_view(), + A_precon_residual + ->template get_const_local_device_view(), + krylov_bases_p->template get_local_device_view(), + mapped_krylov_bases_Ap + ->template get_local_device_view(), final_iter_nums.get_data())); - restart_iter = 0; - } - - auto Ap = mapped_krylov_bases_Ap->create_submatrix( - local_span{local_num_rows * restart_iter, - local_num_rows * (restart_iter + 1)}, - local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); - auto p = krylov_bases_p->create_submatrix( - local_span{local_num_rows * restart_iter, - local_num_rows * (restart_iter + 1)}, - local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); - // compute r*Ap - residual->compute_conj_dot(Ap.get(), tmp_rAp, reduction_tmp); - // normalise - auto Ap_norm = Ap_norms->create_submatrix( - local_span{restart_iter, restart_iter + 1}, local_span{0, num_rhs}); - Ap->compute_squared_norm2(Ap_norm.get(), reduction_tmp); - - // alpha = r*Ap / Ap_norm - // x = x + alpha * p - // r = r - alpha * Ap - exec->run(gcr::make_step_1( - ::gko::detail::get_local(dense_x)->get_device_view(), - ::gko::detail::get_local(residual)->get_device_view(), - ::gko::detail::get_local(p.get())->get_const_device_view(), - ::gko::detail::get_local(Ap.get())->get_const_device_view(), - Ap_norm->get_const_device_view(), tmp_rAp->get_const_device_view(), - stop_status.get_const_data())); - - // apply preconditioner to residual - this->get_preconditioner()->apply(residual, precon_residual); - - // compute and save A*precon_residual - this->get_system_matrix()->apply(precon_residual, A_precon_residual); - - // modified Gram-Schmidt - auto next_Ap = mapped_krylov_bases_Ap->create_submatrix( - local_span{local_num_rows * (restart_iter + 1), - local_num_rows * (restart_iter + 2)}, - local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); - auto next_p = krylov_bases_p->create_submatrix( - local_span{local_num_rows * (restart_iter + 1), - local_num_rows * (restart_iter + 2)}, - local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); - // Ap = Ar - // p = r - next_Ap->copy_from(A_precon_residual); - next_p->copy_from(precon_residual); - for (size_type i = 0; i <= restart_iter; ++i) { - Ap = mapped_krylov_bases_Ap->create_submatrix( - local_span{local_num_rows * i, local_num_rows * (i + 1)}, - local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); - p = krylov_bases_p->create_submatrix( - local_span{local_num_rows * i, local_num_rows * (i + 1)}, - local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); - Ap_norm = Ap_norms->create_submatrix(local_span{i, i + 1}, - local_span{0, num_rhs}); - // tmp_minus_beta = -beta = Ar*Ap/Ap*Ap - A_precon_residual->compute_conj_dot(Ap.get(), tmp_minus_beta, - reduction_tmp); - tmp_minus_beta->inv_scale(Ap_norm.get()); - next_Ap->sub_scaled(tmp_minus_beta, Ap.get()); - next_p->sub_scaled(tmp_minus_beta, p.get()); - } - restart_iter++; - } + + auto stop_criterion = this->get_stop_criterion_factory()->generate( + this->get_system_matrix(), + std::shared_ptr( + converted_b, [](const AbstractMultiVector*) {}), + converted_x, residual); + + int total_iter = -1; + size_type restart_iter = 0; + + /* Memory movement summary for average iteration with krylov_dim d: + * (4d+22+4/d)n+(d+1+1/d) * values + matrix/preconditioner storage + * 1x SpMV: 2n * values + storage + * 1x Preconditioner: 2n * values + storage + * 1x step 1 (scal, axpys) 6n + * 1x dot 2n + * MGS: (4d+10)n+(d+1) + * = sum k=0 to d-1 of (8k+8)n+(2k+2) /d + 6n + * 1x dots 2(k+1)n in iteration k (0-based) + * 2x axpys 6(k+1)n in iteration k (0-based) + * 1x scals 2(k+1) in iteration k (0-based) + * 1x norm2 n + * 1x sq_norm2 n + * 2x copy 4n + * Restart: (4/d)n+1/d (every dth iteration) + * (2+1)x copy 4n+1 + */ + while (true) { + ++total_iter; + // compute residual norm + residual->compute_norm2(residual_norm, reduction_tmp); + + // Should the iteration stop? + auto all_stopped = stop_criterion->update() + .num_iterations(total_iter) + .residual(residual) + .residual_norm(residual_norm) + .solution(converted_x) + .check(RelativeStoppingId, true, + &stop_status, &one_changed); + + // Log current iteration + this->template log( + this, converted_b, converted_x, total_iter, residual, + residual_norm, nullptr, &stop_status, all_stopped); + // Check stopping criterion + if (all_stopped) { + break; + } + + // If krylov_dim reached, restart with new initial guess + if (restart_iter == krylov_dim) { + // Restart + // p(:, 1) = precon_residual(:) + // Ap(:, 1) = A_precon_residual(:) + // final_iter_nums = {0, ..., 0} + exec->run(gcr::make_restart( + precon_residual + ->template get_const_local_device_view(), + A_precon_residual + ->template get_const_local_device_view(), + krylov_bases_p + ->template get_local_device_view(), + mapped_krylov_bases_Ap + ->template get_local_device_view(), + final_iter_nums.get_data())); + restart_iter = 0; + } + + auto Ap = mapped_krylov_bases_Ap->create_subview( + local_span{local_num_rows * restart_iter, + local_num_rows * (restart_iter + 1)}, + local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); + auto p = krylov_bases_p->create_subview( + local_span{local_num_rows * restart_iter, + local_num_rows * (restart_iter + 1)}, + local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); + // compute r*Ap + residual->compute_conj_dot(Ap.get(), tmp_rAp, reduction_tmp); + // normalise + auto Ap_norm = Ap_norms->create_subview( + local_span{restart_iter, restart_iter + 1}, + local_span{0, num_rhs}); + Ap->compute_squared_norm2(Ap_norm.get(), reduction_tmp); + + // alpha = r*Ap / Ap_norm + // x = x + alpha * p + // r = r - alpha * Ap + exec->run(gcr::make_step_1( + converted_x->template get_local_device_view(), + residual->template get_local_device_view(), + p->template get_const_local_device_view(), + Ap->template get_const_local_device_view(), + Ap_norm->get_const_device_view(), + tmp_rAp->get_const_device_view(), + stop_status.get_const_data())); + + // apply preconditioner to residual + this->get_preconditioner()->apply(residual, precon_residual); + + // compute and save A*precon_residual + this->get_system_matrix()->apply(precon_residual, + A_precon_residual); + + // modified Gram-Schmidt + auto next_Ap = mapped_krylov_bases_Ap->create_subview( + local_span{local_num_rows * (restart_iter + 1), + local_num_rows * (restart_iter + 2)}, + local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); + auto next_p = krylov_bases_p->create_subview( + local_span{local_num_rows * (restart_iter + 1), + local_num_rows * (restart_iter + 2)}, + local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); + // Ap = Ar + // p = r + next_Ap->copy_from(A_precon_residual); + next_p->copy_from(precon_residual); + for (size_type i = 0; i <= restart_iter; ++i) { + Ap = mapped_krylov_bases_Ap->create_subview( + local_span{local_num_rows * i, + local_num_rows * (i + 1)}, + local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); + p = krylov_bases_p->create_subview( + local_span{local_num_rows * i, + local_num_rows * (i + 1)}, + local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); + Ap_norm = Ap_norms->create_subview(local_span{i, i + 1}, + local_span{0, num_rhs}); + // tmp_minus_beta = -beta = Ar*Ap/Ap*Ap + A_precon_residual->compute_conj_dot( + Ap.get(), tmp_minus_beta, reduction_tmp); + tmp_minus_beta->inv_scale(Ap_norm.get()); + next_Ap->sub_scaled(tmp_minus_beta, Ap.get()); + next_p->sub_scaled(tmp_minus_beta, p.get()); + } + restart_iter++; + } + }, + b, x); } @@ -320,20 +327,15 @@ Gcr::Gcr(const Factory* factory, template -void Gcr::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Gcr::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone.get()); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/core/solver/gmres.cpp b/core/solver/gmres.cpp index 4bfb4f1e615..bb770d3228d 100644 --- a/core/solver/gmres.cpp +++ b/core/solver/gmres.cpp @@ -7,12 +7,13 @@ #include #include +#include #include #include #include #include +#include #include -#include #include #include #include @@ -20,7 +21,6 @@ #include "core/config/config_helper.hpp" #include "core/config/solver_config.hpp" -#include "core/distributed/helpers.hpp" #include "core/mpi/mpi_op.hpp" #include "core/solver/common_gmres_kernels.hpp" #include "core/solver/gmres_kernels.hpp" @@ -125,25 +125,10 @@ std::unique_ptr Gmres::conj_transpose() const } -template -void Gmres::apply_impl(const LinOp* b, LinOp* x) const -{ - if (!this->get_system_matrix()) { - return; - } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); - }, - b, x); -} - - template struct help_compute_norm { - template static void compute_next_krylov_norm_into_hessenberg( - const VectorType* next_krylov, + const AbstractMultiVector* next_krylov, matrix::MultiVector* hessenberg_norm_entry, matrix::MultiVector>*, array& reduction_tmp) @@ -154,9 +139,10 @@ struct help_compute_norm { // Orthogonalization helper functions -template +template void orthogonalize_mgs(matrix::MultiVector* hessenberg_iter, - VectorType* krylov_bases, VectorType* next_krylov, + AbstractMultiVector* krylov_bases, + AbstractMultiVector* next_krylov, array& reduction_tmp, size_type restart_iter, size_type num_rows, size_type num_rhs, size_type local_num_rows) @@ -167,9 +153,9 @@ void orthogonalize_mgs(matrix::MultiVector* hessenberg_iter, // i) // next_krylov -= hessenberg(i, restart_iter) * krylov_bases(:, // i) - auto hessenberg_entry = - hessenberg_iter->create_submatrix(span{i, i + 1}, span{0, num_rhs}); - auto krylov_basis = krylov_bases->create_submatrix( + auto hessenberg_entry = hessenberg_iter->create_subview( + local_span{i, i + 1}, local_span{0, num_rhs}); + auto krylov_basis = krylov_bases->create_subview( local_span{local_num_rows * i, local_num_rows * (i + 1)}, local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); krylov_basis->compute_conj_dot(next_krylov, hessenberg_entry, @@ -181,32 +167,29 @@ void orthogonalize_mgs(matrix::MultiVector* hessenberg_iter, template void finish_reduce(matrix::MultiVector* hessenberg_iter, - matrix::MultiVector* next_krylov, - const size_type num_rhs, const size_type restart_iter) + AbstractMultiVector* next_krylov, const size_type num_rhs, + const size_type restart_iter) { - return; -} - - #if GINKGO_BUILD_MPI -template -void finish_reduce(matrix::MultiVector* hessenberg_iter, - experimental::distributed::Vector* next_krylov, - const size_type num_rhs, const size_type restart_iter) -{ + auto dist_vec = dynamic_cast*>( + next_krylov); + if (!dist_vec) { + return; + } + auto exec = hessenberg_iter->get_executor(); - const auto comm = next_krylov->get_communicator(); + const auto comm = dist_vec->get_communicator(); exec->synchronize(); // hessenberg_iter is the size of all non-zeros for this iteration, but we // are not setting the last values for each rhs here. Values that would be // below the diagonal in the "full" matrix are skipped, because they will // be used to hold the norm of next_krylov for each rhs. - auto hessenberg_reduce = hessenberg_iter->create_submatrix( - span{0, restart_iter + 1}, span{0, num_rhs}); + auto hessenberg_reduce = hessenberg_iter->create_subview( + local_span{0, restart_iter + 1}, local_span{0, num_rhs}); int message_size = static_cast((restart_iter + 1) * num_rhs); auto sum_op = gko::experimental::mpi::sum(); if (experimental::mpi::requires_host_buffer(exec, comm)) { - ::gko::detail::DenseCache host_reduction_buffer; + gko::detail::DenseCache host_reduction_buffer; host_reduction_buffer.init(exec->get_master(), hessenberg_reduce->get_size()); host_reduction_buffer->copy_from(hessenberg_reduce); @@ -217,34 +200,36 @@ void finish_reduce(matrix::MultiVector* hessenberg_iter, comm.all_reduce(exec, hessenberg_reduce->get_values(), message_size, sum_op.get_op()); } -} +#else + return; #endif +} -template +template void orthogonalize_cgs(matrix::MultiVector* hessenberg_iter, - VectorType* krylov_bases, VectorType* next_krylov, - size_type restart_iter, size_type num_rows, - size_type num_rhs, size_type local_num_rows) + AbstractMultiVector* krylov_bases, + AbstractMultiVector* next_krylov, size_type restart_iter, + size_type num_rows, size_type num_rhs, + size_type local_num_rows) { auto exec = hessenberg_iter->get_executor(); // hessenberg(0:restart_iter, restart_iter) = krylov_basis' * // next_krylov - auto krylov_basis_small = krylov_bases->create_submatrix( + auto krylov_basis_small = krylov_bases->create_subview( local_span{0, local_num_rows * (restart_iter + 1)}, local_span{0, num_rhs}, dim<2>{num_rows * (restart_iter + 1), num_rhs}); exec->run(gmres::make_multi_dot( - gko::detail::get_local(krylov_basis_small.get()) - ->get_const_device_view(), - gko::detail::get_local(next_krylov)->get_const_device_view(), + krylov_basis_small->template get_const_local_device_view(), + next_krylov->template get_const_local_device_view(), hessenberg_iter->get_device_view())); finish_reduce(hessenberg_iter, next_krylov, num_rhs, restart_iter); for (size_type i = 0; i <= restart_iter; i++) { // next_krylov -= hessenberg(i, restart_iter) * krylov_bases(:, // i) - auto hessenberg_entry = - hessenberg_iter->create_submatrix(span{i, i + 1}, span{0, num_rhs}); - auto krylov_col = krylov_bases->create_submatrix( + auto hessenberg_entry = hessenberg_iter->create_subview( + local_span{i, i + 1}, local_span{0, num_rhs}); + auto krylov_col = krylov_bases->create_subview( local_span{local_num_rows * i, local_num_rows * (i + 1)}, local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); next_krylov->sub_scaled(hessenberg_entry, krylov_col); @@ -252,9 +237,10 @@ void orthogonalize_cgs(matrix::MultiVector* hessenberg_iter, } -template +template void orthogonalize_cgs2(matrix::MultiVector* hessenberg_iter, - VectorType* krylov_bases, VectorType* next_krylov, + AbstractMultiVector* krylov_bases, + AbstractMultiVector* next_krylov, matrix::MultiVector* hessenberg_aux, const matrix::MultiVector* one_op, size_type restart_iter, size_type num_rows, @@ -263,32 +249,30 @@ void orthogonalize_cgs2(matrix::MultiVector* hessenberg_iter, auto exec = hessenberg_iter->get_executor(); // hessenberg(0:restart_iter, restart_iter) = krylov_bases' * // next_krylov - auto krylov_basis_small = krylov_bases->create_submatrix( + auto krylov_basis_small = krylov_bases->create_subview( local_span{0, local_num_rows * (restart_iter + 1)}, local_span{0, num_rhs}, dim<2>{num_rows * (restart_iter + 1), num_rhs}); exec->run(gmres::make_multi_dot( - gko::detail::get_local(krylov_basis_small.get()) - ->get_const_device_view(), - gko::detail::get_local(next_krylov)->get_const_device_view(), + krylov_basis_small->template get_const_local_device_view(), + next_krylov->template get_const_local_device_view(), hessenberg_iter->get_device_view())); finish_reduce(hessenberg_iter, next_krylov, num_rhs, restart_iter); for (size_type i = 0; i <= restart_iter; i++) { // next_krylov -= hessenberg(i, restart_iter) * krylov_bases(:, // i) - auto hessenberg_entry = - hessenberg_iter->create_submatrix(span{i, i + 1}, span{0, num_rhs}); - auto krylov_col = krylov_bases->create_submatrix( + auto hessenberg_entry = hessenberg_iter->create_subview( + local_span{i, i + 1}, local_span{0, num_rhs}); + auto krylov_col = krylov_bases->create_subview( local_span{local_num_rows * i, local_num_rows * (i + 1)}, local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); next_krylov->sub_scaled(hessenberg_entry, krylov_col); } // Re-orthogonalize - auto hessenberg_aux_iter = hessenberg_aux->create_submatrix( - span{0, restart_iter + 2}, span{0, num_rhs}); + auto hessenberg_aux_iter = hessenberg_aux->create_subview( + local_span{0, restart_iter + 2}, local_span{0, num_rhs}); exec->run(gmres::make_multi_dot( - gko::detail::get_local(krylov_basis_small.get()) - ->get_const_device_view(), - gko::detail::get_local(next_krylov)->get_const_device_view(), + krylov_basis_small->template get_const_local_device_view(), + next_krylov->template get_const_local_device_view(), hessenberg_aux_iter->get_device_view())); finish_reduce(hessenberg_aux_iter.get(), next_krylov, num_rhs, restart_iter); @@ -296,9 +280,9 @@ void orthogonalize_cgs2(matrix::MultiVector* hessenberg_iter, for (size_type i = 0; i <= restart_iter; i++) { // next_krylov -= hessenberg(i, restart_iter) * krylov_bases(:, // i) - auto hessenberg_entry = - hessenberg_aux->create_submatrix(span{i, i + 1}, span{0, num_rhs}); - auto krylov_col = krylov_bases->create_submatrix( + auto hessenberg_entry = hessenberg_aux->create_subview( + local_span{i, i + 1}, local_span{0, num_rhs}); + auto krylov_col = krylov_bases->create_subview( local_span{local_num_rows * i, local_num_rows * (i + 1)}, local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); next_krylov->sub_scaled(hessenberg_entry, krylov_col); @@ -311,9 +295,8 @@ void orthogonalize_cgs2(matrix::MultiVector* hessenberg_iter, template struct help_compute_norm::value>> { - template static void compute_next_krylov_norm_into_hessenberg( - const VectorType* next_krylov, + const AbstractMultiVector* next_krylov, matrix::MultiVector* hessenberg_norm_entry, matrix::MultiVector>* next_krylov_norm_tmp, array& reduction_tmp) @@ -325,12 +308,19 @@ struct help_compute_norm -template -void Gmres::apply_dense_impl(const VectorType* dense_b, - VectorType* dense_x) const +void Gmres::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - using Vector = VectorType; - using LocalVector = matrix::MultiVector; + if (!this->get_system_matrix()) { + return; + } + + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + + using LocalVector = matrix::MultiVector; using NormVector = typename LocalVector::absolute_type; using ws = workspace_traits; @@ -341,7 +331,7 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, const auto is_flexible = this->get_parameters().flexible; const auto num_rows = this->get_size()[0]; const auto local_num_rows = - ::gko::detail::get_local(dense_b)->get_size()[0]; + dense_b->template get_const_local_device_view().size[0]; const auto num_rhs = dense_b->get_size()[1]; const auto krylov_dim = this->get_krylov_dim(); GKO_SOLVER_VECTOR(residual, dense_b); @@ -350,7 +340,7 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, auto krylov_bases = this->create_workspace_op_with_type_of( ws::krylov_bases, dense_b, dim<2>{num_rows * (krylov_dim + 1), num_rhs}, dim<2>{local_num_rows * (krylov_dim + 1), num_rhs}); - VectorType* preconditioned_krylov_bases = nullptr; + AbstractMultiVector* preconditioned_krylov_bases = nullptr; if (is_flexible) { preconditioned_krylov_bases = this->create_workspace_op_with_type_of( ws::preconditioned_krylov_bases, dense_b, @@ -400,7 +390,7 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, GKO_SOLVER_ONE_MINUS_ONE(); bool one_changed{}; - GKO_SOLVER_STOP_REDUCTION_ARRAYS(); + GKO_SOLVER_STOP_REDUCTION_ARRAYS(converted_b->get_size()[1]); auto& final_iter_nums = this->template create_workspace_array( ws::final_iter_nums, num_rhs); @@ -409,8 +399,8 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, // givens_sin = givens_cos = 0 // reset stop status exec->run(gmres::make_initialize( - gko::detail::get_local(dense_b)->get_const_device_view(), - gko::detail::get_local(residual)->get_device_view(), + dense_b->template get_const_local_device_view(), + residual->template get_local_device_view(), givens_sin->get_device_view(), givens_cos->get_device_view(), stop_status.get_data())); // residual = residual - Ax @@ -422,16 +412,17 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, // krylov_bases(:, 1) = residual / residual_norm // final_iter_nums = {0, ..., 0} exec->run(gmres::make_restart( - gko::detail::get_local(residual)->get_const_device_view(), + residual->template get_const_local_device_view(), residual_norm->get_const_device_view(), residual_norm_collection->get_device_view(), - gko::detail::get_local(krylov_bases)->get_device_view(), + krylov_bases->template get_local_device_view(), final_iter_nums.get_data())); auto stop_criterion = this->get_stop_criterion_factory()->generate( this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, - residual); + std::shared_ptr( + dense_b, [](const AbstractMultiVector*) {}), + dense_x, residual); int total_iter = -1; size_type restart_iter = 0; @@ -481,10 +472,10 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, stop_status.get_const_data())); // before_preconditioner = krylov_bases * y exec->run(gmres::make_multi_axpy( - gko::detail::get_local(krylov_bases)->get_const_device_view(), + krylov_bases->template get_const_local_device_view(), y->get_const_device_view(), - gko::detail::get_local(before_preconditioner) - ->get_device_view(), + before_preconditioner + ->template get_local_device_view(), final_iter_nums.get_const_data(), stop_status.get_data())); // x = x + get_preconditioner() * before_preconditioner @@ -502,30 +493,29 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, // krylov_bases(:, 1) = residual / residual_norm // final_iter_nums = {0, ..., 0} exec->run(gmres::make_restart( - gko::detail::get_local(residual)->get_const_device_view(), + residual->template get_const_local_device_view(), residual_norm->get_const_device_view(), residual_norm_collection->get_device_view(), - gko::detail::get_local(krylov_bases)->get_device_view(), + krylov_bases->template get_local_device_view(), final_iter_nums.get_data())); restart_iter = 0; } - auto this_krylov = krylov_bases->create_submatrix( + auto this_krylov = krylov_bases->create_subview( local_span{local_num_rows * restart_iter, local_num_rows * (restart_iter + 1)}, local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); - auto next_krylov = krylov_bases->create_submatrix( + auto next_krylov = krylov_bases->create_subview( local_span{local_num_rows * (restart_iter + 1), local_num_rows * (restart_iter + 2)}, local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); - std::unique_ptr preconditioned_krylov; + std::unique_ptr preconditioned_krylov; auto preconditioned_krylov_vector = preconditioned_vector; if (is_flexible) { - preconditioned_krylov = - preconditioned_krylov_bases->create_submatrix( - local_span{local_num_rows * restart_iter, - local_num_rows * (restart_iter + 1)}, - local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); + preconditioned_krylov = preconditioned_krylov_bases->create_subview( + local_span{local_num_rows * restart_iter, + local_num_rows * (restart_iter + 1)}, + local_span{0, num_rhs}, dim<2>{num_rows, num_rhs}); preconditioned_krylov_vector = preconditioned_krylov.get(); } // preconditioned_krylov_vector = get_preconditioner() * this_krylov @@ -565,8 +555,9 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, // hessenberg(restart_iter+1, restart_iter) = norm(next_krylov) // (stored in hessenberg(restart_iter, (restart_iter + 1) * num_rhs)) // next_krylov /= hessenberg(restart_iter+1, restart_iter) - auto hessenberg_norm_entry = hessenberg_iter->create_submatrix( - span{restart_iter + 1, restart_iter + 2}, span{0, num_rhs}); + auto hessenberg_norm_entry = hessenberg_iter->create_subview( + local_span{restart_iter + 1, restart_iter + 2}, + local_span{0, num_rhs}); help_compute_norm::compute_next_krylov_norm_into_hessenberg( next_krylov.get(), hessenberg_norm_entry.get(), next_krylov_norm_tmp, reduction_tmp); @@ -605,8 +596,8 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, restart_iter++; } - auto hessenberg_small = hessenberg->create_submatrix( - span{0, restart_iter}, span{0, num_rhs * restart_iter}); + auto hessenberg_small = hessenberg->create_subview( + local_span{0, restart_iter}, local_span{0, num_rhs * restart_iter}); // Solve upper triangular. // y = hessenberg \ residual_norm_collection @@ -616,28 +607,28 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, final_iter_nums.get_const_data(), stop_status.get_const_data())); if (is_flexible) { auto preconditioned_krylov_bases_small = - preconditioned_krylov_bases->create_submatrix( + preconditioned_krylov_bases->create_subview( local_span{0, local_num_rows * (restart_iter + 1)}, local_span{0, num_rhs}, dim<2>{num_rows * (restart_iter + 1), num_rhs}); // after_preconditioner = preconditioned_krylov_bases * y exec->run(gmres::make_multi_axpy( - gko::detail::get_local(preconditioned_krylov_bases_small.get()) - ->get_const_device_view(), + preconditioned_krylov_bases_small + ->template get_const_local_device_view(), y->get_const_device_view(), - gko::detail::get_local(after_preconditioner)->get_device_view(), + after_preconditioner->template get_local_device_view(), final_iter_nums.get_const_data(), stop_status.get_data())); } else { - auto krylov_bases_small = krylov_bases->create_submatrix( + auto krylov_bases_small = krylov_bases->create_subview( local_span{0, local_num_rows * (restart_iter + 1)}, local_span{0, num_rhs}, dim<2>{num_rows * (restart_iter + 1), num_rhs}); // before_preconditioner = krylov_bases * y exec->run(gmres::make_multi_axpy( - gko::detail::get_local(krylov_bases_small.get()) - ->get_const_device_view(), + krylov_bases_small + ->template get_const_local_device_view(), y->get_const_device_view(), - gko::detail::get_local(before_preconditioner)->get_device_view(), + before_preconditioner->template get_local_device_view(), final_iter_nums.get_const_data(), stop_status.get_data())); // after_preconditioner = get_preconditioner() * before_preconditioner @@ -650,20 +641,15 @@ void Gmres::apply_dense_impl(const VectorType* dense_b, template -void Gmres::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Gmres::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/core/solver/idr.cpp b/core/solver/idr.cpp index 51e959badf2..d3797c12506 100644 --- a/core/solver/idr.cpp +++ b/core/solver/idr.cpp @@ -10,16 +10,25 @@ #include #include #include -#include #include #include "core/config/config_helper.hpp" #include "core/config/solver_config.hpp" #include "core/distributed/helpers.hpp" +#include "core/matrix/dense_kernels.hpp" #include "core/solver/idr_kernels.hpp" #include "core/solver/solver_boilerplate.hpp" namespace gko { +namespace matrix { +namespace dense { + + +GKO_REGISTER_OPERATION(simple_apply, dense::simple_apply); + + +} // namespace dense +} // namespace matrix namespace solver { namespace idr { namespace { @@ -102,7 +111,6 @@ void Idr::iterate(const VectorType* dense_b, { using std::swap; using SubspaceType = typename VectorType::value_type; - using Vector = matrix::MultiVector; using AbsType = remove_complex; using ws = workspace_traits; @@ -122,17 +130,17 @@ void Idr::iterate(const VectorType* dense_b, GKO_SOLVER_VECTOR(t, dense_b); GKO_SOLVER_VECTOR(helper, dense_b); - auto m = this->template create_workspace_op( + auto m = this->template create_workspace_op( ws::m, gko::dim<2>{subspace_dim, subspace_dim * nrhs}); - auto g = this->template create_workspace_op( + auto g = this->template create_workspace_op( ws::g, gko::dim<2>{problem_size, subspace_dim * nrhs}); - auto u = this->template create_workspace_op( + auto u = this->template create_workspace_op( ws::u, gko::dim<2>{problem_size, subspace_dim * nrhs}); - auto f = this->template create_workspace_op( + auto f = this->template create_workspace_op( ws::f, gko::dim<2>{subspace_dim, nrhs}); - auto c = this->template create_workspace_op( + auto c = this->template create_workspace_op( ws::c, gko::dim<2>{subspace_dim, nrhs}); auto omega = @@ -148,7 +156,7 @@ void Idr::iterate(const VectorType* dense_b, // Stored in column major order and complex conjugated. So, if the // matrix containing the subspace vectors in row major order is called P, // subspace_vectors actually contains P^H. - auto subspace_vectors = this->template create_workspace_op( + auto subspace_vectors = this->template create_workspace_op( ws::subspace, gko::dim<2>(subspace_dim, problem_size)); GKO_SOLVER_ONE_MINUS_ONE(); @@ -158,7 +166,7 @@ void Idr::iterate(const VectorType* dense_b, subspace_neg_one_op->fill(-one()); bool one_changed{}; - GKO_SOLVER_STOP_REDUCTION_ARRAYS(); + GKO_SOLVER_STOP_REDUCTION_ARRAYS(dense_b->get_size()[1]); // Initialization // m = identity @@ -168,10 +176,9 @@ void Idr::iterate(const VectorType* dense_b, std::default_random_engine(15)); subspace_vectors->read(subspace_vectors_data); } - exec->run(idr::make_initialize( - nrhs, gko::detail::get_local(m)->get_device_view(), - gko::detail::get_local(subspace_vectors)->get_device_view(), - is_deterministic, stop_status)); + exec->run(idr::make_initialize(nrhs, m->get_device_view(), + subspace_vectors->get_device_view(), + is_deterministic, stop_status)); // omega = 1 omega->fill(one()); @@ -187,8 +194,8 @@ void Idr::iterate(const VectorType* dense_b, auto stop_criterion = this->get_stop_criterion_factory()->generate( this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, - residual); + std::shared_ptr(dense_b, [](const VectorType*) {}), + dense_x, residual); int total_iter = -1; @@ -228,30 +235,28 @@ void Idr::iterate(const VectorType* dense_b, } // f = P^H * residual - subspace_vectors->apply(residual, f); + exec->run(gko::matrix::dense::make_simple_apply( + subspace_vectors->get_const_device_view(), + residual->get_const_device_view(), f->get_device_view())); for (size_type k = 0; k < subspace_dim; k++) { // c = M \ f = (c_1, ..., c_s)^T // v = residual - sum i=[k,s) of (c_i * g_i) exec->run(idr::make_step_1( - nrhs, k, gko::detail::get_local(m)->get_const_device_view(), - gko::detail::get_local(f)->get_const_device_view(), - gko::detail::get_local(residual)->get_const_device_view(), - gko::detail::get_local(g)->get_const_device_view(), - gko::detail::get_local(c)->get_device_view(), - gko::detail::get_local(v)->get_device_view(), stop_status)); + nrhs, k, m->get_const_device_view(), f->get_const_device_view(), + residual->get_const_device_view(), g->get_const_device_view(), + c->get_device_view(), v->get_device_view(), stop_status)); this->get_preconditioner()->apply(v, helper); // u_k = omega * precond_vector + sum i=[k,s) of (c_i * u_i) - exec->run(idr::make_step_2( - nrhs, k, gko::detail::get_local(omega)->get_const_device_view(), - gko::detail::get_local(helper)->get_const_device_view(), - gko::detail::get_local(c)->get_const_device_view(), - gko::detail::get_local(u)->get_device_view(), stop_status)); + exec->run(idr::make_step_2(nrhs, k, omega->get_const_device_view(), + helper->get_const_device_view(), + c->get_const_device_view(), + u->get_device_view(), stop_status)); - auto u_k = u->create_submatrix(span{0, problem_size}, - span{k * nrhs, (k + 1) * nrhs}); + auto u_k = u->create_subview(local_span{0, problem_size}, + local_span{k * nrhs, (k + 1) * nrhs}); // g_k = Au_k this->get_system_matrix()->apply(u_k, helper); @@ -270,17 +275,11 @@ void Idr::iterate(const VectorType* dense_b, // dense_x += beta * u_k // f = (0,...,0,f_k+1 - beta * m_k+1,k,...,f_s-1 - beta * m_s-1,k) exec->run(idr::make_step_3( - nrhs, k, - gko::detail::get_local(subspace_vectors) - ->get_const_device_view(), - gko::detail::get_local(g)->get_device_view(), - gko::detail::get_local(helper)->get_device_view(), - gko::detail::get_local(u)->get_device_view(), - gko::detail::get_local(m)->get_device_view(), - gko::detail::get_local(f)->get_device_view(), - gko::detail::get_local(alpha)->get_device_view(), - gko::detail::get_local(residual)->get_device_view(), - gko::detail::get_local(dense_x)->get_device_view(), + nrhs, k, subspace_vectors->get_const_device_view(), + g->get_device_view(), helper->get_device_view(), + u->get_device_view(), m->get_device_view(), + f->get_device_view(), alpha->get_device_view(), + residual->get_device_view(), dense_x->get_device_view(), stop_status)); } @@ -300,10 +299,10 @@ void Idr::iterate(const VectorType* dense_b, // end if // residual -= omega * t // dense_x += omega * v - exec->run(idr::make_compute_omega( - nrhs, kappa, gko::detail::get_local(tht)->get_const_device_view(), - gko::detail::get_local(residual_norm)->get_const_device_view(), - gko::detail::get_local(omega)->get_device_view(), stop_status)); + exec->run( + idr::make_compute_omega(nrhs, kappa, tht->get_const_device_view(), + residual_norm->get_const_device_view(), + omega->get_device_view(), stop_status)); t->scale(subspace_neg_one_op); residual->add_scaled(omega, t); @@ -330,46 +329,41 @@ Idr::Idr(const Factory* factory, template -void Idr::apply_impl(const LinOp* b, LinOp* x) const +void Idr::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - // If ValueType is complex, the subspace matrix P will be complex - // anyway. - if (!is_complex() && this->get_complex_subspace()) { - auto complex_b = dense_b->make_complex(); - auto complex_x = dense_x->make_complex(); - this->iterate(complex_b.get(), complex_x.get()); - complex_x->get_real( - dynamic_cast< - matrix::MultiVector>*>( - dense_x)); - } else { - this->iterate(dense_b, dense_x); - } - }, - b, x); + auto converted_b = + as>(b->as_precision(this)); + if (!is_complex() && this->get_complex_subspace()) { + auto converted_x = as>>( + x->as_precision(this)); + auto complex_b = converted_b->make_complex(); + auto complex_x = converted_x->make_complex(); + this->iterate( + as>>(complex_b.get()), + as>>(complex_x.get())); + complex_x->get_real(converted_x.get()); + } else { + auto converted_x = + as>(x->as_precision(this)); + this->iterate(converted_b.get(), converted_x.get()); + } } template -void Idr::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Idr::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/core/solver/ir.cpp b/core/solver/ir.cpp index 6a800837d4c..bca2579da73 100644 --- a/core/solver/ir.cpp +++ b/core/solver/ir.cpp @@ -6,13 +6,12 @@ #include -#include #include #include #include +#include "core/base/dispatch_helper.hpp" #include "core/config/config_helper.hpp" -#include "core/distributed/helpers.hpp" #include "core/solver/ir_kernels.hpp" #include "core/solver/solver_base.hpp" #include "core/solver/solver_boilerplate.hpp" @@ -197,7 +196,8 @@ std::unique_ptr Ir::conj_transpose() const template -void Ir::apply_impl(const LinOp* b, LinOp* x) const +void Ir::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { this->apply_with_initial_guess_impl(b, x, this->get_default_initial_guess()); @@ -205,26 +205,9 @@ void Ir::apply_impl(const LinOp* b, LinOp* x) const template -void Ir::apply_with_initial_guess_impl( - const LinOp* b, LinOp* x, initial_guess_mode guess) const -{ - if (!this->get_system_matrix()) { - return; - } - experimental::precision_dispatch_real_complex_distributed( - [this, guess](auto dense_b, auto dense_x) { - prepare_initial_guess(dense_b, dense_x, guess); - this->apply_dense_impl(dense_b, dense_x, guess); - }, - b, x); -} - - -template -template -void Ir::apply_dense_impl(const VectorType* dense_b, - VectorType* dense_x, - initial_guess_mode guess) const +void Ir::apply_with_initial_guess_prepared_impl( + const AbstractMultiVector* dense_b, AbstractMultiVector* dense_x, + initial_guess_mode guess) const { using Vector = matrix::MultiVector; using ws = workspace_traits; @@ -245,13 +228,14 @@ void Ir::apply_dense_impl(const VectorType* dense_b, this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, residual); } // zero input the residual is dense_b - const VectorType* residual_ptr = + const AbstractMultiVector* residual_ptr = guess == initial_guess_mode::zero ? dense_b : residual; auto stop_criterion = this->get_stop_criterion_factory()->generate( this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, - residual_ptr); + std::shared_ptr( + dense_b, [](const AbstractMultiVector*) {}), + dense_x, residual_ptr); int iter = -1; while (true) { @@ -290,8 +274,26 @@ void Ir::apply_dense_impl(const VectorType* dense_b, template -void Ir::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Ir::apply_with_initial_guess_impl( + const AbstractMultiVector* b, AbstractMultiVector* x, + initial_guess_mode guess) const +{ + if (!this->get_system_matrix()) { + return; + } + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + prepare_initial_guess(dense_b, dense_x, guess); + this->apply_with_initial_guess_prepared_impl(dense_b, dense_x, guess); +} + +template +void Ir::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { this->apply_with_initial_guess_impl(alpha, b, beta, x, this->get_default_initial_guess()); @@ -299,22 +301,22 @@ void Ir::apply_impl(const LinOp* alpha, const LinOp* b, template void Ir::apply_with_initial_guess_impl( - const LinOp* alpha, const LinOp* b, const LinOp* beta, LinOp* x, + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x, initial_guess_mode guess) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this, guess](auto dense_alpha, auto dense_b, auto dense_beta, - auto dense_x) { - prepare_initial_guess(dense_b, dense_x, guess); - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get(), guess); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + auto converted_b = b->as_precision(this); + auto converted_x = x->as_precision(this); + auto dense_b = converted_b.get(); + auto dense_x = converted_x.get(); + prepare_initial_guess(dense_b, dense_x, guess); + auto x_clone = dense_x->clone(); + this->apply_with_initial_guess_prepared_impl(dense_b, x_clone.get(), guess); + dense_x->scale(beta); + dense_x->add_scaled(alpha, x_clone); } diff --git a/core/solver/lower_trs.cpp b/core/solver/lower_trs.cpp index bd83c539992..2dc49eab0e2 100644 --- a/core/solver/lower_trs.cpp +++ b/core/solver/lower_trs.cpp @@ -9,13 +9,13 @@ #include #include #include -#include #include #include #include #include #include +#include "core/base/dispatch_helper.hpp" #include "core/config/config_helper.hpp" #include "core/config/trisolver_config.hpp" #include "core/solver/lower_trs_kernels.hpp" @@ -161,13 +161,14 @@ static bool needs_transpose(std::shared_ptr exec) template -void LowerTrs::apply_impl(const LinOp* b, LinOp* x) const +void LowerTrs::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { + apply_precision_dispatch( + [this](auto view_b, auto view_x) { using Vector = matrix::MultiVector; using ws = workspace_traits; const auto exec = this->get_executor(); @@ -184,9 +185,9 @@ void LowerTrs::apply_impl(const LinOp* b, LinOp* x) const using optional_view = std::optional>; if (needs_transpose(exec)) { trans_b = this->template create_workspace_op( - ws::transposed_b, gko::transpose(dense_b->get_size())); + ws::transposed_b, gko::transpose(view_b.size)); trans_x = this->template create_workspace_op( - ws::transposed_x, gko::transpose(dense_x->get_size())); + ws::transposed_x, gko::transpose(view_x.size)); } exec->run(lower_trs::make_solve( this->get_system_matrix().get(), this->solve_struct_.get(), @@ -195,29 +196,21 @@ void LowerTrs::apply_impl(const LinOp* b, LinOp* x) const : optional_view{}, trans_x ? optional_view{trans_x->get_device_view()} : optional_view{}, - dense_b->get_const_device_view(), dense_x->get_device_view())); + view_b, view_x)); }, b, x); } template -void LowerTrs::apply_impl(const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - LinOp* x) const +void LowerTrs::apply_impl( + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/core/solver/minres.cpp b/core/solver/minres.cpp index eb3e8adca6e..09322149d90 100644 --- a/core/solver/minres.cpp +++ b/core/solver/minres.cpp @@ -10,12 +10,11 @@ #include #include #include -#include #include +#include "core/base/dispatch_helper.hpp" #include "core/config/config_helper.hpp" #include "core/config/solver_config.hpp" -#include "core/distributed/helpers.hpp" #include "core/solver/minres_kernels.hpp" #include "core/solver/solver_boilerplate.hpp" @@ -67,15 +66,210 @@ bool Minres::apply_uses_initial_guess() const } +/** + * This Minres implementation is based on Anne Grennbaum's 'Iterative Methods + * for Solving Linear Systems' (DOI: 10.1137/1.9781611970937) Ch. 2 and Ch. 8. + * Most variable names are taken from that reference, with the exception that + * the vector `w` and `w_tilde` from the reference are called `z` and `z_tilde`. + * The variable declaration have a comment to specify the name used in the + * reference. By reusing already allocated memory the number of necessary + * vectors is reduced to seven temporary vectors. The operations are grouped + * into point-wise scalar and vector updates, operator applications and + * (possibly) global reductions. With some reordering, as many point-wise + * updates are grouped together into a scalar and vector step respectively to + * reduce the number of kernel launches. The algorithm uses a recursion to + * compute an approximate residual norm. The residual is neither computed + * exactly, nor approximately, since that would require additional operations. + */ template -void Minres::apply_impl(const LinOp* b, LinOp* x) const +void Minres::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); + + precision_dispatch( + [this](auto converted_b, auto converted_x) { + using std::swap; + constexpr uint8 RelativeStoppingId{1}; + + auto exec = this->get_executor(); + this->setup_workspace(); + + GKO_SOLVER_VECTOR(r, converted_b); + GKO_SOLVER_VECTOR(z, converted_b); // z = w_k+1 + GKO_SOLVER_VECTOR(p, converted_b); // p = p_k-1 + GKO_SOLVER_VECTOR(q, converted_b); // q = q_k+1 + GKO_SOLVER_VECTOR(v, converted_b); // v = v_k + + GKO_SOLVER_VECTOR(z_tilde, converted_b); // z_tilde = w_tilde_k+1 + GKO_SOLVER_VECTOR(p_prev, converted_b); // p_prev = p_k-2 + GKO_SOLVER_VECTOR(q_prev, converted_b); // q_prev = q_k + + GKO_SOLVER_SCALAR(alpha, converted_b); // alpha = T(k, k) + GKO_SOLVER_SCALAR(beta, + converted_b); // beta = T(k + 1, k) = T(k, k + 1) + GKO_SOLVER_SCALAR(gamma, converted_b); // gamma = T(k - 1, k) + GKO_SOLVER_SCALAR(delta, converted_b); // delta = T(k - 2, k) + GKO_SOLVER_SCALAR(eta_next, converted_b); + GKO_SOLVER_SCALAR(eta, converted_b); + // this is the approximation of the residual norm squared, it is set + // to + // ||z||^2, but it could also use beta^2 or ||r||^2. It is based on + // the description of phi in: CHOI, Sou-Cheng. Iterative methods for + // singular linear equations and least-squares problems. 2006. + GKO_SOLVER_SCALAR(tau, converted_b); + + GKO_SOLVER_SCALAR(cos_prev, converted_b); + GKO_SOLVER_SCALAR(cos, converted_b); + GKO_SOLVER_SCALAR(sin_prev, converted_b); + GKO_SOLVER_SCALAR(sin, converted_b); + + GKO_SOLVER_ONE_MINUS_ONE(); + + bool one_changed{}; + GKO_SOLVER_STOP_REDUCTION_ARRAYS(converted_b->get_size()[1]); + + // r = converted_b + r->copy_from(converted_b); + this->get_system_matrix()->apply(neg_one_op, converted_x, one_op, + r); + auto stop_criterion = this->get_stop_criterion_factory()->generate( + this->get_system_matrix(), + std::shared_ptr( + converted_b, [](const AbstractMultiVector*) {}), + converted_x, r); + + // z = M^-1 * r + // beta = + // tau = + this->get_preconditioner()->apply(r, z); + r->compute_conj_dot(z, beta, reduction_tmp); + z->compute_conj_dot(z, tau, reduction_tmp); + + // beta = sqrt(beta) + // eta = eta_next = beta + // delta = gamma = cos_prev = sin_prev = cos = sin = 0 + // q = r / beta + // z = z / beta + // p = p_prev = q_prev = v = 0 + exec->run(minres::make_initialize( + r->template get_const_local_device_view(), + z->template get_local_device_view(), + p->template get_local_device_view(), + p_prev->template get_local_device_view(), + q->template get_local_device_view(), + q_prev->template get_local_device_view(), + v->template get_local_device_view(), + beta->get_device_view(), gamma->get_device_view(), + delta->get_device_view(), cos_prev->get_device_view(), + cos->get_device_view(), sin_prev->get_device_view(), + sin->get_device_view(), eta_next->get_device_view(), + eta->get_device_view(), stop_status)); + + int iter = -1; + /* Memory movement summary: + * 27n * values + matrix/preconditioner storage + * 1x SpMV: 2n * values + storage + * 1x Preconditioner: 2n * values + storage + * 2x dot 4n + * 1x axpy 3n + * 1x step 1 (axpys) 16n + */ + while (true) { + ++iter; + bool all_stopped = stop_criterion->update() + .num_iterations(iter) + .residual(nullptr) + .implicit_sq_residual_norm(tau) + .solution(converted_x) + .check(RelativeStoppingId, true, + &stop_status, &one_changed); + this->template log( + this, converted_b, converted_x, iter, r, nullptr, tau, + &stop_status, all_stopped); + if (all_stopped) { + break; + } + + // Lanzcos (partial) update: + // v = A * z - beta * q_prev + // alpha = + // v = v - alpha * q + // z_tilde = M * v + // beta = + this->get_system_matrix()->apply(one_op, z, neg_one_op, v); + v->compute_conj_dot(z, alpha, reduction_tmp); + v->sub_scaled(alpha, q); + this->get_preconditioner()->apply(v, z_tilde); + v->compute_conj_dot(z_tilde, beta, reduction_tmp); + + // Updates scalars (row vectors) + // finish Lanzcos: + // beta = sqrt(beta) + // + // apply two previous givens rotation to new column: + // delta = sin_prev * gamma // 0 if iter = 0, 1 + // tmp_d = gamma + // tmp_a = alpha + // gamma = cos_prev * cos * tmp_d + sin * tmp_a // 0 if iter = + // 0 alpha = -conj(sin) * cos_prev * tmp_d + cos * tmp_a + // + // compute and apply new Givens rotation: + // sin_prev = sin + // cos_prev = cos + // cos, sin = givens_rot(alpha, beta) + // alpha = cos * alpha + sin * beta + // + // apply new Givens rotation to eta: + // eta = eta_next + // eta_next = -conj(sin) * eta + // + // update the squared residual norm approximation: + // tau = abs(sin)^2 * tau + exec->run(minres::make_step_1( + alpha->get_device_view(), beta->get_device_view(), + gamma->get_device_view(), delta->get_device_view(), + cos_prev->get_device_view(), cos->get_device_view(), + sin_prev->get_device_view(), sin->get_device_view(), + eta->get_device_view(), eta_next->get_device_view(), + tau->get_device_view(), stop_status)); + + + // update vectors + // update search direction and solution: + // swap(p, p_prev) + // p = (z - gamma * p_prev - delta * p) / alpha + // x = x + cos * eta * p + // + // finish Lanzcos: + // q_prev = v + // q_tmp = q + // q = v / beta + // v = q_tmp * beta + // z = z_tilde / beta + // + // store previous beta in gamma: + // gamma = beta + swap(p, p_prev); + exec->run(minres::make_step_2( + converted_x->template get_local_device_view(), + p->template get_local_device_view(), + p_prev->template get_const_local_device_view(), + z->template get_local_device_view(), + z_tilde->template get_const_local_device_view(), + q->template get_local_device_view(), + q_prev->template get_local_device_view(), + v->template get_local_device_view(), + alpha->get_const_device_view(), + beta->get_const_device_view(), + gamma->get_const_device_view(), + delta->get_const_device_view(), + cos->get_const_device_view(), eta->get_const_device_view(), + stop_status)); + swap(gamma, beta); + } }, b, x); } @@ -94,231 +288,16 @@ typename Minres::parameters_type Minres::parse( } -/** - * This Minres implementation is based on Anne Grennbaum's 'Iterative Methods - * for Solving Linear Systems' (DOI: 10.1137/1.9781611970937) Ch. 2 and Ch. 8. - * Most variable names are taken from that reference, with the exception that - * the vector `w` and `w_tilde` from the reference are called `z` and `z_tilde`. - * The variable declaration have a comment to specify the name used in the - * reference. By reusing already allocated memory the number of necessary - * vectors is reduced to seven temporary vectors. The operations are grouped - * into point-wise scalar and vector updates, operator applications and - * (possibly) global reductions. With some reordering, as many point-wise - * updates are grouped together into a scalar and vector step respectively to - * reduce the number of kernel launches. The algorithm uses a recursion to - * compute an approximate residual norm. The residual is neither computed - * exactly, nor approximately, since that would require additional operations. - */ -template -template -void Minres::apply_dense_impl(const VectorType* dense_b, - VectorType* dense_x) const -{ - using std::swap; - using LocalVector = matrix::MultiVector; - using NormVector = typename LocalVector::absolute_type; - - constexpr uint8 RelativeStoppingId{1}; - - auto exec = this->get_executor(); - this->setup_workspace(); - - GKO_SOLVER_VECTOR(r, dense_b); - GKO_SOLVER_VECTOR(z, dense_b); // z = w_k+1 - GKO_SOLVER_VECTOR(p, dense_b); // p = p_k-1 - GKO_SOLVER_VECTOR(q, dense_b); // q = q_k+1 - GKO_SOLVER_VECTOR(v, dense_b); // v = v_k - - GKO_SOLVER_VECTOR(z_tilde, dense_b); // z_tilde = w_tilde_k+1 - GKO_SOLVER_VECTOR(p_prev, dense_b); // p_prev = p_k-2 - GKO_SOLVER_VECTOR(q_prev, dense_b); // q_prev = q_k - - GKO_SOLVER_SCALAR(alpha, dense_b); // alpha = T(k, k) - GKO_SOLVER_SCALAR(beta, dense_b); // beta = T(k + 1, k) = T(k, k + 1) - GKO_SOLVER_SCALAR(gamma, dense_b); // gamma = T(k - 1, k) - GKO_SOLVER_SCALAR(delta, dense_b); // delta = T(k - 2, k) - GKO_SOLVER_SCALAR(eta_next, dense_b); - GKO_SOLVER_SCALAR(eta, dense_b); - // this is the approximation of the residual norm squared, it is set to - // ||z||^2, but it could also use beta^2 or ||r||^2. It is based on the - // description of phi in: - // CHOI, Sou-Cheng. Iterative methods for singular linear equations and - // least-squares problems. 2006. - GKO_SOLVER_SCALAR(tau, dense_b); - - GKO_SOLVER_SCALAR(cos_prev, dense_b); - GKO_SOLVER_SCALAR(cos, dense_b); - GKO_SOLVER_SCALAR(sin_prev, dense_b); - GKO_SOLVER_SCALAR(sin, dense_b); - - GKO_SOLVER_ONE_MINUS_ONE(); - - bool one_changed{}; - GKO_SOLVER_STOP_REDUCTION_ARRAYS(); - - // r = dense_b - r->copy_from(dense_b); - this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, r); - auto stop_criterion = this->get_stop_criterion_factory()->generate( - this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, r); - - // z = M^-1 * r - // beta = - // tau = - this->get_preconditioner()->apply(r, z); - r->compute_conj_dot(z, beta, reduction_tmp); - z->compute_conj_dot(z, tau, reduction_tmp); - - // beta = sqrt(beta) - // eta = eta_next = beta - // delta = gamma = cos_prev = sin_prev = cos = sin = 0 - // q = r / beta - // z = z / beta - // p = p_prev = q_prev = v = 0 - exec->run(minres::make_initialize( - gko::detail::get_local(r)->get_const_device_view(), - gko::detail::get_local(z)->get_device_view(), - gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(p_prev)->get_device_view(), - gko::detail::get_local(q)->get_device_view(), - gko::detail::get_local(q_prev)->get_device_view(), - gko::detail::get_local(v)->get_device_view(), - gko::detail::get_local(beta)->get_device_view(), - gko::detail::get_local(gamma)->get_device_view(), - gko::detail::get_local(delta)->get_device_view(), - gko::detail::get_local(cos_prev)->get_device_view(), - gko::detail::get_local(cos)->get_device_view(), - gko::detail::get_local(sin_prev)->get_device_view(), - gko::detail::get_local(sin)->get_device_view(), - gko::detail::get_local(eta_next)->get_device_view(), - gko::detail::get_local(eta)->get_device_view(), stop_status)); - - int iter = -1; - /* Memory movement summary: - * 27n * values + matrix/preconditioner storage - * 1x SpMV: 2n * values + storage - * 1x Preconditioner: 2n * values + storage - * 2x dot 4n - * 1x axpy 3n - * 1x step 1 (axpys) 16n - */ - while (true) { - ++iter; - bool all_stopped = - stop_criterion->update() - .num_iterations(iter) - .residual(nullptr) - .implicit_sq_residual_norm(tau) - .solution(dense_x) - .check(RelativeStoppingId, true, &stop_status, &one_changed); - this->template log( - this, dense_b, dense_x, iter, r, nullptr, tau, &stop_status, - all_stopped); - if (all_stopped) { - break; - } - - // Lanzcos (partial) update: - // v = A * z - beta * q_prev - // alpha = - // v = v - alpha * q - // z_tilde = M * v - // beta = - this->get_system_matrix()->apply(one_op, z, neg_one_op, v); - v->compute_conj_dot(z, alpha, reduction_tmp); - v->sub_scaled(alpha, q); - this->get_preconditioner()->apply(v, z_tilde); - v->compute_conj_dot(z_tilde, beta, reduction_tmp); - - // Updates scalars (row vectors) - // finish Lanzcos: - // beta = sqrt(beta) - // - // apply two previous givens rotation to new column: - // delta = sin_prev * gamma // 0 if iter = 0, 1 - // tmp_d = gamma - // tmp_a = alpha - // gamma = cos_prev * cos * tmp_d + sin * tmp_a // 0 if iter = 0 - // alpha = -conj(sin) * cos_prev * tmp_d + cos * tmp_a - // - // compute and apply new Givens rotation: - // sin_prev = sin - // cos_prev = cos - // cos, sin = givens_rot(alpha, beta) - // alpha = cos * alpha + sin * beta - // - // apply new Givens rotation to eta: - // eta = eta_next - // eta_next = -conj(sin) * eta - // - // update the squared residual norm approximation: - // tau = abs(sin)^2 * tau - exec->run(minres::make_step_1( - gko::detail::get_local(alpha)->get_device_view(), - gko::detail::get_local(beta)->get_device_view(), - gko::detail::get_local(gamma)->get_device_view(), - gko::detail::get_local(delta)->get_device_view(), - gko::detail::get_local(cos_prev)->get_device_view(), - gko::detail::get_local(cos)->get_device_view(), - gko::detail::get_local(sin_prev)->get_device_view(), - gko::detail::get_local(sin)->get_device_view(), - gko::detail::get_local(eta)->get_device_view(), - gko::detail::get_local(eta_next)->get_device_view(), - gko::detail::get_local(tau)->get_device_view(), stop_status)); - - - // update vectors - // update search direction and solution: - // swap(p, p_prev) - // p = (z - gamma * p_prev - delta * p) / alpha - // x = x + cos * eta * p - // - // finish Lanzcos: - // q_prev = v - // q_tmp = q - // q = v / beta - // v = q_tmp * beta - // z = z_tilde / beta - // - // store previous beta in gamma: - // gamma = beta - swap(p, p_prev); - exec->run(minres::make_step_2( - gko::detail::get_local(dense_x)->get_device_view(), - gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(p_prev)->get_const_device_view(), - gko::detail::get_local(z)->get_device_view(), - gko::detail::get_local(z_tilde)->get_const_device_view(), - gko::detail::get_local(q)->get_device_view(), - gko::detail::get_local(q_prev)->get_device_view(), - gko::detail::get_local(v)->get_device_view(), - gko::detail::get_local(alpha)->get_const_device_view(), - gko::detail::get_local(beta)->get_const_device_view(), - gko::detail::get_local(gamma)->get_const_device_view(), - gko::detail::get_local(delta)->get_const_device_view(), - gko::detail::get_local(cos)->get_const_device_view(), - gko::detail::get_local(eta)->get_const_device_view(), stop_status)); - swap(gamma, beta); - } -} - - template -void Minres::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Minres::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/core/solver/multigrid.cpp b/core/solver/multigrid.cpp index 39f64ad2154..d4ebf7f80ea 100644 --- a/core/solver/multigrid.cpp +++ b/core/solver/multigrid.cpp @@ -102,7 +102,7 @@ void handle_list( auto gen_default_smoother = [&] { auto exec = matrix->get_executor(); #if GINKGO_BUILD_MPI - if (gko::detail::is_distributed(matrix.get())) { + if (experimental::distributed::detail::is_distributed(matrix.get())) { using experimental::distributed::Matrix; return run, Matrix, @@ -269,7 +269,8 @@ class MultigridState { */ void run_mg_cycle(multigrid::cycle cycle, size_type level, const std::shared_ptr& matrix, - const LinOp* b, LinOp* x, cycle_mode mode); + const AbstractMultiVector* b, AbstractMultiVector* x, + cycle_mode mode); /** * @copydoc run_cycle @@ -280,18 +281,19 @@ class MultigridState { */ template void run_cycle(multigrid::cycle cycle, size_type level, - const std::shared_ptr& matrix, const LinOp* b, - LinOp* x, cycle_mode mode); + const std::shared_ptr& matrix, + const AbstractMultiVector* b, AbstractMultiVector* x, + cycle_mode mode); // current level's nrows x nrhs - std::vector> r_list; + std::vector> r_list; // next level's nrows x nrhs - std::vector> g_list; - std::vector> e_list; + std::vector> g_list; + std::vector> e_list; // constant 1 x 1 - std::vector> one_list; - std::vector> next_one_list; - std::vector> neg_one_list; + std::vector> one_list; + std::vector> next_one_list; + std::vector> neg_one_list; const LinOp* system_matrix; const Multigrid* multigrid; size_type nrhs; @@ -332,7 +334,8 @@ void MultigridState::generate(const LinOp* system_matrix_in, [&, this](auto mg_level, auto i, auto cycle, auto current_nrows, auto next_nrows) { #if GINKGO_BUILD_MPI - if (gko::detail::is_distributed(system_matrix_in)) { + if (experimental::distributed::detail::is_distributed( + system_matrix_in)) { using value_type = typename std::decay_t::value_type; using VectorType = @@ -348,14 +351,17 @@ void MultigridState::generate(const LinOp* system_matrix_in, auto current_comm = distributed_fine->get_communicator(); auto next_comm = distributed_coarse->get_communicator(); auto current_local_nrows = - ::gko::detail::run_matrix(fine, [](auto* fine_mat) { - return fine_mat->get_diag_matrix()->get_size()[0]; - }); + experimental::distributed::detail::run_matrix( + fine, [](auto* fine_mat) { + return fine_mat->get_diag_matrix() + ->get_size()[0]; + }); auto next_local_nrows = - ::gko::detail::run_matrix(coarse, [](auto* coarse_mat) { - return coarse_mat->get_off_diag_matrix() - ->get_size()[0]; - }); + experimental::distributed::detail::run_matrix( + coarse, [](auto* coarse_mat) { + return coarse_mat->get_off_diag_matrix() + ->get_size()[0]; + }); this->allocate_memory( i, cycle, current_comm, next_comm, current_nrows, next_nrows, current_local_nrows, next_local_nrows); @@ -458,7 +464,8 @@ void MultigridState::allocate_memory( void MultigridState::run_mg_cycle(multigrid::cycle cycle, size_type level, const std::shared_ptr& matrix, - const LinOp* b, LinOp* x, cycle_mode mode) + const AbstractMultiVector* b, + AbstractMultiVector* x, cycle_mode mode) { if (level == multigrid->get_mg_level_list().size()) { multigrid->get_coarsest_solver()->apply(b, x); @@ -475,7 +482,8 @@ void MultigridState::run_mg_cycle(multigrid::cycle cycle, size_type level, std::complex, std::complex>( mg_level, [&, this](auto mg_level) { #if GINKGO_BUILD_MPI - if (gko::detail::is_distributed(matrix.get())) { + if (experimental::distributed::detail::is_distributed( + matrix.get())) { using value_type = typename std::decay_t::value_type; this->run_cycle< @@ -496,7 +504,8 @@ void MultigridState::run_mg_cycle(multigrid::cycle cycle, size_type level, template void MultigridState::run_cycle(multigrid::cycle cycle, size_type level, const std::shared_ptr& matrix, - const LinOp* b, LinOp* x, cycle_mode mode) + const AbstractMultiVector* b, + AbstractMultiVector* x, cycle_mode mode) { using value_type = typename VectorType::value_type; auto total_level = multigrid->get_mg_level_list().size(); @@ -788,7 +797,8 @@ void Multigrid::generate() // TODO: maybe remove fixed index type auto gen_default_solver = [&]() -> std::unique_ptr { #if GINKGO_BUILD_MPI - if (gko::detail::is_distributed(matrix.get())) { + if (gko::experimental::distributed::detail::is_distributed( + matrix.get())) { using absolute_value_type = remove_complex; using experimental::distributed::Matrix; return run, @@ -869,14 +879,16 @@ void Multigrid::generate() } -void Multigrid::apply_impl(const LinOp* b, LinOp* x) const +void Multigrid::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { this->apply_with_initial_guess_impl(b, x, this->get_default_initial_guess()); } -void Multigrid::apply_with_initial_guess_impl(const LinOp* b, LinOp* x, +void Multigrid::apply_with_initial_guess_impl(const AbstractMultiVector* b, + AbstractMultiVector* x, initial_guess_mode guess) const { if (!this->get_system_matrix() || !this->get_system_matrix()->get_size()) { @@ -886,10 +898,10 @@ void Multigrid::apply_with_initial_guess_impl(const LinOp* b, LinOp* x, auto lambda = [this, guess](auto mg_level, auto b, auto x) { using value_type = typename std::decay_t::value_type; - experimental::precision_dispatch_real_complex_distributed( - [this, guess](auto dense_b, auto dense_x) { - prepare_initial_guess(dense_b, dense_x, guess); - this->apply_dense_impl(dense_b, dense_x, guess); + precision_dispatch( + [this, guess](auto converted_b, auto converted_x) { + prepare_initial_guess(converted_b, converted_x, guess); + this->apply_dense_impl(converted_b, converted_x, guess); }, b, x); }; @@ -906,17 +918,20 @@ void Multigrid::apply_with_initial_guess_impl(const LinOp* b, LinOp* x, } -void Multigrid::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Multigrid::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { this->apply_with_initial_guess_impl(alpha, b, beta, x, this->get_default_initial_guess()); } -void Multigrid::apply_with_initial_guess_impl(const LinOp* alpha, - const LinOp* b, const LinOp* beta, - LinOp* x, +void Multigrid::apply_with_initial_guess_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x, initial_guess_mode guess) const { if (!this->get_system_matrix() || !this->get_system_matrix()->get_size()) { @@ -927,16 +942,15 @@ void Multigrid::apply_with_initial_guess_impl(const LinOp* alpha, auto x) { using value_type = typename std::decay_t::value_type; - experimental::precision_dispatch_real_complex_distributed( - [this, guess](auto dense_alpha, auto dense_b, auto dense_beta, - auto dense_x) { - prepare_initial_guess(dense_b, dense_x, guess); - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get(), guess); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); + precision_dispatch( + [this, guess, alpha, beta](auto converted_b, auto converted_x) { + prepare_initial_guess(converted_b, converted_x, guess); + auto x_clone = converted_x->clone(); + this->apply_dense_impl(converted_b, x_clone.get(), guess); + converted_x->scale(beta); + converted_x->add_scaled(alpha, x_clone); }, - alpha, b, beta, x); + b, x); }; auto first_mg_level = this->get_mg_level_list().front(); run -void Multigrid::apply_dense_impl(const VectorType* b, VectorType* x, +void Multigrid::apply_dense_impl(const AbstractMultiVector* b, + AbstractMultiVector* x, initial_guess_mode guess) const { using ws = workspace_traits; @@ -960,7 +974,7 @@ void Multigrid::apply_dense_impl(const VectorType* b, VectorType* x, cache_.state->generate(this->get_system_matrix().get(), this, b->get_size()[1]); } - auto lambda = [&, this](auto mg_level, auto b, auto x) { + auto lambda = [&, this](auto mg_level, auto b_, auto x_) { using value_type = typename std::decay_t::value_type; auto exec = this->get_executor(); @@ -969,13 +983,14 @@ void Multigrid::apply_dense_impl(const VectorType* b, VectorType* x, constexpr uint8 RelativeStoppingId{1}; auto& stop_status = this->template create_workspace_array( - ws::stop, b->get_size()[1]); + ws::stop, b_->get_size()[1]); bool one_changed{}; exec->run(multigrid::make_initialize(stop_status)); auto stop_criterion = this->get_stop_criterion_factory()->generate( this->get_system_matrix(), - std::shared_ptr(b, null_deleter{}), x, - nullptr); + std::shared_ptr( + b_, null_deleter{}), + x_, nullptr); int iter = -1; while (true) { @@ -987,11 +1002,11 @@ void Multigrid::apply_dense_impl(const VectorType* b, VectorType* x, // currently, the residual will computed additionally in // stop_criterion when users require the corresponding // residual check. - .solution(x) + .solution(x_) .check(RelativeStoppingId, true, &stop_status, &one_changed); this->template log( - this, b, x, iter, nullptr, nullptr, nullptr, &stop_status, + this, b_, x_, iter, nullptr, nullptr, nullptr, &stop_status, all_stopped); if (all_stopped) { break; @@ -1002,7 +1017,7 @@ void Multigrid::apply_dense_impl(const VectorType* b, VectorType* x, mode = mode | multigrid::cycle_mode::x_is_zero; } cache_.state->run_mg_cycle(this->get_parameters().cycle, 0, - this->get_system_matrix(), b, x, mode); + this->get_system_matrix(), b_, x_, mode); } }; diff --git a/core/solver/pipe_cg.cpp b/core/solver/pipe_cg.cpp index 34d3175f656..1fc1098ad0c 100644 --- a/core/solver/pipe_cg.cpp +++ b/core/solver/pipe_cg.cpp @@ -10,13 +10,10 @@ #include #include #include -#include -#include #include -#include +#include "core/base/dispatch_helper.hpp" #include "core/config/solver_config.hpp" -#include "core/distributed/helpers.hpp" #include "core/solver/pipe_cg_kernels.hpp" #include "core/solver/solver_boilerplate.hpp" @@ -77,247 +74,245 @@ std::unique_ptr PipeCg::conj_transpose() const template -void PipeCg::apply_impl(const LinOp* b, LinOp* x) const +void PipeCg::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_b, auto dense_x) { - this->apply_dense_impl(dense_b, dense_x); + + precision_dispatch( + [this](auto converted_b, auto converted_x) { + using std::swap; + using LocalVector = matrix::MultiVector; + + constexpr uint8 RelativeStoppingId{1}; + + auto exec = this->get_executor(); + this->setup_workspace(); + + // we combine the two vectors r and w, formerly created with + // GKO_SOLVER_VECTOR(r, converted_b); + // GKO_SOLVER_VECTOR(w, converted_b); + // into rw that we later slice for efficient dot product computation + auto local_original_size = + converted_b->template get_const_local_device_view() + .size; + auto global_original_size = converted_b->get_size(); + dim<2> local_conjoined_size = {local_original_size[0], + local_original_size[1] * 2}; + dim<2> global_conjoined_size = {global_original_size[0], + local_original_size[1] * 2}; + + AbstractMultiVector* rw = this->create_workspace_op_with_type_of( + GKO_SOLVER_TRAITS::rw, converted_b, global_conjoined_size, + local_conjoined_size); + auto r_unique = rw->create_subview( + local_span{0, local_original_size[0]}, + local_span{0, local_original_size[1]}, global_original_size); + auto* r = r_unique.get(); + auto w_unique = rw->create_subview( + local_span{0, local_original_size[0]}, + local_span{local_original_size[1], + local_original_size[1] + local_original_size[1]}, + global_original_size); + auto* w = w_unique.get(); + + // z now consists of two identical repeating parts: z1 and z2, + // again, for the same reason + GKO_SOLVER_VECTOR(z, rw); + auto z1_unique = z->create_subview( + local_span{0, local_original_size[0]}, + local_span{0, local_original_size[1]}, global_original_size); + auto* z1 = z1_unique.get(); + auto z2_unique = z->create_subview( + local_span{0, local_original_size[0]}, + local_span{local_original_size[1], + local_original_size[1] + local_original_size[1]}, + global_original_size); + auto* z2 = z2_unique.get(); + + GKO_SOLVER_VECTOR(p, converted_b); + GKO_SOLVER_VECTOR(m, converted_b); + GKO_SOLVER_VECTOR(n, converted_b); + GKO_SOLVER_VECTOR(q, converted_b); + GKO_SOLVER_VECTOR(f, converted_b); + GKO_SOLVER_VECTOR(g, converted_b); + + // rho and delta become combined as well + GKO_SOLVER_SCALAR(rhodelta, rw); + auto rho_unique = rhodelta->create_subview( + local_span{0, 1}, local_span{0, local_original_size[1]}, + dim<2>{1, global_original_size[1]}); + auto* rho = rho_unique.get(); + auto delta_unique = rhodelta->create_subview( + local_span{0, 1}, + local_span{local_original_size[1], + local_original_size[1] + local_original_size[1]}, + dim<2>{1, global_original_size[1]}); + auto* delta = delta_unique.get(); + + GKO_SOLVER_SCALAR(beta, converted_b); + GKO_SOLVER_SCALAR(prev_rho, converted_b); + + GKO_SOLVER_ONE_MINUS_ONE(); + + bool one_changed{}; + + // needs to match the size of the combined rhodelta + auto& stop_status = + this->template create_workspace_array( + GKO_SOLVER_TRAITS::stop, global_original_size[1]); + auto& reduction_tmp = this->template create_workspace_array( + GKO_SOLVER_TRAITS::tmp); + + // r = b + // prev_rho = 1.0 + exec->run(pipe_cg::make_initialize_1( + converted_b->template get_const_local_device_view(), + r->template get_local_device_view(), + prev_rho->get_device_view(), stop_status)); + // r = r - Ax + this->get_system_matrix()->apply(neg_one_op, converted_x, one_op, + r); + // z = preconditioner * r + this->get_preconditioner()->apply(r, z1); + // z2 = z1 + z2->copy_from(z1); + // w = A * z + this->get_system_matrix()->apply(z1, w); + // m = preconditioner * w + this->get_preconditioner()->apply(w, m); + // n = A * m + this->get_system_matrix()->apply(m, n); + // merged dot products + // rho = dot(r, z1) + // delta = dot(w, z2) + rw->compute_conj_dot(z, rhodelta, reduction_tmp); + + // check for an early termination + auto stop_criterion = this->get_stop_criterion_factory()->generate( + this->get_system_matrix(), + std::shared_ptr( + converted_b, [](const AbstractMultiVector*) {}), + converted_x, r); + int iter = 0; + bool all_stopped = stop_criterion->update() + .num_iterations(iter) + .residual(r) + .implicit_sq_residual_norm(rho) + .solution(converted_x) + .check(RelativeStoppingId, true, + &stop_status, &one_changed); + this->template log( + this, converted_b, converted_x, iter, r, nullptr, rho, + &stop_status, all_stopped); + if (all_stopped) { + return; + } + + // beta = delta + // p = z + // q = w + // f = m + // g = n + exec->run(pipe_cg::make_initialize_2( + p->template get_local_device_view(), + q->template get_local_device_view(), + f->template get_local_device_view(), + g->template get_local_device_view(), + beta->get_device_view(), + z1->template get_const_local_device_view(), + w->template get_const_local_device_view(), + m->template get_const_local_device_view(), + n->template get_const_local_device_view(), + delta->get_const_device_view())); + + /* Memory movement summary: + TODO + */ + while (true) { + // tmp = rho / beta + // x = x + tmp * p + // r = r - tmp * q + // z = z - tmp * f + // w = w - tmp * g + // it's the only place where z is updated so we updated both z1 + // and z2 here + exec->run(pipe_cg::make_step_1( + converted_x->template get_local_device_view(), + r->template get_local_device_view(), + z1->template get_local_device_view(), + z2->template get_local_device_view(), + w->template get_local_device_view(), + p->template get_const_local_device_view(), + q->template get_const_local_device_view(), + f->template get_const_local_device_view(), + g->template get_const_local_device_view(), + rho->get_const_device_view(), beta->get_const_device_view(), + stop_status)); + + // m = preconditioner * w + this->get_preconditioner()->apply(w, m); + // n = A * m + this->get_system_matrix()->apply(m, n); + // prev_rho = rho + prev_rho->copy_from(rho); + // merged dot products + // rho = dot(r, z1) + // delta = dot(w, z2) + rw->compute_conj_dot(z, rhodelta, reduction_tmp); + // check + ++iter; + bool all_stopped = stop_criterion->update() + .num_iterations(iter) + .residual(r) + .implicit_sq_residual_norm(rho) + .solution(converted_x) + .check(RelativeStoppingId, true, + &stop_status, &one_changed); + this->template log( + this, converted_b, converted_x, iter, r, nullptr, rho, + &stop_status, all_stopped); + if (all_stopped) { + break; + } + + // tmp = rho / prev_rho + // beta = delta - |tmp|^2 * beta + // p = z + tmp * p + // q = w + tmp * q + // f = m + tmp * f + // g = n + tmp * g + exec->run(pipe_cg::make_step_2( + beta->get_device_view(), + p->template get_local_device_view(), + q->template get_local_device_view(), + f->template get_local_device_view(), + g->template get_local_device_view(), + z1->template get_const_local_device_view(), + w->template get_const_local_device_view(), + m->template get_const_local_device_view(), + n->template get_const_local_device_view(), + prev_rho->get_const_device_view(), + rho->get_const_device_view(), + delta->get_const_device_view(), stop_status)); + } }, b, x); } template -template -void PipeCg::apply_dense_impl(const VectorType* dense_b, - VectorType* dense_x) const -{ - using std::swap; - using LocalVector = matrix::MultiVector; - - constexpr uint8 RelativeStoppingId{1}; - - auto exec = this->get_executor(); - this->setup_workspace(); - - // we combine the two vectors r and w, formerly created with - // GKO_SOLVER_VECTOR(r, dense_b); - // GKO_SOLVER_VECTOR(w, dense_b); - // into rw that we later slice for efficient dot product computation - auto b_stride = dense_b->get_stride(); - - auto local_original_size = ::gko::detail::get_local(dense_b)->get_size(); - auto global_original_size = dense_b->get_size(); - dim<2> local_conjoined_size = {local_original_size[0], b_stride * 2}; - dim<2> global_conjoined_size = {global_original_size[0], b_stride * 2}; - - VectorType* rw = - this->template create_workspace_op_with_type_of( - GKO_SOLVER_TRAITS::rw, dense_b, global_conjoined_size, - local_conjoined_size); - auto r_unique = rw->create_submatrix(local_span{0, local_original_size[0]}, - local_span{0, local_original_size[1]}, - global_original_size); - auto* r = r_unique.get(); - auto w_unique = rw->create_submatrix( - local_span{0, local_original_size[0]}, - local_span{b_stride, b_stride + local_original_size[1]}, - global_original_size); - auto* w = w_unique.get(); - - // z now consists of two identical repeating parts: z1 and z2, again, for - // the same reason - GKO_SOLVER_VECTOR(z, rw); - auto z1_unique = z->create_submatrix(local_span{0, local_original_size[0]}, - local_span{0, local_original_size[1]}, - global_original_size); - auto* z1 = z1_unique.get(); - auto z2_unique = z->create_submatrix( - local_span{0, local_original_size[0]}, - local_span{b_stride, b_stride + local_original_size[1]}, - global_original_size); - auto* z2 = z2_unique.get(); - - GKO_SOLVER_VECTOR(p, dense_b); - GKO_SOLVER_VECTOR(m, dense_b); - GKO_SOLVER_VECTOR(n, dense_b); - GKO_SOLVER_VECTOR(q, dense_b); - GKO_SOLVER_VECTOR(f, dense_b); - GKO_SOLVER_VECTOR(g, dense_b); - - // rho and delta become combined as well - GKO_SOLVER_SCALAR(rhodelta, rw); - auto rho_unique = rhodelta->create_submatrix( - local_span{0, 1}, local_span{0, local_original_size[1]}, - dim<2>{1, global_original_size[1]}); - auto* rho = rho_unique.get(); - auto delta_unique = rhodelta->create_submatrix( - local_span{0, 1}, - local_span{b_stride, b_stride + local_original_size[1]}, - dim<2>{1, global_original_size[1]}); - auto* delta = delta_unique.get(); - - GKO_SOLVER_SCALAR(beta, dense_b); - GKO_SOLVER_SCALAR(prev_rho, dense_b); - - GKO_SOLVER_ONE_MINUS_ONE(); - - bool one_changed{}; - - // needs to match the size of the combined rhodelta - auto& stop_status = this->template create_workspace_array( - GKO_SOLVER_TRAITS::stop, global_original_size[1]); - auto& reduction_tmp = - this->template create_workspace_array(GKO_SOLVER_TRAITS::tmp); - - // r = b - // prev_rho = 1.0 - exec->run(pipe_cg::make_initialize_1( - gko::detail::get_local(dense_b)->get_const_device_view(), - gko::detail::get_local(r)->get_device_view(), - prev_rho->get_device_view(), stop_status)); - // r = r - Ax - this->get_system_matrix()->apply(neg_one_op, dense_x, one_op, r); - // z = preconditioner * r - this->get_preconditioner()->apply(r, z1); - // z2 = z1 - z2->copy_from(z1); - // w = A * z - this->get_system_matrix()->apply(z1, w); - // m = preconditioner * w - this->get_preconditioner()->apply(w, m); - // n = A * m - this->get_system_matrix()->apply(m, n); - // merged dot products - // rho = dot(r, z1) - // delta = dot(w, z2) - rw->compute_conj_dot(z, rhodelta, reduction_tmp); - - // check for an early termination - auto stop_criterion = this->get_stop_criterion_factory()->generate( - this->get_system_matrix(), - std::shared_ptr(dense_b, [](const LinOp*) {}), dense_x, r); - int iter = 0; - bool all_stopped = - stop_criterion->update() - .num_iterations(iter) - .residual(r) - .implicit_sq_residual_norm(rho) - .solution(dense_x) - .check(RelativeStoppingId, true, &stop_status, &one_changed); - this->template log( - this, dense_b, dense_x, iter, r, nullptr, rho, &stop_status, - all_stopped); - if (all_stopped) { - return; - } - - // beta = delta - // p = z - // q = w - // f = m - // g = n - exec->run(pipe_cg::make_initialize_2( - gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(q)->get_device_view(), - gko::detail::get_local(f)->get_device_view(), - gko::detail::get_local(g)->get_device_view(), beta->get_device_view(), - gko::detail::get_local(z1)->get_const_device_view(), - gko::detail::get_local(w)->get_const_device_view(), - gko::detail::get_local(m)->get_const_device_view(), - gko::detail::get_local(n)->get_const_device_view(), - delta->get_const_device_view())); - - /* Memory movement summary: - TODO - */ - while (true) { - // tmp = rho / beta - // x = x + tmp * p - // r = r - tmp * q - // z = z - tmp * f - // w = w - tmp * g - // it's the only place where z is updated so we updated both z1 and z2 - // here - exec->run(pipe_cg::make_step_1( - gko::detail::get_local(dense_x)->get_device_view(), - gko::detail::get_local(r)->get_device_view(), - gko::detail::get_local(z1)->get_device_view(), - gko::detail::get_local(z2)->get_device_view(), - gko::detail::get_local(w)->get_device_view(), - gko::detail::get_local(p)->get_const_device_view(), - gko::detail::get_local(q)->get_const_device_view(), - gko::detail::get_local(f)->get_const_device_view(), - gko::detail::get_local(g)->get_const_device_view(), - rho->get_const_device_view(), beta->get_const_device_view(), - stop_status)); - - // m = preconditioner * w - this->get_preconditioner()->apply(w, m); - // n = A * m - this->get_system_matrix()->apply(m, n); - // prev_rho = rho - prev_rho->copy_from(rho); - // merged dot products - // rho = dot(r, z1) - // delta = dot(w, z2) - rw->compute_conj_dot(z, rhodelta, reduction_tmp); - // check - ++iter; - bool all_stopped = - stop_criterion->update() - .num_iterations(iter) - .residual(r) - .implicit_sq_residual_norm(rho) - .solution(dense_x) - .check(RelativeStoppingId, true, &stop_status, &one_changed); - this->template log( - this, dense_b, dense_x, iter, r, nullptr, rho, &stop_status, - all_stopped); - if (all_stopped) { - break; - } - - // tmp = rho / prev_rho - // beta = delta - |tmp|^2 * beta - // p = z + tmp * p - // q = w + tmp * q - // f = m + tmp * f - // g = n + tmp * g - exec->run(pipe_cg::make_step_2( - beta->get_device_view(), - gko::detail::get_local(p)->get_device_view(), - gko::detail::get_local(q)->get_device_view(), - gko::detail::get_local(f)->get_device_view(), - gko::detail::get_local(g)->get_device_view(), - gko::detail::get_local(z1)->get_const_device_view(), - gko::detail::get_local(w)->get_const_device_view(), - gko::detail::get_local(m)->get_const_device_view(), - gko::detail::get_local(n)->get_const_device_view(), - prev_rho->get_const_device_view(), rho->get_const_device_view(), - delta->get_const_device_view(), stop_status)); - } -} - - -template -void PipeCg::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void PipeCg::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - experimental::precision_dispatch_real_complex_distributed( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_dense_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/core/solver/solver_base.hpp b/core/solver/solver_base.hpp index cdf87692dc9..d43d63c570a 100644 --- a/core/solver/solver_base.hpp +++ b/core/solver/solver_base.hpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -20,12 +20,12 @@ namespace solver { * @param x the input vectors * @param guess the input guess */ -template -void prepare_initial_guess(const VectorType* b, VectorType* x, - initial_guess_mode guess) +inline void prepare_initial_guess(const AbstractMultiVector* b, + AbstractMultiVector* x, + initial_guess_mode guess) { if (guess == initial_guess_mode::zero) { - x->fill(zero()); + x->fill(0.0); } else if (guess == initial_guess_mode::rhs) { x->copy_from(b); } diff --git a/core/solver/solver_boilerplate.hpp b/core/solver/solver_boilerplate.hpp index db380833ca5..75b70f3cfe7 100644 --- a/core/solver/solver_boilerplate.hpp +++ b/core/solver/solver_boilerplate.hpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -25,9 +25,9 @@ auto neg_one_op = this->template create_workspace_fixed_scalar( \ GKO_SOLVER_TRAITS::minus_one, 1, -one()) -#define GKO_SOLVER_STOP_REDUCTION_ARRAYS() \ +#define GKO_SOLVER_STOP_REDUCTION_ARRAYS(_cols) \ auto& stop_status = \ this->template create_workspace_array( \ - GKO_SOLVER_TRAITS::stop, dense_b->get_size()[1]); \ + GKO_SOLVER_TRAITS::stop, _cols); \ auto& reduction_tmp = \ this->template create_workspace_array(GKO_SOLVER_TRAITS::tmp) diff --git a/core/solver/update_residual.hpp b/core/solver/update_residual.hpp index af62ed57a71..9f2c4eb0239 100644 --- a/core/solver/update_residual.hpp +++ b/core/solver/update_residual.hpp @@ -17,10 +17,12 @@ namespace gko { namespace solver { -template -bool update_residual(SolverType* solver, int iter, const VectorType* dense_b, - VectorType* dense_x, VectorType* residual, - const VectorType*& residual_ptr, +template +bool update_residual(SolverType* solver, int iter, + const AbstractMultiVector* dense_b, + AbstractMultiVector* dense_x, + AbstractMultiVector* residual, + const AbstractMultiVector*& residual_ptr, std::unique_ptr& stop_criterion, array& stop_status, LogFunc log) { diff --git a/core/solver/upper_trs.cpp b/core/solver/upper_trs.cpp index f2fc56935e9..021014e8c23 100644 --- a/core/solver/upper_trs.cpp +++ b/core/solver/upper_trs.cpp @@ -9,13 +9,13 @@ #include #include #include -#include #include #include #include #include #include +#include "core/base/dispatch_helper.hpp" #include "core/config/config_helper.hpp" #include "core/config/trisolver_config.hpp" #include "core/solver/upper_trs_kernels.hpp" @@ -162,13 +162,14 @@ static bool needs_transpose(std::shared_ptr exec) template -void UpperTrs::apply_impl(const LinOp* b, LinOp* x) const +void UpperTrs::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { + apply_precision_dispatch( + [this](auto view_b, auto view_x) { using Vector = matrix::MultiVector; using ws = workspace_traits; const auto exec = this->get_executor(); @@ -185,9 +186,9 @@ void UpperTrs::apply_impl(const LinOp* b, LinOp* x) const using optional_view = std::optional>; if (needs_transpose(exec)) { trans_b = this->template create_workspace_op( - ws::transposed_b, gko::transpose(dense_b->get_size())); + ws::transposed_b, gko::transpose(view_b.size)); trans_x = this->template create_workspace_op( - ws::transposed_x, gko::transpose(dense_x->get_size())); + ws::transposed_x, gko::transpose(view_x.size)); } exec->run(upper_trs::make_solve( this->get_system_matrix().get(), this->solve_struct_.get(), @@ -196,29 +197,21 @@ void UpperTrs::apply_impl(const LinOp* b, LinOp* x) const : optional_view{}, trans_x ? optional_view{trans_x->get_device_view()} : optional_view{}, - dense_b->get_const_device_view(), dense_x->get_device_view())); + view_b, view_x)); }, b, x); } template -void UpperTrs::apply_impl(const LinOp* alpha, - const LinOp* b, - const LinOp* beta, - LinOp* x) const +void UpperTrs::apply_impl( + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const { if (!this->get_system_matrix()) { return; } - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto x_clone = dense_x->clone(); - this->apply_impl(dense_b, x_clone.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, x_clone); - }, - alpha, b, beta, x); + LinOp::apply_impl(alpha, b, beta, x); } diff --git a/include/ginkgo/core/factorization/factorization.hpp b/include/ginkgo/core/factorization/factorization.hpp index f645201d25d..14275cfa94f 100644 --- a/include/ginkgo/core/factorization/factorization.hpp +++ b/include/ginkgo/core/factorization/factorization.hpp @@ -184,10 +184,13 @@ class Factorization : public LinOp { Factorization(std::unique_ptr> factors, storage_type type); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: storage_type storage_type_; diff --git a/include/ginkgo/core/solver/bicg.hpp b/include/ginkgo/core/solver/bicg.hpp index 68f044af555..d57c7b99091 100644 --- a/include/ginkgo/core/solver/bicg.hpp +++ b/include/ginkgo/core/solver/bicg.hpp @@ -99,13 +99,13 @@ class Bicg config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_dense_impl(const matrix::MultiVector* b, - matrix::MultiVector* x) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit Bicg(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/bicgstab.hpp b/include/ginkgo/core/solver/bicgstab.hpp index 0392d4484cb..d6158ea93ee 100644 --- a/include/ginkgo/core/solver/bicgstab.hpp +++ b/include/ginkgo/core/solver/bicgstab.hpp @@ -96,13 +96,13 @@ class Bicgstab config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - template - void apply_dense_impl(const VectorType* b, VectorType* x) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit Bicgstab(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/cb_gmres.hpp b/include/ginkgo/core/solver/cb_gmres.hpp index ffaf0bc59b5..4620e8a337b 100644 --- a/include/ginkgo/core/solver/cb_gmres.hpp +++ b/include/ginkgo/core/solver/cb_gmres.hpp @@ -164,13 +164,13 @@ class CbGmres : public LinOp, config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_dense_impl(const matrix::MultiVector* b, - matrix::MultiVector* x) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit CbGmres(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/cg.hpp b/include/ginkgo/core/solver/cg.hpp index ecee57bcd5f..45e16b21f05 100644 --- a/include/ginkgo/core/solver/cg.hpp +++ b/include/ginkgo/core/solver/cg.hpp @@ -62,7 +62,7 @@ class Cg : public LinOp, * * @return true as iterative solvers use the data in x as an initial guess. */ - bool apply_uses_initial_guess() const override { return true; } + bool apply_uses_initial_guess() const override; class Factory; @@ -92,13 +92,13 @@ class Cg : public LinOp, config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - template - void apply_dense_impl(const VectorType* b, VectorType* x) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit Cg(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/cgs.hpp b/include/ginkgo/core/solver/cgs.hpp index 492b40469a9..d194577bb33 100644 --- a/include/ginkgo/core/solver/cgs.hpp +++ b/include/ginkgo/core/solver/cgs.hpp @@ -89,13 +89,13 @@ class Cgs config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - template - void apply_dense_impl(const VectorType* b, VectorType* x) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit Cgs(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/chebyshev.hpp b/include/ginkgo/core/solver/chebyshev.hpp index 280a81f4e01..7dfbf962a4c 100644 --- a/include/ginkgo/core/solver/chebyshev.hpp +++ b/include/ginkgo/core/solver/chebyshev.hpp @@ -165,20 +165,29 @@ class Chebyshev final config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; - - template - void apply_dense_impl(const VectorType* b, VectorType* x, - initial_guess_mode guess) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; - - void apply_with_initial_guess_impl(const LinOp* b, LinOp* x, + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; + + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; + + // applies the solver, can only be called after both vectors have been + // converted to the precision of this and prepare_initial_guess has been + // called + void apply_with_initial_guess_prepared_impl(const AbstractMultiVector* b, + AbstractMultiVector* x, + initial_guess_mode guess) const; + + void apply_with_initial_guess_impl(const AbstractMultiVector* b, + AbstractMultiVector* x, initial_guess_mode guess) const override; - void apply_with_initial_guess_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x, + void apply_with_initial_guess_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x, initial_guess_mode guess) const override; explicit Chebyshev(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/direct.hpp b/include/ginkgo/core/solver/direct.hpp index 7de9d3215c3..19cf1b4215a 100644 --- a/include/ginkgo/core/solver/direct.hpp +++ b/include/ginkgo/core/solver/direct.hpp @@ -100,10 +100,13 @@ class Direct : public LinOp, Direct(const Factory* factory, std::shared_ptr system_matrix); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: using lower_type = gko::solver::LowerTrs; diff --git a/include/ginkgo/core/solver/fcg.hpp b/include/ginkgo/core/solver/fcg.hpp index df6b77ea522..9dbaeed3557 100644 --- a/include/ginkgo/core/solver/fcg.hpp +++ b/include/ginkgo/core/solver/fcg.hpp @@ -97,13 +97,13 @@ class Fcg config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - template - void apply_dense_impl(const VectorType* b, VectorType* x) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit Fcg(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/gcr.hpp b/include/ginkgo/core/solver/gcr.hpp index db34545fb00..93d1cc92375 100644 --- a/include/ginkgo/core/solver/gcr.hpp +++ b/include/ginkgo/core/solver/gcr.hpp @@ -109,13 +109,16 @@ class Gcr config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; template void apply_dense_impl(const VectorType* b, VectorType* x) const; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit Gcr(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/gmres.hpp b/include/ginkgo/core/solver/gmres.hpp index acb782bf7f3..5e87dc916ff 100644 --- a/include/ginkgo/core/solver/gmres.hpp +++ b/include/ginkgo/core/solver/gmres.hpp @@ -141,13 +141,13 @@ class Gmres config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - template - void apply_dense_impl(const VectorType* b, VectorType* x) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit Gmres(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/idr.hpp b/include/ginkgo/core/solver/idr.hpp index 03efe95b81a..f2b25c83107 100644 --- a/include/ginkgo/core/solver/idr.hpp +++ b/include/ginkgo/core/solver/idr.hpp @@ -212,10 +212,13 @@ class Idr config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; template void iterate(const VectorType* dense_b, VectorType* dense_x) const; diff --git a/include/ginkgo/core/solver/ir.hpp b/include/ginkgo/core/solver/ir.hpp index 1c3f8f33587..9273f8a0127 100644 --- a/include/ginkgo/core/solver/ir.hpp +++ b/include/ginkgo/core/solver/ir.hpp @@ -202,20 +202,29 @@ class Ir : public LinOp, config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; - - template - void apply_dense_impl(const VectorType* b, VectorType* x, - initial_guess_mode guess) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; - - void apply_with_initial_guess_impl(const LinOp* b, LinOp* x, + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; + + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; + + // applies the solver, can only be called after both vectors have been + // converted to the precision of this and prepare_initial_guess has been + // called + void apply_with_initial_guess_prepared_impl(const AbstractMultiVector* b, + AbstractMultiVector* x, + initial_guess_mode guess) const; + + void apply_with_initial_guess_impl(const AbstractMultiVector* b, + AbstractMultiVector* x, initial_guess_mode guess) const override; - void apply_with_initial_guess_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x, + void apply_with_initial_guess_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x, initial_guess_mode guess) const override; void set_relaxation_factor( diff --git a/include/ginkgo/core/solver/minres.hpp b/include/ginkgo/core/solver/minres.hpp index a8a9c0d44f0..da8e61b3c8b 100644 --- a/include/ginkgo/core/solver/minres.hpp +++ b/include/ginkgo/core/solver/minres.hpp @@ -101,13 +101,13 @@ class Minres config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - template - void apply_dense_impl(const VectorType* dense_b, VectorType* dense_x) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit Minres(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/multigrid.hpp b/include/ginkgo/core/solver/multigrid.hpp index 8bf5bf24e4b..2aa01f2e778 100644 --- a/include/ginkgo/core/solver/multigrid.hpp +++ b/include/ginkgo/core/solver/multigrid.hpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -398,20 +397,25 @@ class Multigrid : public LinOp, config::make_type_descriptor<>()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; - void apply_with_initial_guess_impl(const LinOp* b, LinOp* x, + void apply_with_initial_guess_impl(const AbstractMultiVector* b, + AbstractMultiVector* x, initial_guess_mode guess) const override; - void apply_with_initial_guess_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x, + void apply_with_initial_guess_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x, initial_guess_mode guess) const override; - template - void apply_dense_impl(const VectorType* b, VectorType* x, + void apply_dense_impl(const AbstractMultiVector* b, AbstractMultiVector* x, initial_guess_mode guess) const; /** diff --git a/include/ginkgo/core/solver/pipe_cg.hpp b/include/ginkgo/core/solver/pipe_cg.hpp index 9dc1753bf61..6aea8b2dcbf 100644 --- a/include/ginkgo/core/solver/pipe_cg.hpp +++ b/include/ginkgo/core/solver/pipe_cg.hpp @@ -106,13 +106,13 @@ class PipeCg config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - template - void apply_dense_impl(const VectorType* b, VectorType* x) const; - - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; explicit PipeCg(std::shared_ptr exec); diff --git a/include/ginkgo/core/solver/solver_base.hpp b/include/ginkgo/core/solver/solver_base.hpp index 66e22186105..ee0f568d3fd 100644 --- a/include/ginkgo/core/solver/solver_base.hpp +++ b/include/ginkgo/core/solver/solver_base.hpp @@ -79,10 +79,12 @@ class ApplyWithInitialGuess { * @param x the output vector(s) where the result is stored * @param guess the input guess to handle the input vector(s) */ - virtual void apply_with_initial_guess(const LinOp* b, LinOp* x, + virtual void apply_with_initial_guess(const AbstractMultiVector* b, + AbstractMultiVector* x, initial_guess_mode guess) const = 0; - void apply_with_initial_guess(ptr_param b, ptr_param x, + void apply_with_initial_guess(ptr_param b, + ptr_param x, initial_guess_mode guess) const { apply_with_initial_guess(b.get(), x.get(), guess); @@ -100,15 +102,17 @@ class ApplyWithInitialGuess { * @param x output vector(s) * @param guess the input guess to handle the input vector(s) */ - virtual void apply_with_initial_guess(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x, + virtual void apply_with_initial_guess(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x, initial_guess_mode guess) const = 0; - void apply_with_initial_guess(ptr_param alpha, - ptr_param b, - ptr_param beta, - ptr_param x, + void apply_with_initial_guess(ptr_param alpha, + ptr_param b, + ptr_param beta, + ptr_param x, initial_guess_mode guess) const { apply_with_initial_guess(alpha.get(), b.get(), beta.get(), x.get(), @@ -168,10 +172,12 @@ class EnableApplyWithInitialGuess : public ApplyWithInitialGuess { {} /** - * @copydoc apply_with_initial_guess(const LinOp*, LinOp*, - * initial_guess_mode) + * @copydoc apply_with_initial_guess(const AbstractMultiVector*, + * AbstractMultiVector*, + * initial_guess_mode) */ - void apply_with_initial_guess(const LinOp* b, LinOp* x, + void apply_with_initial_guess(const AbstractMultiVector* b, + AbstractMultiVector* x, initial_guess_mode guess) const override { self()->template log(self(), b, x); @@ -186,11 +192,14 @@ class EnableApplyWithInitialGuess : public ApplyWithInitialGuess { } /** - * @copydoc apply_with_initial_guess(const LinOp*,const LinOp*,const LinOp*, - * LinOp*, initial_guess_mode) + * @copydoc apply_with_initial_guess(const AbstractMultiVector*,const + * AbstractMultiVector*,const AbstractMultiVector*, + * AbstractMultiVector*, initial_guess_mode) */ - void apply_with_initial_guess(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x, + void apply_with_initial_guess(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x, initial_guess_mode guess) const override { self()->template log( @@ -216,14 +225,16 @@ class EnableApplyWithInitialGuess : public ApplyWithInitialGuess { * according to the initial_guess_mode */ virtual void apply_with_initial_guess_impl( - const LinOp* b, LinOp* x, initial_guess_mode guess) const = 0; + const AbstractMultiVector* b, AbstractMultiVector* x, + initial_guess_mode guess) const = 0; /** * The class should override this method and must modify the input vectors * according to the initial_guess_mode */ virtual void apply_with_initial_guess_impl( - const LinOp* alpha, const LinOp* b, const LinOp* beta, LinOp* x, + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x, initial_guess_mode guess) const = 0; GKO_ENABLE_SELF(DerivedType); @@ -377,9 +388,9 @@ class SolverBaseLinOp { return system_matrix_; } - const LinOp* get_workspace_op(int vector_id) const + const AbstractMultiVector* get_workspace_op(int vector_id) const { - return workspace_.get_op(vector_id); + return workspace_.get_vector(vector_id); } virtual int get_num_workspace_ops() const { return 0; } @@ -412,77 +423,68 @@ class SolverBaseLinOp { workspace_.set_size(num_operators, num_arrays); } - template - LinOpType* create_workspace_op(int vector_id, gko::dim<2> size) const + template >> + VectorType* create_workspace_op(int vector_id, gko::dim<2> size) const { - return workspace_.template create_or_get_op( + return as(workspace_.create_or_get_vector( vector_id, [&] { - return LinOpType::create(this->workspace_.get_executor(), size); + return VectorType::create(this->workspace_.get_executor(), + size); }, - typeid(LinOpType), size, size[1]); - } - - template - LinOpType* create_workspace_op_with_config_of(int vector_id, - const LinOpType* vec) const - { - return workspace_.template create_or_get_op( - vector_id, [&] { return LinOpType::create_with_config_of(vec); }, - typeid(*vec), vec->get_size(), vec->get_stride()); + typeid(VectorType), size)); } - template - LinOpType* create_workspace_op_with_type_of(int vector_id, - const LinOpType* vec, - dim<2> size) const + template >> + VectorType* create_workspace_op_with_config_of(int vector_id, + const VectorType* vec) const { - return workspace_.template create_or_get_op( - vector_id, - [&] { - return LinOpType::create_with_type_of( - vec, workspace_.get_executor(), size, size[1]); - }, - typeid(*vec), size, size[1]); + return as(workspace_.create_or_get_vector( + vector_id, [&] { return VectorType::create_with_config_of(vec); }, + typeid(*vec), vec->get_size())); } - template - LinOpType* create_workspace_op_with_type_of(int vector_id, - const LinOpType* vec, - dim<2> global_size, - dim<2> local_size) const + template >> + VectorType* create_workspace_op_with_type_of(int vector_id, + const VectorType* vec, + dim<2> global_size, + dim<2> local_size) const { - return workspace_.template create_or_get_op( + return as(workspace_.create_or_get_vector( vector_id, [&] { - return LinOpType::create_with_type_of( - vec, workspace_.get_executor(), global_size, local_size, - local_size[1]); + return VectorType::create_with_type_of( + vec, workspace_.get_executor(), global_size, local_size); }, - typeid(*vec), global_size, local_size[1]); + typeid(*vec), global_size)); } template matrix::MultiVector* create_workspace_scalar( int vector_id, size_type size) const { - return workspace_ - .template create_or_get_op>( + return as>( + workspace_.create_or_get_vector( vector_id, [&] { return matrix::MultiVector::create( workspace_.get_executor(), dim<2>{1, size}); }, - typeid(matrix::MultiVector), gko::dim<2>{1, size}, - size); + typeid(matrix::MultiVector), gko::dim<2>{1, size})); } template const matrix::MultiVector* create_workspace_fixed_scalar( int vector_id, size_type size, ValueType val) const { - return workspace_ - .template create_or_get_op>( + return as>( + workspace_.create_or_get_vector( vector_id, [&] { auto mat = matrix::MultiVector::create( @@ -490,8 +492,7 @@ class SolverBaseLinOp { mat->fill(val); return mat; }, - typeid(matrix::MultiVector), gko::dim<2>{1, size}, - size); + typeid(matrix::MultiVector), gko::dim<2>{1, size})); } template diff --git a/include/ginkgo/core/solver/triangular.hpp b/include/ginkgo/core/solver/triangular.hpp index 78a26d2e7ae..7ed22654f5c 100644 --- a/include/ginkgo/core/solver/triangular.hpp +++ b/include/ginkgo/core/solver/triangular.hpp @@ -156,10 +156,13 @@ class LowerTrs : public LinOp, protected: using CsrMatrix = matrix::Csr; - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; /** * Generates the analysis structure from the system matrix and the right @@ -312,10 +315,13 @@ class UpperTrs : public LinOp, protected: using CsrMatrix = matrix::Csr; - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; /** * Generates the analysis structure from the system matrix and the right diff --git a/include/ginkgo/core/solver/workspace.hpp b/include/ginkgo/core/solver/workspace.hpp index afc7a959bfc..509ddac6bb7 100644 --- a/include/ginkgo/core/solver/workspace.hpp +++ b/include/ginkgo/core/solver/workspace.hpp @@ -93,37 +93,35 @@ class workspace { return *this; } - template - LinOpType* create_or_get_op(int op_id, CreateOperation create, - const std::type_info& expected_type, - dim<2> size, size_type stride) + template + AbstractMultiVector* create_or_get_vector( + int op_id, CreateOperation create, const std::type_info& expected_type, + dim<2> size) { - GKO_ASSERT(op_id >= 0 && op_id < operators_.size()); + GKO_ASSERT(op_id >= 0 && op_id < vectors_.size()); // does the existing object have the wrong type? // vector types may vary e.g. if users derive from MultiVector - auto stored_op = operators_[op_id].get(); - LinOpType* op{}; + auto stored_op = vectors_[op_id].get(); + AbstractMultiVector* op{}; if (!stored_op || typeid(*stored_op) != expected_type) { auto new_op = create(); op = new_op.get(); - operators_[op_id] = std::move(new_op); + vectors_[op_id] = std::move(new_op); return op; } // does the existing object have the wrong dimensions? - op = dynamic_cast(operators_[op_id].get()); - GKO_ASSERT(op); - if (op->get_size() != size || op->get_stride() != stride) { + if (stored_op->get_size() != size) { auto new_op = create(); - op = new_op.get(); - operators_[op_id] = std::move(new_op); + stored_op = new_op.get(); + vectors_[op_id] = std::move(new_op); } - return op; + return stored_op; } - const LinOp* get_op(int op_id) const + const AbstractMultiVector* get_vector(int op_id) const { - GKO_ASSERT(op_id >= 0 && op_id < operators_.size()); - return operators_[op_id].get(); + GKO_ASSERT(op_id >= 0 && op_id < vectors_.size()); + return vectors_[op_id].get(); } template @@ -155,13 +153,13 @@ class workspace { void set_size(int num_operators, int num_arrays) { - operators_.resize(num_operators); + vectors_.resize(num_operators); arrays_.resize(num_arrays); } void clear() { - for (auto& op : operators_) { + for (auto& op : vectors_) { op.reset(); } for (auto& array : arrays_) { @@ -171,7 +169,7 @@ class workspace { private: std::shared_ptr exec_; - std::vector> operators_; + std::vector> vectors_; std::vector arrays_; }; From 63965d6c7cb3064e8c2014d9eee58a821c41cc57 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:04:33 +0200 Subject: [PATCH 15/21] fix hierarchy change for dense cache --- core/base/dense_cache.cpp | 2 +- core/base/dense_cache_accessor.hpp | 5 +++-- include/ginkgo/core/base/dense_cache.hpp | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/base/dense_cache.cpp b/core/base/dense_cache.cpp index c2bcfc12c65..3d7e0308dfd 100644 --- a/core/base/dense_cache.cpp +++ b/core/base/dense_cache.cpp @@ -98,7 +98,7 @@ double ScalarCacheAccessor::get_value(const ScalarCache& cache) } -const std::map>& +const std::map>& ScalarCacheAccessor::get_scalars(const ScalarCache& cache) { return cache.scalars; diff --git a/core/base/dense_cache_accessor.hpp b/core/base/dense_cache_accessor.hpp index ea5e16a8966..6d1f2701149 100644 --- a/core/base/dense_cache_accessor.hpp +++ b/core/base/dense_cache_accessor.hpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2025 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -38,7 +38,8 @@ class ScalarCacheAccessor { static double get_value(const ScalarCache& cache); // access to the scalars - static const std::map>& + static const std::map>& get_scalars(const ScalarCache& cache); }; diff --git a/include/ginkgo/core/base/dense_cache.hpp b/include/ginkgo/core/base/dense_cache.hpp index 788947f6e14..30d34d7f08f 100644 --- a/include/ginkgo/core/base/dense_cache.hpp +++ b/include/ginkgo/core/base/dense_cache.hpp @@ -18,7 +18,7 @@ namespace gko { -class LinOp; +class AbstractMultiVector; namespace matrix { @@ -172,7 +172,8 @@ struct ScalarCache { private: std::shared_ptr exec; double value; - mutable std::map> scalars; + mutable std::map> + scalars; }; From 37f546a7e262821ce3e5aeb0554f6c2883bbf010 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:04:59 +0200 Subject: [PATCH 16/21] fix hierarchy change for multigrid level --- core/multigrid/fixed_coarsening.cpp | 53 +++++++++++++++---- core/multigrid/pgm.cpp | 47 ++++++++++++++-- core/multigrid/rs.cpp | 11 ++-- .../core/multigrid/fixed_coarsening.hpp | 31 +++-------- include/ginkgo/core/multigrid/pgm.hpp | 33 +++--------- include/ginkgo/core/multigrid/rs.hpp | 9 ++-- 6 files changed, 113 insertions(+), 71 deletions(-) diff --git a/core/multigrid/fixed_coarsening.cpp b/core/multigrid/fixed_coarsening.cpp index c20dcff6b7f..6fcbb2f0fdd 100644 --- a/core/multigrid/fixed_coarsening.cpp +++ b/core/multigrid/fixed_coarsening.cpp @@ -34,6 +34,45 @@ GKO_REGISTER_OPERATION(fill_seq_array, components::fill_seq_array); } // namespace fixed_coarsening +template +void FixedCoarsening::apply_impl( + const AbstractMultiVector* b, AbstractMultiVector* x) const +{ + this->get_composition()->apply(b, x); +} + + +template +void FixedCoarsening::apply_impl( + const AbstractMultiVector* alpha, const AbstractMultiVector* b, + const AbstractMultiVector* beta, AbstractMultiVector* x) const +{ + this->get_composition()->apply(alpha, b, beta, x); +} + + +template +FixedCoarsening::FixedCoarsening( + std::shared_ptr exec) + : LinOp(std::move(exec)) +{} + + +template +FixedCoarsening::FixedCoarsening( + const Factory* factory, std::shared_ptr system_matrix) + : LinOp(factory->get_executor(), system_matrix->get_size()), + EnableMultigridLevel(system_matrix), + parameters_{factory->get_parameters()}, + system_matrix_{system_matrix} +{ + if (system_matrix_->get_size()[0] != 0) { + // generate on the existing matrix + this->generate(); + } +} + + template void FixedCoarsening::generate() { @@ -75,15 +114,11 @@ void FixedCoarsening::generate() auto prolong_op = gko::as(share(restrict_op->transpose())); // TODO: Can be done with submatrix index_set. - auto coarse_matrix = - share(csr_type::create(exec, gko::dim<2>{coarse_dim, coarse_dim})); - coarse_matrix->set_strategy(fixed_coarsening_op->get_strategy()); - auto tmp = csr_type::create(exec, gko::dim<2>{fine_dim, coarse_dim}); - tmp->set_strategy(fixed_coarsening_op->get_strategy()); - fixed_coarsening_op->apply(prolong_op, tmp); - restrict_op->apply(tmp, coarse_matrix); - - this->set_multigrid_level(prolong_op, coarse_matrix, restrict_op); + auto tmp = fixed_coarsening_op->multiply(prolong_op); + auto coarse_mtx = share(restrict_op->multiply(tmp)); + coarse_mtx->set_strategy(fixed_coarsening_op->get_strategy()); + + this->set_multigrid_level(prolong_op, coarse_mtx, restrict_op); } diff --git a/core/multigrid/pgm.cpp b/core/multigrid/pgm.cpp index 8c3e1f11d2f..b8ab9b3885b 100644 --- a/core/multigrid/pgm.cpp +++ b/core/multigrid/pgm.cpp @@ -182,6 +182,48 @@ Pgm::parse(const config::pnode& config, } +template +void Pgm::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const +{ + this->get_composition()->apply(b, x); +} + + +template +void Pgm::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const +{ + this->get_composition()->apply(alpha, b, beta, x); +} + + +template +Pgm::Pgm(std::shared_ptr exec) + : LinOp(std::move(exec)) +{} + + +template +Pgm::Pgm(const Factory* factory, + std::shared_ptr system_matrix) + : LinOp(factory->get_executor(), system_matrix->get_size()), + EnableMultigridLevel(system_matrix), + parameters_{factory->get_parameters()}, + system_matrix_{system_matrix}, + agg_(factory->get_executor(), system_matrix_->get_size()[0]) +{ + GKO_ASSERT(parameters_.max_unassigned_ratio <= 1.0); + GKO_ASSERT(parameters_.max_unassigned_ratio >= 0.0); + if (system_matrix_->get_size()[0] != 0) { + // generate on the existed matrix + this->generate(); + } +} + + template std::tuple, std::shared_ptr, std::shared_ptr> @@ -207,11 +249,10 @@ Pgm::generate_local( // compute weight_mtx = (abs(mtx) + abs(mtx'))/2; auto abs_mtx = local_matrix->compute_absolute(); // abs_mtx is already real valuetype, so transpose is enough - auto weight_mtx = gko::as(abs_mtx->transpose()); auto half_scalar = initialize>({0.5}, exec); - auto identity = matrix::Identity::create(exec, num_rows); // W = (abs_mtx + transpose(abs_mtx))/2 - abs_mtx->apply(half_scalar, identity, half_scalar, weight_mtx); + auto weight_mtx = abs_mtx->scale_add( + half_scalar, half_scalar, as(abs_mtx->transpose())); // Extract the diagonal value of matrix auto diag = weight_mtx->extract_diagonal(); for (int i = 0; i < parameters_.max_iterations; i++) { diff --git a/core/multigrid/rs.cpp b/core/multigrid/rs.cpp index f6fd244e15f..3dd796d337f 100644 --- a/core/multigrid/rs.cpp +++ b/core/multigrid/rs.cpp @@ -110,15 +110,10 @@ void Rs::generate() auto restrict_op = share(as(prolong_op->transpose())); // coarse matrix (Ac = R A P) - auto coarse_matrix = share( - csr_type::create(exec, gko::dim<2>{coarse_dim_size, coarse_dim_size})); - coarse_matrix->set_strategy(rs_op->get_strategy()); - - auto tmp = csr_type::create(exec, gko::dim<2>{fine_dim, coarse_dim_size}); + auto tmp = rs_op->multiply(prolong_op); tmp->set_strategy(rs_op->get_strategy()); - - rs_op->apply(prolong_op, tmp); - restrict_op->apply(tmp, coarse_matrix); + auto coarse_matrix = share(restrict_op->multiply(tmp)); + coarse_matrix->set_strategy(rs_op->get_strategy()); this->set_multigrid_level(prolong_op, coarse_matrix, restrict_op); } diff --git a/include/ginkgo/core/multigrid/fixed_coarsening.hpp b/include/ginkgo/core/multigrid/fixed_coarsening.hpp index b6ca62ba6ec..67f551413eb 100644 --- a/include/ginkgo/core/multigrid/fixed_coarsening.hpp +++ b/include/ginkgo/core/multigrid/fixed_coarsening.hpp @@ -80,33 +80,18 @@ class FixedCoarsening : public LinOp, public EnableMultigridLevel { GKO_ENABLE_BUILD_METHOD(Factory); protected: - void apply_impl(const LinOp* b, LinOp* x) const override - { - this->get_composition()->apply(b, x); - } + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override - { - this->get_composition()->apply(alpha, b, beta, x); - } + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; - explicit FixedCoarsening(std::shared_ptr exec) - : LinOp(std::move(exec)) - {} + explicit FixedCoarsening(std::shared_ptr exec); explicit FixedCoarsening(const Factory* factory, - std::shared_ptr system_matrix) - : LinOp(factory->get_executor(), system_matrix->get_size()), - EnableMultigridLevel(system_matrix), - parameters_{factory->get_parameters()}, - system_matrix_{system_matrix} - { - if (system_matrix_->get_size()[0] != 0) { - // generate on the existing matrix - this->generate(); - } - } + std::shared_ptr system_matrix); void generate(); diff --git a/include/ginkgo/core/multigrid/pgm.hpp b/include/ginkgo/core/multigrid/pgm.hpp index 741f8c3b0a7..bfc7e887d58 100644 --- a/include/ginkgo/core/multigrid/pgm.hpp +++ b/include/ginkgo/core/multigrid/pgm.hpp @@ -147,35 +147,18 @@ class Pgm : public LinOp, public EnableMultigridLevel { config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override - { - this->get_composition()->apply(b, x); - } + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override - { - this->get_composition()->apply(alpha, b, beta, x); - } + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; - explicit Pgm(std::shared_ptr exec) : LinOp(std::move(exec)) - {} + explicit Pgm(std::shared_ptr exec); explicit Pgm(const Factory* factory, - std::shared_ptr system_matrix) - : LinOp(factory->get_executor(), system_matrix->get_size()), - EnableMultigridLevel(system_matrix), - parameters_{factory->get_parameters()}, - system_matrix_{system_matrix}, - agg_(factory->get_executor(), system_matrix_->get_size()[0]) - { - GKO_ASSERT(parameters_.max_unassigned_ratio <= 1.0); - GKO_ASSERT(parameters_.max_unassigned_ratio >= 0.0); - if (system_matrix_->get_size()[0] != 0) { - // generate on the existed matrix - this->generate(); - } - } + std::shared_ptr system_matrix); void generate(); diff --git a/include/ginkgo/core/multigrid/rs.hpp b/include/ginkgo/core/multigrid/rs.hpp index 74373e6491d..f7048d3b5d8 100644 --- a/include/ginkgo/core/multigrid/rs.hpp +++ b/include/ginkgo/core/multigrid/rs.hpp @@ -87,13 +87,16 @@ class Rs : public LinOp, public EnableMultigridLevel { config::make_type_descriptor()); protected: - void apply_impl(const LinOp* b, LinOp* x) const override + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override { this->get_composition()->apply(b, x); } - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override { this->get_composition()->apply(alpha, b, beta, x); } From 36c8eed532b13a24b22b0615ffaa73d2fe893443 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:05:23 +0200 Subject: [PATCH 17/21] fix hierarchy change for benchmarks --- benchmark/blas/blas_common.hpp | 42 ++++++---- benchmark/preconditioner/preconditioner.cpp | 6 +- benchmark/solver/distributed/solver.cpp | 3 +- benchmark/solver/solver_common.hpp | 11 ++- benchmark/sparse_blas/operations.cpp | 6 +- benchmark/utils/cuda_linops.cpp | 33 +++++--- benchmark/utils/dpcpp_linops.dp.cpp | 3 +- benchmark/utils/general.hpp | 18 ++-- benchmark/utils/generator.hpp | 18 ++++ benchmark/utils/hip_linops.hip.cpp | 30 ++++--- benchmark/utils/loggers.hpp | 92 ++++++++++----------- benchmark/utils/overhead_linop.hpp | 9 +- 12 files changed, 153 insertions(+), 118 deletions(-) diff --git a/benchmark/blas/blas_common.hpp b/benchmark/blas/blas_common.hpp index 65a94aadcdf..34b0c6447f0 100644 --- a/benchmark/blas/blas_common.hpp +++ b/benchmark/blas/blas_common.hpp @@ -92,8 +92,8 @@ class CopyOperation : public BenchmarkOperation { } private: - std::unique_ptr in_; - std::unique_ptr out_; + std::unique_ptr in_; + std::unique_ptr out_; }; @@ -132,8 +132,8 @@ class AxpyOperation : public BenchmarkOperation { private: std::unique_ptr> alpha_; - std::unique_ptr x_; - std::unique_ptr y_; + std::unique_ptr x_; + std::unique_ptr y_; }; @@ -172,8 +172,8 @@ class SubScaledOperation : public BenchmarkOperation { private: std::unique_ptr> alpha_; - std::unique_ptr x_; - std::unique_ptr y_; + std::unique_ptr x_; + std::unique_ptr y_; }; @@ -208,7 +208,7 @@ class ScalOperation : public BenchmarkOperation { private: std::unique_ptr> alpha_; - std::unique_ptr y_; + std::unique_ptr y_; }; @@ -245,8 +245,8 @@ class DotOperation : public BenchmarkOperation { private: std::unique_ptr> alpha_; - std::unique_ptr x_; - std::unique_ptr y_; + std::unique_ptr x_; + std::unique_ptr y_; }; @@ -279,7 +279,7 @@ class NormOperation : public BenchmarkOperation { private: std::unique_ptr> alpha_; - std::unique_ptr y_; + std::unique_ptr y_; }; @@ -293,8 +293,11 @@ class ApplyOperation : public BenchmarkOperation { { // Since dense distributed matrices are not supported we can use // local_size == global_size - A_ = generator.create_multi_vector_strided(exec, gko::dim<2>{n, k}, - gko::dim<2>{n, k}, stride_A); + A_ = generator + .create_multi_vector_strided(exec, gko::dim<2>{n, k}, + gko::dim<2>{n, k}, stride_A) + ->as_const_dense_view() + ->clone(); B_ = generator.create_multi_vector_strided(exec, gko::dim<2>{k, m}, gko::dim<2>{k, m}, stride_B); C_ = generator.create_multi_vector_strided(exec, gko::dim<2>{n, m}, @@ -320,8 +323,8 @@ class ApplyOperation : public BenchmarkOperation { private: std::unique_ptr A_; - std::unique_ptr B_; - std::unique_ptr C_; + std::unique_ptr B_; + std::unique_ptr C_; }; @@ -336,8 +339,11 @@ class AdvancedApplyOperation : public BenchmarkOperation { { // Since dense distributed matrices are not supported we can use // local_size == global_size - A_ = generator.create_multi_vector_strided(exec, gko::dim<2>{n, k}, - gko::dim<2>{n, k}, stride_A); + A_ = generator + .create_multi_vector_strided(exec, gko::dim<2>{n, k}, + gko::dim<2>{n, k}, stride_A) + ->as_const_dense_view() + ->clone(); B_ = generator.create_multi_vector_strided(exec, gko::dim<2>{k, m}, gko::dim<2>{k, m}, stride_B); C_ = generator.create_multi_vector_strided(exec, gko::dim<2>{n, m}, @@ -372,8 +378,8 @@ class AdvancedApplyOperation : public BenchmarkOperation { std::unique_ptr> alpha_; std::unique_ptr> beta_; std::unique_ptr A_; - std::unique_ptr B_; - std::unique_ptr C_; + std::unique_ptr B_; + std::unique_ptr C_; }; diff --git a/benchmark/preconditioner/preconditioner.cpp b/benchmark/preconditioner/preconditioner.cpp index f07c7c6c430..8730d416107 100644 --- a/benchmark/preconditioner/preconditioner.cpp +++ b/benchmark/preconditioner/preconditioner.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -101,8 +101,8 @@ std::string encode_parameters(const char* precond_name) struct preconditioner_benchmark_state { - std::unique_ptr x; - std::unique_ptr b; + std::unique_ptr x; + std::unique_ptr b; std::shared_ptr system_matrix; }; diff --git a/benchmark/solver/distributed/solver.cpp b/benchmark/solver/distributed/solver.cpp index a064aa02c68..05063f054cf 100644 --- a/benchmark/solver/distributed/solver.cpp +++ b/benchmark/solver/distributed/solver.cpp @@ -31,7 +31,8 @@ struct Generator : public DistributedDefaultSystemGenerator { if (FLAGS_rhs_generation == "sinus") { gko::dim<2> vec_size{system_matrix->get_size()[0], FLAGS_nrhs}; gko::dim<2> local_vec_size{ - gko::detail::get_local(system_matrix)->get_size()[1], + gko::experimental::distributed::detail::get_local(system_matrix) + ->get_size()[1], FLAGS_nrhs}; return create_normalized_manufactured_rhs( exec, system_matrix, diff --git a/benchmark/solver/solver_common.hpp b/benchmark/solver/solver_common.hpp index e706c4d637a..b680b0e57b5 100644 --- a/benchmark/solver/solver_common.hpp +++ b/benchmark/solver/solver_common.hpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -321,7 +321,8 @@ struct SolverGenerator : DefaultSystemGenerator<> { } else { gko::dim<2> vec_size{system_matrix->get_size()[0], FLAGS_nrhs}; gko::dim<2> local_vec_size{ - gko::detail::get_local(system_matrix)->get_size()[1], + gko::experimental::distributed::detail::get_local(system_matrix) + ->get_size()[1], FLAGS_nrhs}; if (FLAGS_rhs_generation == "1") { return create_multi_vector(exec, vec_size, local_vec_size, @@ -346,7 +347,9 @@ struct SolverGenerator : DefaultSystemGenerator<> { { gko::dim<2> vec_size{system_matrix->get_size()[1], FLAGS_nrhs}; gko::dim<2> local_vec_size{ - gko::detail::get_local(system_matrix)->get_size()[1], FLAGS_nrhs}; + gko::experimental::distributed::detail::get_local(system_matrix) + ->get_size()[1], + FLAGS_nrhs}; if (FLAGS_initial_guess_generation == "0") { return create_multi_vector(exec, vec_size, local_vec_size, gko::zero()); @@ -432,7 +435,7 @@ struct SolverBenchmark : Benchmark> { solver_benchmark_state state; if (FLAGS_overhead) { - state.system_matrix = generator.initialize({1.0}, exec); + state.system_matrix = generator.generate_overhead_operator(exec); state.b = generator.initialize( {std::numeric_limits::quiet_NaN()}, exec); state.x = generator.initialize({0.0}, exec); diff --git a/benchmark/sparse_blas/operations.cpp b/benchmark/sparse_blas/operations.cpp index 2849c119620..9fa586e7335 100644 --- a/benchmark/sparse_blas/operations.cpp +++ b/benchmark/sparse_blas/operations.cpp @@ -252,8 +252,8 @@ class SpgeamOperation : public BenchmarkOperation { { auto ref = gko::ReferenceExecutor::create(); auto correct = gko::make_temporary_clone(ref, mtx2_); - gko::make_temporary_clone(ref, mtx_)->apply(scalar_, id_, scalar_, - correct.get()); + gko::make_temporary_clone(ref, mtx_)->scale_add(scalar_, scalar_, + correct.get()); return validate_result(correct.get(), mtx_out_); } @@ -661,7 +661,7 @@ class SymbolicCholeskyOperation : public BenchmarkOperation { {gko::one()}, exec); const auto id = gko::matrix::Identity::create(exec, mtx_->get_size()[0]); - lt_factor->apply(scalar, id, scalar, symm_result); + lt_factor->scale_add(scalar, scalar, symm_result); return std::make_pair( validate_symbolic_factorization(mtx_, symm_result.get()), 0.0); } diff --git a/benchmark/utils/cuda_linops.cpp b/benchmark/utils/cuda_linops.cpp index a9c2b5be8b5..3d5997dffc5 100644 --- a/benchmark/utils/cuda_linops.cpp +++ b/benchmark/utils/cuda_linops.cpp @@ -129,7 +129,8 @@ class CusparseCsrEx CusparseCsrEx& operator=(const CusparseCsrEx& other) = default; protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { auto dense_b = gko::as>(b); auto dense_x = gko::as>(x); @@ -162,9 +163,10 @@ class CusparseCsrEx // DEVICE for Ginkgo } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, - gko::LinOp* x) const override GKO_NOT_IMPLEMENTED; + void apply_impl( + const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override GKO_NOT_IMPLEMENTED; CusparseCsrEx(std::shared_ptr exec, const gko::dim<2>& size = gko::dim<2>{}) @@ -192,7 +194,8 @@ template void cusparse_generic_spmv(std::shared_ptr gpu_exec, const cusparseSpMatDescr_t mat, const gko::array& scalars, - const gko::LinOp* b, gko::LinOp* x, + const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x, cusparseOperation_t trans, cusparseSpMVAlg_t alg) { cudaDataType_t cu_value = gko::kernels::cuda::cuda_data_type(); @@ -295,15 +298,17 @@ class CusparseGenericCsr CusparseGenericCsr& operator=(const CusparseGenericCsr& other) = default; protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { cusparse_generic_spmv(this->get_gpu_exec(), mat_, scalars, b, x, trans_, Alg); } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, - gko::LinOp* x) const override GKO_NOT_IMPLEMENTED; + void apply_impl( + const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override GKO_NOT_IMPLEMENTED; CusparseGenericCsr(std::shared_ptr exec, const gko::dim<2>& size = gko::dim<2>{}) @@ -385,15 +390,17 @@ class CusparseGenericCoo CusparseGenericCoo& operator=(const CusparseGenericCoo& other) = default; protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { cusparse_generic_spmv(this->get_gpu_exec(), mat_, scalars, b, x, trans_, default_csr_alg); } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, - gko::LinOp* x) const override GKO_NOT_IMPLEMENTED; + void apply_impl( + const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override GKO_NOT_IMPLEMENTED; CusparseGenericCoo(std::shared_ptr exec, const gko::dim<2>& size = gko::dim<2>{}) diff --git a/benchmark/utils/dpcpp_linops.dp.cpp b/benchmark/utils/dpcpp_linops.dp.cpp index 5b4b65e0a6e..69e54b511b0 100644 --- a/benchmark/utils/dpcpp_linops.dp.cpp +++ b/benchmark/utils/dpcpp_linops.dp.cpp @@ -123,7 +123,8 @@ class OnemklCsr : public OnemklBase, } protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { auto dense_b = gko::as>(b); auto dense_x = gko::as>(x); diff --git a/benchmark/utils/general.hpp b/benchmark/utils/general.hpp index 3bcacae624e..dce6fd5e5bf 100644 --- a/benchmark/utils/general.hpp +++ b/benchmark/utils/general.hpp @@ -455,9 +455,8 @@ ValueType get_norm(const vec* norm) } -template -gko::remove_complex compute_norm2(const VectorType* b) +template +gko::remove_complex compute_norm2(const gko::AbstractMultiVector* b) { auto exec = b->get_executor(); auto b_norm = @@ -484,10 +483,10 @@ gko::remove_complex compute_direct_error(const gko::LinOp* solver, } -template +template gko::remove_complex compute_residual_norm( - const gko::LinOp* system_matrix, const VectorType* b, const VectorType* x) + const gko::LinOp* system_matrix, const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* x) { auto exec = system_matrix->get_executor(); auto one = gko::initialize>({1.0}, exec); @@ -519,10 +518,9 @@ gko::remove_complex compute_max_relative_norm2( clone(absolute_norm->get_executor()->get_master(), absolute_norm); rc_vtype max_relative_norm2 = 0; for (gko::size_type i = 0; i < host_answer_norm->get_size()[1]; i++) { - max_relative_norm2 = std::max( - gko::detail::get_local(host_absolute_norm.get())->at(0, i) / - gko::detail::get_local(host_answer_norm.get())->at(0, i), - max_relative_norm2); + max_relative_norm2 = + std::max(host_absolute_norm->at(0, i) / host_answer_norm->at(0, i), + max_relative_norm2); } return max_relative_norm2; } diff --git a/benchmark/utils/generator.hpp b/benchmark/utils/generator.hpp index 216e5b1b999..2de342e400c 100644 --- a/benchmark/utils/generator.hpp +++ b/benchmark/utils/generator.hpp @@ -116,6 +116,13 @@ struct DefaultSystemGenerator { local_size); } + static std::shared_ptr generate_overhead_operator( + std::shared_ptr exec) + { + return gko::matrix::Dense::create(std::move(exec), + gko::dim<2>{1, 1}); + } + static gko::dim<2> create_default_local_size(gko::dim<2> global_size) { return global_size; @@ -285,6 +292,17 @@ struct DistributedDefaultSystemGenerator { local_size); } + std::shared_ptr generate_overhead_operator( + std::shared_ptr exec) const + { + auto global_size = static_cast(comm.size()); + return generate_matrix_with_default_format( + std::move(exec), + gko::matrix_data{ + gko::dim<2>{global_size, global_size}}, + gko::dim<2>{1, 1}); + } + gko::dim<2> create_default_local_size(gko::dim<2> global_size) const { // This computes Partition::build_from_global_size_uniform manually, diff --git a/benchmark/utils/hip_linops.hip.cpp b/benchmark/utils/hip_linops.hip.cpp index a971e6e8952..45c798e0d6f 100644 --- a/benchmark/utils/hip_linops.hip.cpp +++ b/benchmark/utils/hip_linops.hip.cpp @@ -113,7 +113,8 @@ class HipsparseCsr } protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { auto dense_b = gko::as>(b); auto dense_x = gko::as>(x); @@ -130,9 +131,10 @@ class HipsparseCsr &scalars.get_const_data()[1], dx); } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, - gko::LinOp* x) const override GKO_NOT_IMPLEMENTED; + void apply_impl( + const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override GKO_NOT_IMPLEMENTED; HipsparseCsr(std::shared_ptr exec, const gko::dim<2>& size = gko::dim<2>{}) @@ -186,7 +188,8 @@ class HipsparseCsrmm } protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { auto dense_b = gko::as>(b); auto dense_x = gko::as>(x); @@ -204,9 +207,10 @@ class HipsparseCsrmm dense_x->get_size()[0]); } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, - gko::LinOp* x) const override GKO_NOT_IMPLEMENTED; + void apply_impl( + const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override GKO_NOT_IMPLEMENTED; HipsparseCsrmm(std::shared_ptr exec, const gko::dim<2>& size = gko::dim<2>{}) @@ -282,7 +286,8 @@ class HipsparseHybrid HipsparseHybrid& operator=(const HipsparseHybrid& other) = default; protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { auto dense_b = gko::as>(b); auto dense_x = gko::as>(x); @@ -296,9 +301,10 @@ class HipsparseHybrid &scalars.get_const_data()[1], dx); } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, - gko::LinOp* x) const override GKO_NOT_IMPLEMENTED; + void apply_impl( + const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override GKO_NOT_IMPLEMENTED; HipsparseHybrid(std::shared_ptr exec, const gko::dim<2>& size = gko::dim<2>{}) diff --git a/benchmark/utils/loggers.hpp b/benchmark/utils/loggers.hpp index ec6f65413c3..b30f42c0089 100644 --- a/benchmark/utils/loggers.hpp +++ b/benchmark/utils/loggers.hpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -139,15 +139,32 @@ template struct ResidualLogger : gko::log::Logger { using rc_vtype = gko::remove_complex; - void on_iteration_complete(const gko::LinOp*, - const gko::LinOp* right_hand_side, - const gko::LinOp* solution, - const gko::size_type&, - const gko::LinOp* residual, - const gko::LinOp* residual_norm, - const gko::LinOp* implicit_sq_residual_norm, - const gko::array* status, - bool all_stopped) const override + ResidualLogger(gko::ptr_param matrix, + gko::ptr_param b, + json& rec_res_norms, json& true_res_norms, + json& implicit_res_norms, json& timestamps) + : gko::log::Logger(gko::log::Logger::iteration_complete_mask), + matrix{matrix.get()}, + b{b.get()}, + start{std::chrono::steady_clock::now()}, + rec_res_norms{&rec_res_norms}, + true_res_norms{&true_res_norms}, + has_implicit_res_norm{}, + implicit_res_norms{&implicit_res_norms}, + timestamps{×tamps} + {} + + bool has_implicit_res_norms() const { return has_implicit_res_norm; } + +protected: + void on_iteration_complete( + const gko::LinOp*, const gko::AbstractMultiVector* right_hand_side, + const gko::AbstractMultiVector* solution, const gko::size_type&, + const gko::AbstractMultiVector* residual, + const gko::AbstractMultiVector* residual_norm, + const gko::AbstractMultiVector* implicit_sq_residual_norm, + const gko::array* status, + bool all_stopped) const override { timestamps->push_back(std::chrono::duration( std::chrono::steady_clock::now() - start) @@ -156,19 +173,11 @@ struct ResidualLogger : gko::log::Logger { rec_res_norms->push_back( get_norm(gko::as>(residual_norm))); } else { - gko::detail::vector_dispatch( - residual, [&](const auto v_residual) { - rec_res_norms->push_back(compute_norm2(v_residual)); - }); + rec_res_norms->push_back(compute_norm2(residual)); } if (solution) { - gko::detail::vector_dispatch< - ValueType>(solution, [&](auto v_solution) { - using concrete_type = - std::remove_pointer_t>; - true_res_norms->push_back(compute_residual_norm( - matrix, gko::as(b), v_solution)); - }); + true_res_norms->push_back( + compute_residual_norm(matrix, b, solution)); } else { true_res_norms->push_back(-1.0); } @@ -183,26 +192,9 @@ struct ResidualLogger : gko::log::Logger { } } - ResidualLogger(gko::ptr_param matrix, - gko::ptr_param b, json& rec_res_norms, - json& true_res_norms, json& implicit_res_norms, - json& timestamps) - : gko::log::Logger(gko::log::Logger::iteration_complete_mask), - matrix{matrix.get()}, - b{b.get()}, - start{std::chrono::steady_clock::now()}, - rec_res_norms{&rec_res_norms}, - true_res_norms{&true_res_norms}, - has_implicit_res_norm{}, - implicit_res_norms{&implicit_res_norms}, - timestamps{×tamps} - {} - - bool has_implicit_res_norms() const { return has_implicit_res_norm; } - private: const gko::LinOp* matrix; - const gko::LinOp* b; + const gko::AbstractMultiVector* b; std::chrono::steady_clock::time_point start; json* rec_res_norms; json* true_res_norms; @@ -214,23 +206,23 @@ struct ResidualLogger : gko::log::Logger { // Logs the number of iteration executed struct IterationLogger : gko::log::Logger { - void on_iteration_complete(const gko::LinOp*, const gko::LinOp*, - const gko::LinOp*, - const gko::size_type& num_iterations, - const gko::LinOp*, const gko::LinOp*, - const gko::LinOp*, - const gko::array*, - bool) const override - { - this->num_iters = num_iterations; - } - IterationLogger() : gko::log::Logger(gko::log::Logger::iteration_complete_mask) {} void write_data(json& output) { output["iterations"] = this->num_iters; } +protected: + void on_iteration_complete( + const gko::LinOp*, const gko::AbstractMultiVector*, + const gko::AbstractMultiVector*, const gko::size_type& num_iterations, + const gko::AbstractMultiVector*, const gko::AbstractMultiVector*, + const gko::AbstractMultiVector*, + const gko::array*, bool) const override + { + this->num_iters = num_iterations; + } + private: mutable gko::size_type num_iters{0}; }; diff --git a/benchmark/utils/overhead_linop.hpp b/benchmark/utils/overhead_linop.hpp index cd5fda505f6..912587648f3 100644 --- a/benchmark/utils/overhead_linop.hpp +++ b/benchmark/utils/overhead_linop.hpp @@ -81,7 +81,8 @@ class Overhead : public LinOp, public Preconditionable { GKO_ENABLE_BUILD_METHOD(Factory); protected: - void apply_impl(const LinOp* b, LinOp* x) const override + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override { using Vector = matrix::MultiVector; @@ -98,8 +99,10 @@ class Overhead : public LinOp, public Preconditionable { exec->run(overhead::make_operation4(dense_b, dense_x)); } - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override { auto dense_x = as>(x); From 016e718fa38e84a36ab69d4d1c64f6d71cd178aa Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:05:54 +0200 Subject: [PATCH 18/21] fix hierarchy change for extensions --- extensions/cuda/solver/cudss.cpp | 167 +++++++++--------- .../ginkgo/extensions/cuda/solver/cudss.hpp | 9 +- 2 files changed, 86 insertions(+), 90 deletions(-) diff --git a/extensions/cuda/solver/cudss.cpp b/extensions/cuda/solver/cudss.cpp index 171118d31d8..59390063904 100644 --- a/extensions/cuda/solver/cudss.cpp +++ b/extensions/cuda/solver/cudss.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include #include @@ -266,98 +266,91 @@ void Cudss::refactorize( template -void Cudss::apply_impl(const LinOp* b, LinOp* x) const +void Cudss::apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - using MultiVector = matrix::MultiVector; - const auto exec = this->get_executor(); - const auto nrhs = dense_b->get_size()[1]; - if (nrhs <= 1) { - const auto nrows = dense_b->get_size()[0]; - const auto nrows_i64 = static_cast(nrows); - - if (nrows == 0) { - return; - } - - const bool b_strided = - (dense_b->get_stride() != dense_b->get_size()[1]); - const bool x_strided = - (dense_x->get_stride() != dense_x->get_size()[1]); - - ValueType* b_data = - const_cast(dense_b->get_const_values()); - ValueType* x_data = dense_x->get_values(); - std::unique_ptr b_buf; - std::unique_ptr x_buf; - - if (b_strided) { - b_buf = MultiVector::create(exec, dim<2>{nrows, 1}); - auto mut_b = const_cast>*>(dense_b); - mut_b->create_submatrix(span{0, nrows}, span{0, 1}) - ->convert_to(b_buf); - b_data = b_buf->get_values(); - } - if (x_strided) { - x_buf = MultiVector::create(exec, dim<2>{nrows, 1}); - x_buf->fill(zero()); - x_data = x_buf->get_values(); - } - - cudssMatrix_t cudss_b = nullptr; - cudssMatrix_t cudss_x = nullptr; - - GKO_ASSERT_NO_CUDSS_ERRORS(cudssMatrixCreateDn( - &cudss_b, nrows_i64, 1, nrows_i64, b_data, - cuda_data_type(), CUDSS_LAYOUT_COL_MAJOR)); - GKO_ASSERT_NO_CUDSS_ERRORS(cudssMatrixCreateDn( - &cudss_x, nrows_i64, 1, nrows_i64, x_data, - cuda_data_type(), CUDSS_LAYOUT_COL_MAJOR)); - - GKO_ASSERT_NO_CUDSS_ERRORS(cudssExecute( - state_->handle, CUDSS_PHASE_SOLVE, state_->config, - state_->data, state_->A, cudss_x, cudss_b)); - - GKO_ASSERT_NO_CUDSS_ERRORS(cudssMatrixDestroy(cudss_b)); - GKO_ASSERT_NO_CUDSS_ERRORS(cudssMatrixDestroy(cudss_x)); - - if (x_strided) { - dense_x->create_submatrix(span{0, nrows}, span{0, 1}) - ->copy_from(x_buf); - } - } else { - const auto nrows = dense_b->get_size()[0]; - auto tmp_b = MultiVector::create(exec, dim<2>{nrows, 1}); - auto tmp_x = MultiVector::create(exec, dim<2>{nrows, 1}); - auto mut_b = const_cast>*>(dense_b); - for (size_type j = 0; j < nrhs; ++j) { - mut_b->create_submatrix(span{0, nrows}, span{j, j + 1}) - ->convert_to(tmp_b); - this->apply_impl(tmp_b.get(), tmp_x.get()); - dense_x->create_submatrix(span{0, nrows}, span{j, j + 1}) - ->copy_from(tmp_x); - } - } - }, - b, x); + auto dense_b = as>(b->as_precision(this)); + auto dense_x = as>(x->as_precision(this)); + using MultiVector = matrix::MultiVector; + const auto exec = this->get_executor(); + const auto nrhs = dense_b->get_size()[1]; + if (nrhs <= 1) { + const auto nrows = dense_b->get_size()[0]; + const auto nrows_i64 = static_cast(nrows); + + if (nrows == 0) { + return; + } + + const bool b_strided = + (dense_b->get_stride() != dense_b->get_size()[1]); + const bool x_strided = + (dense_x->get_stride() != dense_x->get_size()[1]); + + ValueType* b_data = const_cast(dense_b->get_const_values()); + ValueType* x_data = dense_x->get_values(); + std::unique_ptr b_buf; + std::unique_ptr x_buf; + + if (b_strided) { + b_buf = MultiVector::create(exec, dim<2>{nrows, 1}); + dense_b->create_subview(local_span{0, nrows}, local_span{0, 1}) + ->convert_to(b_buf); + b_data = b_buf->get_values(); + } + if (x_strided) { + x_buf = MultiVector::create(exec, dim<2>{nrows, 1}); + x_buf->fill(zero()); + x_data = x_buf->get_values(); + } + + cudssMatrix_t cudss_b = nullptr; + cudssMatrix_t cudss_x = nullptr; + + GKO_ASSERT_NO_CUDSS_ERRORS(cudssMatrixCreateDn( + &cudss_b, nrows_i64, 1, nrows_i64, b_data, + cuda_data_type(), CUDSS_LAYOUT_COL_MAJOR)); + GKO_ASSERT_NO_CUDSS_ERRORS(cudssMatrixCreateDn( + &cudss_x, nrows_i64, 1, nrows_i64, x_data, + cuda_data_type(), CUDSS_LAYOUT_COL_MAJOR)); + + GKO_ASSERT_NO_CUDSS_ERRORS( + cudssExecute(state_->handle, CUDSS_PHASE_SOLVE, state_->config, + state_->data, state_->A, cudss_x, cudss_b)); + + GKO_ASSERT_NO_CUDSS_ERRORS(cudssMatrixDestroy(cudss_b)); + GKO_ASSERT_NO_CUDSS_ERRORS(cudssMatrixDestroy(cudss_x)); + + if (x_strided) { + dense_x->create_subview(local_span{0, nrows}, local_span{0, 1}) + ->copy_from(x_buf); + } + } else { + const auto nrows = dense_b->get_size()[0]; + auto tmp_b = MultiVector::create(exec, dim<2>{nrows, 1}); + auto tmp_x = MultiVector::create(exec, dim<2>{nrows, 1}); + for (size_type j = 0; j < nrhs; ++j) { + dense_b->create_subview(local_span{0, nrows}, local_span{j, j + 1}) + ->convert_to(tmp_b); + this->apply_impl(tmp_b.get(), tmp_x.get()); + dense_x->create_subview(local_span{0, nrows}, local_span{j, j + 1}) + ->copy_from(tmp_x); + } + } } template -void Cudss::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const +void Cudss::apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const { - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - auto tmp = dense_x->clone(); - this->apply_impl(dense_b, tmp.get()); - dense_x->scale(dense_beta); - dense_x->add_scaled(dense_alpha, tmp); - }, - alpha, b, beta, x); + auto converted_x = x->as_precision(this); + auto tmp = converted_x->clone(); + this->apply_impl(b, tmp.get()); + converted_x->scale(beta); + converted_x->add_scaled(alpha, tmp); } diff --git a/include/ginkgo/extensions/cuda/solver/cudss.hpp b/include/ginkgo/extensions/cuda/solver/cudss.hpp index 4c7df999756..b5c281d387c 100644 --- a/include/ginkgo/extensions/cuda/solver/cudss.hpp +++ b/include/ginkgo/extensions/cuda/solver/cudss.hpp @@ -172,10 +172,13 @@ class Cudss : public LinOp { Cudss(const Factory* factory, std::shared_ptr system_matrix); - void apply_impl(const LinOp* b, LinOp* x) const override; + void apply_impl(const AbstractMultiVector* b, + AbstractMultiVector* x) const override; - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override; + void apply_impl(const AbstractMultiVector* alpha, + const AbstractMultiVector* b, + const AbstractMultiVector* beta, + AbstractMultiVector* x) const override; private: struct state; From 104a66de8db780784ed6eeba2f4c8ac378c51ff2 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Wed, 8 Apr 2026 12:32:37 +0200 Subject: [PATCH 19/21] remove precision_dispatch implementation Keep the header for alpha release and give a compiler error if used. The header will be removed in the full release. --- cmake/generate_ginkgo_hpp.cmake | 1 + .../ginkgo/core/base/precision_dispatch.hpp | 645 +----------------- 2 files changed, 3 insertions(+), 643 deletions(-) diff --git a/cmake/generate_ginkgo_hpp.cmake b/cmake/generate_ginkgo_hpp.cmake index 2d749678779..3a99549e418 100644 --- a/cmake/generate_ginkgo_hpp.cmake +++ b/cmake/generate_ginkgo_hpp.cmake @@ -14,6 +14,7 @@ function(ginkgo_generate_ginkgo_hpp) OR (file MATCHES "^ginkgo/core/stop/residual_norm_reduction.hpp$") OR (file MATCHES "^ginkgo/core/solver/.*_trs.hpp$") OR (file MATCHES "^ginkgo/core/preconditioner/utils.hpp$") + OR (file MATCHES "^ginkgo/core/base/precision_dispatch.hpp$") ) continue() endif() diff --git a/include/ginkgo/core/base/precision_dispatch.hpp b/include/ginkgo/core/base/precision_dispatch.hpp index 883882e696d..39df8b1031c 100644 --- a/include/ginkgo/core/base/precision_dispatch.hpp +++ b/include/ginkgo/core/base/precision_dispatch.hpp @@ -5,648 +5,7 @@ #ifndef GKO_PUBLIC_CORE_BASE_PRECISION_DISPATCH_HPP_ #define GKO_PUBLIC_CORE_BASE_PRECISION_DISPATCH_HPP_ - -#include -#include -#include -#include -#include - - -namespace gko { - - -/** - * Convert the given LinOp from matrix::MultiVector<...> to - * matrix::MultiVector. The conversion tries to convert the input - * LinOp to all MultiVector types with value type recursively reachable by - * next_precision_base<...> starting from the ValueType template parameter. This - * means that all real-to-real and complex-to-complex conversions for default - * precisions are being considered. If the input matrix is non-const, the - * contents of the modified converted object will be converted back to the input - * matrix when the returned object is destroyed. This may lead to a loss of - * precision! - * - * @param matrix the input matrix which is supposed to be converted. It is - * wrapped unchanged if it is already of type - * matrix::MultiVector, otherwise it will be converted - * to this type if possible. - * - * @returns a detail::temporary_conversion pointing to the (potentially - * converted) object. - * - * @throws NotSupported if the input matrix cannot be converted to - * matrix::MultiVector - * - * @tparam ValueType the value type into whose associated matrix::MultiVector - * type to convert the input LinOp. - */ -template -detail::temporary_conversion>::value, - const matrix::MultiVector, matrix::MultiVector>> -make_temporary_conversion(Ptr&& matrix) -{ - using Pointee = detail::pointee; - using MultiVector = matrix::MultiVector; - using NextMultiVector = matrix::MultiVector>; - using Next2MultiVector = matrix::MultiVector>; - using Next3MultiVector = matrix::MultiVector>; - using MaybeConstMultiVector = - std::conditional_t::value, const MultiVector, - MultiVector>; - auto result = - detail::temporary_conversion::template create< - NextMultiVector, Next2MultiVector, Next3MultiVector>(matrix); - if (!result) { - GKO_NOT_SUPPORTED(matrix); - } - return result; -} - - -/** - * Calls the given function with each given argument LinOp temporarily - * converted into matrix::MultiVector as parameters. - * - * @param fn the given function. It will be passed one (potentially const) - * matrix::MultiVector* parameter per parameter in the - * parameter pack `linops`. - * @param linops the given arguments to be converted and passed on to fn. - * - * @tparam ValueType the value type to use for the parameters of `fn`. - * @tparam Function the function pointer, lambda or other functor type to call - * with the converted arguments. - * @tparam Args the argument type list. - * */ -template -void precision_dispatch(Function fn, Args*... linops) -{ - fn(make_temporary_conversion(linops).get()...); -} - - -/** - * Calls the given function with the given LinOps temporarily converted to - * matrix::MultiVector* as parameters. - * If ValueType is real and both input vectors are complex, uses - * matrix::MultiVector::create_real_view() to convert them into real matrices - * after precision conversion. - * - * @see precision_dispatch() - */ -template -void precision_dispatch_real_complex(Function fn, const LinOp* in, LinOp* out) -{ - // do we need to convert complex MultiVector to real MultiVector? - // all real dense vectors are intra-convertible, thus by casting to - // ConvertibleTo>, we can check whether a LinOp is a - // real dense matrix: - auto complex_to_real = - !(is_complex() || - dynamic_cast>*>(in)); - if (complex_to_real) { - auto dense_in = make_temporary_conversion>(in); - auto dense_out = make_temporary_conversion>(out); - using MultiVector = matrix::MultiVector; - // These dynamic_casts are only needed to make the code compile - // If ValueType is complex, this branch will never be taken - // If ValueType is real, the cast is a no-op - fn(dynamic_cast(dense_in->create_real_view().get()), - dynamic_cast(dense_out->create_real_view().get())); - } else { - precision_dispatch(fn, in, out); - } -} - - -/** - * Calls the given function with the given LinOps temporarily converted to - * matrix::MultiVector* as parameters. - * If ValueType is real and both `in` and `out` are complex, uses - * matrix::MultiVector::create_real_view() to convert them into real matrices - * after precision conversion. - * - * @see precision_dispatch() - */ -template -void precision_dispatch_real_complex(Function fn, const LinOp* alpha, - const LinOp* in, LinOp* out) -{ - // do we need to convert complex MultiVector to real MultiVector? - // all real dense vectors are intra-convertible, thus by casting to - // ConvertibleTo>, we can check whether a LinOp is a - // real dense matrix: - auto complex_to_real = - !(is_complex() || - dynamic_cast>*>(in)); - if (complex_to_real) { - auto dense_in = make_temporary_conversion>(in); - auto dense_out = make_temporary_conversion>(out); - auto dense_alpha = make_temporary_conversion(alpha); - using MultiVector = matrix::MultiVector; - // These dynamic_casts are only needed to make the code compile - // If ValueType is complex, this branch will never be taken - // If ValueType is real, the cast is a no-op - fn(dense_alpha.get(), - dynamic_cast(dense_in->create_real_view().get()), - dynamic_cast(dense_out->create_real_view().get())); - } else { - precision_dispatch(fn, alpha, in, out); - } -} - - -/** - * Calls the given function with the given LinOps temporarily converted to - * matrix::MultiVector* as parameters. - * If ValueType is real and both `in` and `out` are complex, uses - * matrix::MultiVector::get_real_view() to convert them into real matrices after - * precision conversion. - * - * @see precision_dispatch() - */ -template -void precision_dispatch_real_complex(Function fn, const LinOp* alpha, - const LinOp* in, const LinOp* beta, - LinOp* out) -{ - // do we need to convert complex MultiVector to real MultiVector? - // all real dense vectors are intra-convertible, thus by casting to - // ConvertibleTo>, we can check whether a LinOp is a - // real dense matrix: - auto complex_to_real = - !(is_complex() || - dynamic_cast>*>(in)); - if (complex_to_real) { - auto dense_in = make_temporary_conversion>(in); - auto dense_out = make_temporary_conversion>(out); - auto dense_alpha = make_temporary_conversion(alpha); - auto dense_beta = make_temporary_conversion(beta); - using MultiVector = matrix::MultiVector; - // These dynamic_casts are only needed to make the code compile - // If ValueType is complex, this branch will never be taken - // If ValueType is real, the cast is a no-op - fn(dense_alpha.get(), - dynamic_cast(dense_in->create_real_view().get()), - dense_beta.get(), - dynamic_cast(dense_out->create_real_view().get())); - } else { - precision_dispatch(fn, alpha, in, beta, out); - } -} - - -/** - * Calls the given function with each given argument LinOp - * converted into matrix::MultiVector as parameters. - * - * If GINKGO_MIXED_PRECISION is defined, this means that the function will be - * called with its dynamic type as a static type, so the (templated/generic) - * function will be instantiated with all pairs of MultiVector and - * MultiVector> parameter types, and the - * appropriate overload will be called based on the dynamic type of the - * parameter. - * - * If GINKGO_MIXED_PRECISION is not defined, it will behave exactly like - * precision_dispatch. - * - * @param fn the given function. It will be called with one const and one - * non-const matrix::MultiVector<...> parameter based on the dynamic - * type of the inputs (GINKGO_MIXED_PRECISION) or of type - * matrix::MultiVector (no GINKGO_MIXED_PRECISION). - * @param in The first parameter to be cast (GINKGO_MIXED_PRECISION) or - * converted (no GINKGO_MIXED_PRECISION) and used to call `fn`. - * @param out The second parameter to be cast (GINKGO_MIXED_PRECISION) or - * converted (no GINKGO_MIXED_PRECISION) and used to call `fn`. - * - * @tparam ValueType the value type to use for the parameters of `fn` (no - * GINKGO_MIXED_PRECISION). With GINKGO_MIXED_PRECISION - * enabled, it only matters whether this type is complex or - * real. - * @tparam Function the function pointer, lambda or other functor type to call - * with the converted arguments. - */ -template -void mixed_precision_dispatch(Function fn, const LinOp* in, LinOp* out) -{ -#ifdef GINKGO_MIXED_PRECISION - using fst_type = matrix::MultiVector; - using snd_type = matrix::MultiVector>; - using trd_type = matrix::MultiVector>; - using fth_type = matrix::MultiVector>; - auto dispatch_out_vector = [&](auto dense_in) { - if (auto dense_out = dynamic_cast(out)) { - fn(dense_in, dense_out); - } else if (auto dense_out = dynamic_cast(out)) { - fn(dense_in, dense_out); - } else if (auto dense_out = dynamic_cast(out)) { - fn(dense_in, dense_out); - } else if (auto dense_out = dynamic_cast(out)) { - fn(dense_in, dense_out); - } else { - GKO_NOT_SUPPORTED(out); - } - }; - if (auto dense_in = dynamic_cast(in)) { - dispatch_out_vector(dense_in); - } else if (auto dense_in = dynamic_cast(in)) { - dispatch_out_vector(dense_in); - } else if (auto dense_in = dynamic_cast(in)) { - dispatch_out_vector(dense_in); - } else if (auto dense_in = dynamic_cast(in)) { - dispatch_out_vector(dense_in); - } else { - GKO_NOT_SUPPORTED(in); - } -#else - precision_dispatch(fn, in, out); -#endif -} - - -/** - * Calls the given function with the given LinOps cast to their dynamic type - * matrix::MultiVector* as parameters. - * If ValueType is real and both `in` and `out` are complex, uses - * matrix::MultiVector::get_real_view() to convert them into real matrices after - * precision conversion. - * - * @see mixed_precision_dispatch() - */ -template ()>* = nullptr> -void mixed_precision_dispatch_real_complex(Function fn, const LinOp* in, - LinOp* out) -{ -#ifdef GINKGO_MIXED_PRECISION - mixed_precision_dispatch(fn, in, out); -#else - precision_dispatch(fn, in, out); -#endif -} - - -template ()>* = nullptr> -void mixed_precision_dispatch_real_complex(Function fn, const LinOp* in, - LinOp* out) -{ -#ifdef GINKGO_MIXED_PRECISION - if (!dynamic_cast>*>(in)) { - mixed_precision_dispatch>( - [&fn](auto dense_in, auto dense_out) { - fn(dense_in->create_real_view().get(), - dense_out->create_real_view().get()); - }, - in, out); - } else { - mixed_precision_dispatch(fn, in, out); - } -#else - precision_dispatch_real_complex(fn, in, out); -#endif -} - - -namespace experimental { - - -#if GINKGO_BUILD_MPI - - -namespace distributed { - - -/** - * Convert the given LinOp from experimental::distributed::Vector<...> to - * experimental::distributed::Vector. The conversion tries to convert - * the input LinOp to all MultiVector types with value type recursively - * reachable by next_precision_base<...> starting from the ValueType template - * parameter. This means that all real-to-real and complex-to-complex - * conversions for default precisions are being considered. If the input matrix - * is non-const, the contents of the modified converted object will be converted - * back to the input matrix when the returned object is destroyed. This may lead - * to a loss of precision! - * - * @param matrix the input matrix which is supposed to be converted. It is - * wrapped unchanged if it is already of type - * experimental::distributed::Vector, otherwise it - * will be converted to this type if possible. - * - * @returns a detail::temporary_conversion pointing to the (potentially - * converted) object. - * - * @throws NotSupported if the input matrix cannot be converted to - * experimental::distributed::Vector - * - * @tparam ValueType the value type into whose associated Vector type to - * convert the input LinOp. - */ -template -gko::detail::temporary_conversion> make_temporary_conversion( - LinOp* matrix) -{ - auto result = - gko::detail::temporary_conversion>::template create< - Vector>, - Vector>, - Vector>>(matrix); - if (!result) { - GKO_NOT_SUPPORTED(matrix); - } - return result; -} - - -/** - * @copydoc make_temporary_conversion - */ -template -gko::detail::temporary_conversion> -make_temporary_conversion(const LinOp* matrix) -{ - auto result = gko::detail::temporary_conversion>:: - template create>, - Vector>, - Vector>>(matrix); - if (!result) { - GKO_NOT_SUPPORTED(matrix); - } - return result; -} - - -/** - * Calls the given function with each given argument LinOp temporarily - * converted into experimental::distributed::Vector as parameters. - * - * @param fn the given function. It will be passed one (potentially const) - * experimental::distributed::Vector* parameter per - * parameter in the parameter pack `linops`. - * @param linops the given arguments to be converted and passed on to fn. - * - * @tparam ValueType the value type to use for the parameters of `fn`. - * @tparam Function the function pointer, lambda or other functor type to call - * with the converted arguments. - * @tparam Args the argument type list. - */ -template -void precision_dispatch(Function fn, Args*... linops) -{ - fn(distributed::make_temporary_conversion(linops).get()...); -} - - -template -void mixed_precision_dispatch(Function fn, const LinOp* in, LinOp* out) -{ -#ifdef GINKGO_MIXED_PRECISION - using fst_type = Vector; - using snd_type = Vector>; - using trd_type = Vector>; - auto dispatch_out_vector = [&](auto vector_in) { - if (auto vector_out = dynamic_cast(out)) { - fn(vector_in, vector_out); - } else if (auto vector_out = dynamic_cast(out)) { - fn(vector_in, vector_out); - } else if (auto vector_out = dynamic_cast(out)) { - fn(vector_in, vector_out); - } else { - GKO_NOT_SUPPORTED(out); - } - }; - if (auto vector_in = dynamic_cast(in)) { - dispatch_out_vector(vector_in); - } else if (auto vector_in = dynamic_cast(in)) { - dispatch_out_vector(vector_in); - } else if (auto vector_in = dynamic_cast(in)) { - dispatch_out_vector(vector_in); - } else { - GKO_NOT_SUPPORTED(in); - } -#else - // avoid ambiguous - distributed::precision_dispatch(fn, in, out); -#endif -} - - -/** - * Calls the given function with the given LinOps temporarily converted to - * experimental::distributed::Vector* as parameters. - * If ValueType is real and both input vectors are complex, uses - * experimental::distributed::Vector::get_real_view() to convert them into real - * matrices after precision conversion. - * - * @see precision_dispatch() - */ -template -void precision_dispatch_real_complex(Function fn, const LinOp* in, LinOp* out) -{ - auto complex_to_real = !( - is_complex() || - dynamic_cast>*>( - in)); - if (complex_to_real) { - auto dense_in = - distributed::make_temporary_conversion>(in); - auto dense_out = - distributed::make_temporary_conversion>(out); - using Vector = experimental::distributed::Vector; - // These dynamic_casts are only needed to make the code compile - // If ValueType is complex, this branch will never be taken - // If ValueType is real, the cast is a no-op - fn(dynamic_cast(dense_in->create_real_view().get()), - dynamic_cast(dense_out->create_real_view().get())); - } else { - distributed::precision_dispatch(fn, in, out); - } -} - - -template -void mixed_precision_dispatch_real_complex(Function fn, const LinOp* in, - LinOp* out) -{ - auto complex_to_real = !( - is_complex() || - dynamic_cast>*>( - in)); - if (complex_to_real) { - distributed::mixed_precision_dispatch>( - [&fn](auto vector_in, auto vector_out) { - fn(vector_in->create_real_view().get(), - vector_out->create_real_view().get()); - }, - in, out); - } else { - distributed::mixed_precision_dispatch(fn, in, out); - } -} - - -/** - * @copydoc precision_dispatch_real_complex(Function, const LinOp*, LinOp*) - */ -template -void precision_dispatch_real_complex(Function fn, const LinOp* alpha, - const LinOp* in, LinOp* out) -{ - auto complex_to_real = !( - is_complex() || - dynamic_cast>*>( - in)); - if (complex_to_real) { - auto dense_in = - distributed::make_temporary_conversion>(in); - auto dense_out = - distributed::make_temporary_conversion>(out); - auto dense_alpha = gko::make_temporary_conversion(alpha); - using Vector = experimental::distributed::Vector; - // These dynamic_casts are only needed to make the code compile - // If ValueType is complex, this branch will never be taken - // If ValueType is real, the cast is a no-op - fn(dense_alpha.get(), - dynamic_cast(dense_in->create_real_view().get()), - dynamic_cast(dense_out->create_real_view().get())); - } else { - fn(gko::make_temporary_conversion(alpha).get(), - distributed::make_temporary_conversion(in).get(), - distributed::make_temporary_conversion(out).get()); - } -} - - -/** - * @copydoc precision_dispatch_real_complex(Function, const LinOp*, LinOp*) - */ -template -void precision_dispatch_real_complex(Function fn, const LinOp* alpha, - const LinOp* in, const LinOp* beta, - LinOp* out) -{ - auto complex_to_real = !( - is_complex() || - dynamic_cast>*>( - in)); - if (complex_to_real) { - auto dense_in = - distributed::make_temporary_conversion>(in); - auto dense_out = - distributed::make_temporary_conversion>(out); - auto dense_alpha = gko::make_temporary_conversion(alpha); - auto dense_beta = gko::make_temporary_conversion(beta); - using Vector = experimental::distributed::Vector; - // These dynamic_casts are only needed to make the code compile - // If ValueType is complex, this branch will never be taken - // If ValueType is real, the cast is a no-op - fn(dense_alpha.get(), - dynamic_cast(dense_in->create_real_view().get()), - dense_beta.get(), - dynamic_cast(dense_out->create_real_view().get())); - } else { - fn(gko::make_temporary_conversion(alpha).get(), - distributed::make_temporary_conversion(in).get(), - gko::make_temporary_conversion(beta).get(), - distributed::make_temporary_conversion(out).get()); - } -} - - -} // namespace distributed - - -/** - * Calls the given function with the given LinOps temporarily converted to - * either experimental::distributed::Vector* or - * matrix::MultiVector as parameters. The choice depends on the - * runtime type of `in` and `out` is assumed to fall into the same category. If - * ValueType is real and both input vectors are complex, uses - * experimental::distributed::Vector::get_real_view(), or - * matrix::MultiVector::get_real_view() to convert them into real matrices after - * precision conversion. - * - * @see precision_dispatch() - * @see distributed::precision_dispatch() - */ -template -void precision_dispatch_real_complex_distributed(Function fn, const LinOp* in, - LinOp* out) -{ - if (dynamic_cast(in)) { - experimental::distributed::precision_dispatch_real_complex( - fn, in, out); - } else { - gko::precision_dispatch_real_complex(fn, in, out); - } -} - - -/** - * @copydoc precision_dispatch_real_complex_distributed(Function, const LinOp*, - * LinOp*) - */ -template -void precision_dispatch_real_complex_distributed(Function fn, - const LinOp* alpha, - const LinOp* in, LinOp* out) -{ - if (dynamic_cast(in)) { - experimental::distributed::precision_dispatch_real_complex( - fn, alpha, in, out); - } else { - gko::precision_dispatch_real_complex(fn, alpha, in, out); - } -} - - -/** - * @copydoc precision_dispatch_real_complex_distributed(Function, const LinOp*, - * LinOp*) - */ -template -void precision_dispatch_real_complex_distributed(Function fn, - const LinOp* alpha, - const LinOp* in, - const LinOp* beta, LinOp* out) -{ - if (dynamic_cast(in)) { - experimental::distributed::precision_dispatch_real_complex( - fn, alpha, in, beta, out); - - } else { - gko::precision_dispatch_real_complex(fn, alpha, in, beta, - out); - } -} - - -#else - - -/** - * Calls the given function with the given LinOps temporarily converted to - * matrix::MultiVector as parameters. - * If ValueType is real and both input vectors are complex, uses - * experimental::distributed::Vector::get_real_view(), or - * matrix::MultiVector::get_real_view() to convert them into real matrices after - * precision conversion. - * - * @see precision_dispatch() - */ -template -void precision_dispatch_real_complex_distributed(Function fn, Args*... args) -{ - precision_dispatch_real_complex(fn, args...); -} - - -#endif - - -} // namespace experimental -} // namespace gko - +#error \ + "This header is deprecated. Use MultiVector::as_precision to convert a vector to the desired precision." #endif // GKO_PUBLIC_CORE_BASE_PRECISION_DISPATCH_HPP_ From 6448a51e7f30efaea842d5193370d3d2e5356c5a Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Mon, 17 Aug 2026 17:53:03 +0200 Subject: [PATCH 20/21] update tests --- benchmark/test/reference/blas.profile.stderr | 6 +- .../multi_vector_distributed.simple.stderr | 2 +- ..._vector_distributed_dcomplex.simple.stderr | 2 +- core/test/base/block_operator.cpp | 10 +- core/test/base/combination.cpp | 16 ++- core/test/base/composition.cpp | 10 +- core/test/base/lin_op.cpp | 116 ++++++++++------ core/test/base/mtx_io.cpp | 10 +- core/test/base/multivector.cpp | 43 ++---- core/test/base/perturbation.cpp | 20 ++- core/test/base/temporary_conversion.cpp | 27 ++-- core/test/log/logger.cpp | 30 ++-- core/test/log/profiler_hook.cpp | 59 ++++---- core/test/matrix/multivector.cpp | 55 +++----- core/test/mpi/distributed/CMakeLists.txt | 1 - core/test/mpi/distributed/helpers.cpp | 106 --------------- core/test/mpi/distributed/matrix.cpp | 10 +- .../test/mpi/distributed/solver/multigrid.cpp | 20 ++- core/test/preconditioner/isai.cpp | 10 +- core/test/solver/multigrid.cpp | 20 ++- core/test/solver/workspace.cpp | 128 ++++++++---------- core/test/stop/criterion.cpp | 22 +-- cuda/test/base/lin_op.cpp | 62 ++++++--- examples/custom-logger/custom-logger.cpp | 16 +-- .../custom-matrix-format.cpp | 9 +- examples/mixed-spmv/mixed-spmv.cpp | 2 +- .../performance-debugging.cpp | 9 +- reference/test/base/composition.cpp | 10 +- reference/test/base/perturbation.cpp | 48 ------- reference/test/factorization/ic_kernels.cpp | 10 +- reference/test/factorization/ilu_kernels.cpp | 10 +- .../test/factorization/par_ic_kernels.cpp | 10 +- .../test/factorization/par_ict_kernels.cpp | 10 +- .../test/factorization/par_ilu_kernels.cpp | 10 +- .../test/factorization/par_ilut_kernels.cpp | 10 +- reference/test/matrix/coo_kernels.cpp | 89 ------------ reference/test/matrix/csr_kernels.cpp | 58 -------- reference/test/matrix/dense_kernels.cpp | 24 ---- reference/test/matrix/diagonal_kernels.cpp | 62 --------- reference/test/matrix/ell_kernels.cpp | 26 ---- reference/test/matrix/hybrid_kernels.cpp | 59 -------- reference/test/matrix/identity.cpp | 32 ----- reference/test/matrix/multivector_kernels.cpp | 120 ++++++---------- reference/test/matrix/sellp_kernels.cpp | 55 -------- .../test/matrix/sparsity_csr_kernels.cpp | 36 ----- reference/test/preconditioner/ic.cpp | 42 ------ reference/test/preconditioner/ilu.cpp | 83 ------------ .../test/preconditioner/jacobi_kernels.cpp | 51 ------- reference/test/solver/bicg_kernels.cpp | 47 ------- reference/test/solver/bicgstab_kernels.cpp | 47 ------- reference/test/solver/cb_gmres_kernels.cpp | 51 ------- reference/test/solver/cg_kernels.cpp | 47 ------- reference/test/solver/cgs_kernels.cpp | 26 ---- reference/test/solver/chebyshev_kernels.cpp | 53 -------- reference/test/solver/fcg_kernels.cpp | 47 ------- reference/test/solver/gcr_kernels.cpp | 49 ------- reference/test/solver/gmres_kernels.cpp | 49 ------- reference/test/solver/idr_kernels.cpp | 47 ------- reference/test/solver/ir_kernels.cpp | 50 ------- reference/test/solver/lower_trs_kernels.cpp | 50 ------- reference/test/solver/multigrid_kernels.cpp | 47 +++++-- reference/test/solver/pipe_cg_kernels.cpp | 47 ------- reference/test/solver/upper_trs_kernels.cpp | 50 ------- test/matrix/csr_kernels2.cpp | 5 +- test/matrix/dense_kernels.cpp | 33 ----- test/matrix/matrix.cpp | 18 +-- test/matrix/multivector_kernels.cpp | 108 --------------- test/mpi/distributed/vector.cpp | 4 +- test/mpi/solver/solver.cpp | 10 +- test/solver/solver.cpp | 21 +-- 70 files changed, 553 insertions(+), 2029 deletions(-) delete mode 100644 core/test/mpi/distributed/helpers.cpp diff --git a/benchmark/test/reference/blas.profile.stderr b/benchmark/test/reference/blas.profile.stderr index 78f2ba9132f..cd9a34f7e46 100644 --- a/benchmark/test/reference/blas.profile.stderr +++ b/benchmark/test/reference/blas.profile.stderr @@ -2,8 +2,8 @@ Running on ReferenceExecutor Running with 0 warm iterations and 1 running iterations The random seed for right hand sides is 42 The operations are copy,axpy,scal -Running test case n = 100 -DEBUG: begin n = 100 +Running test case n = 100 +DEBUG: begin n = 100 Running blas: copy DEBUG: begin copy DEBUG: begin multivector::fill @@ -37,4 +37,4 @@ DEBUG: begin multivector::scale DEBUG: end multivector::scale DEBUG: end repetition DEBUG: end scal -DEBUG: end n = 100 +DEBUG: end n = 100 diff --git a/benchmark/test/reference/multi_vector_distributed.simple.stderr b/benchmark/test/reference/multi_vector_distributed.simple.stderr index ff505a3f1c9..ae83d316659 100644 --- a/benchmark/test/reference/multi_vector_distributed.simple.stderr +++ b/benchmark/test/reference/multi_vector_distributed.simple.stderr @@ -2,7 +2,7 @@ Running on ReferenceExecutor Running with 2 warm iterations and 10 running iterations The random seed for right hand sides is 42 The operations are copy,axpy,scal -Running test case n = 100 +Running test case n = 100 Running blas: copy Running blas: axpy Running blas: scal diff --git a/benchmark/test/reference/multi_vector_distributed_dcomplex.simple.stderr b/benchmark/test/reference/multi_vector_distributed_dcomplex.simple.stderr index ff505a3f1c9..ae83d316659 100644 --- a/benchmark/test/reference/multi_vector_distributed_dcomplex.simple.stderr +++ b/benchmark/test/reference/multi_vector_distributed_dcomplex.simple.stderr @@ -2,7 +2,7 @@ Running on ReferenceExecutor Running with 2 warm iterations and 10 running iterations The random seed for right hand sides is 42 The operations are copy,axpy,scal -Running test case n = 100 +Running test case n = 100 Running blas: copy Running blas: axpy Running blas: scal diff --git a/core/test/base/block_operator.cpp b/core/test/base/block_operator.cpp index 41613bf0af7..d9f056359a4 100644 --- a/core/test/base/block_operator.cpp +++ b/core/test/base/block_operator.cpp @@ -23,10 +23,14 @@ struct DummyOperator : public gko::LinOp { : gko::LinOp(std::move(exec), size) {} - void apply_impl(const LinOp* b, LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/core/test/base/combination.cpp b/core/test/base/combination.cpp index 29522edc704..4572a13d73a 100644 --- a/core/test/base/combination.cpp +++ b/core/test/base/combination.cpp @@ -19,10 +19,14 @@ struct DummyOperator : public gko::LinOp { : gko::LinOp(exec, gko::dim<2>{1, 1}) {} - void apply_impl(const LinOp* b, LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; @@ -36,13 +40,13 @@ class Combination : public ::testing::Test { : exec{gko::ReferenceExecutor::create()}, operators{std::make_shared(exec), std::make_shared(exec)}, - coefficients{std::make_shared(exec), - std::make_shared(exec)} + coefficients{MultiVector::create(exec, {1, 1}), + MultiVector::create(exec, {1, 1})} {} std::shared_ptr exec; std::vector> operators; - std::vector> coefficients; + std::vector> coefficients; }; TYPED_TEST_SUITE(Combination, gko::test::ValueTypes, TypenameNameGenerator); diff --git a/core/test/base/composition.cpp b/core/test/base/composition.cpp index 94cb4f8e050..1c62e69ac5f 100644 --- a/core/test/base/composition.cpp +++ b/core/test/base/composition.cpp @@ -21,10 +21,14 @@ struct DummyOperator : public gko::LinOp, : gko::LinOp(exec, size) {} - void apply_impl(const LinOp* b, LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/core/test/base/lin_op.cpp b/core/test/base/lin_op.cpp index c933f25c768..d9148cd363a 100644 --- a/core/test/base/lin_op.cpp +++ b/core/test/base/lin_op.cpp @@ -15,6 +15,7 @@ #include #include "core/test/utils.hpp" +#include "core/test/utils/dummy_vector.hpp" namespace { @@ -26,28 +27,32 @@ struct DummyLogger : gko::log::Logger { gko::log::Logger::linop_factory_events_mask) {} - void on_linop_apply_started(const gko::LinOp*, const gko::LinOp*, - const gko::LinOp*) const override + void on_linop_apply_started(const gko::LinOp*, + const gko::AbstractMultiVector*, + const gko::AbstractMultiVector*) const override { linop_apply_started++; } - void on_linop_apply_completed(const gko::LinOp*, const gko::LinOp*, - const gko::LinOp*) const override + void on_linop_apply_completed( + const gko::LinOp*, const gko::AbstractMultiVector*, + const gko::AbstractMultiVector*) const override { linop_apply_completed++; } - void on_linop_advanced_apply_started(const gko::LinOp*, const gko::LinOp*, - const gko::LinOp*, const gko::LinOp*, - const gko::LinOp*) const override + void on_linop_advanced_apply_started( + const gko::LinOp*, const gko::AbstractMultiVector*, + const gko::AbstractMultiVector*, const gko::AbstractMultiVector*, + const gko::AbstractMultiVector*) const override { linop_advanced_apply_started++; } - void on_linop_advanced_apply_completed(const gko::LinOp*, const gko::LinOp*, - const gko::LinOp*, const gko::LinOp*, - const gko::LinOp*) const override + void on_linop_advanced_apply_completed( + const gko::LinOp*, const gko::AbstractMultiVector*, + const gko::AbstractMultiVector*, const gko::AbstractMultiVector*, + const gko::AbstractMultiVector*) const override { linop_advanced_apply_completed++; } @@ -74,6 +79,17 @@ struct DummyLogger : gko::log::Logger { }; +class AccessVector : public AbstractDummyVector { +public: + using AbstractDummyVector::AbstractDummyVector; + using AbstractDummyVector::create; + + void access() const { last_access = this->get_executor(); } + + mutable std::shared_ptr last_access; +}; + + class DummyLinOp : public gko::LinOp, public gko::EnableCloneable, public gko::EnableCreateMethod { @@ -93,23 +109,26 @@ class DummyLinOp : public gko::LinOp, mutable std::shared_ptr last_beta_access; protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { this->access(); - static_cast(b)->access(); - static_cast(x)->access(); + dynamic_cast(b)->access(); + dynamic_cast(x)->access(); last_b_access = b->get_executor(); last_x_access = x->get_executor(); } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override { this->access(); - static_cast(alpha)->access(); - static_cast(b)->access(); - static_cast(beta)->access(); - static_cast(x)->access(); + dynamic_cast(alpha)->access(); + dynamic_cast(b)->access(); + dynamic_cast(beta)->access(); + dynamic_cast(x)->access(); last_alpha_access = alpha->get_executor(); last_b_access = b->get_executor(); last_beta_access = beta->get_executor(); @@ -148,10 +167,10 @@ class LinOpApply : public ::testing::Test { : ref{gko::ReferenceExecutor::create()}, ref2{gko::ReferenceExecutor::create()}, op{DummyLinOp::create(ref2, gko::dim<2>{3, 5})}, - alpha{DummyLinOp::create(ref, gko::dim<2>{1})}, - beta{DummyLinOp::create(ref, gko::dim<2>{1})}, - b{DummyLinOp::create(ref, gko::dim<2>{5, 4})}, - x{DummyLinOp::create(ref, gko::dim<2>{3, 4})}, + alpha{AccessVector::create(ref, gko::dim<2>{1})}, + beta{AccessVector::create(ref, gko::dim<2>{1})}, + b{AccessVector::create(ref, gko::dim<2>{5, 4})}, + x{AccessVector::create(ref, gko::dim<2>{3, 4})}, logger{std::make_shared()} { op->add_logger(logger); @@ -160,10 +179,10 @@ class LinOpApply : public ::testing::Test { std::shared_ptr ref; std::shared_ptr ref2; std::unique_ptr op; - std::unique_ptr alpha; - std::unique_ptr beta; - std::unique_ptr b; - std::unique_ptr x; + std::unique_ptr alpha; + std::unique_ptr beta; + std::unique_ptr b; + std::unique_ptr x; std::shared_ptr logger; }; @@ -186,7 +205,7 @@ TEST_F(LinOpApply, CallsExtendedImpl) TEST_F(LinOpApply, FailsOnWrongBSize) { - auto wrong = DummyLinOp::create(ref, gko::dim<2>{3, 4}); + auto wrong = AccessVector::create(ref, gko::dim<2>{3, 4}); ASSERT_THROW(op->apply(wrong, x), gko::DimensionMismatch); } @@ -194,7 +213,7 @@ TEST_F(LinOpApply, FailsOnWrongBSize) TEST_F(LinOpApply, FailsOnWrongSolutionRows) { - auto wrong = DummyLinOp::create(ref, gko::dim<2>{5, 4}); + auto wrong = AccessVector::create(ref, gko::dim<2>{5, 4}); ASSERT_THROW(op->apply(b, wrong), gko::DimensionMismatch); } @@ -202,7 +221,7 @@ TEST_F(LinOpApply, FailsOnWrongSolutionRows) TEST_F(LinOpApply, FailsOnWrongSolutionColumns) { - auto wrong = DummyLinOp::create(ref, gko::dim<2>{3, 5}); + auto wrong = AccessVector::create(ref, gko::dim<2>{3, 5}); ASSERT_THROW(op->apply(b, wrong), gko::DimensionMismatch); } @@ -210,7 +229,7 @@ TEST_F(LinOpApply, FailsOnWrongSolutionColumns) TEST_F(LinOpApply, ExtendedFailsOnWrongBSize) { - auto wrong = DummyLinOp::create(ref, gko::dim<2>{3, 4}); + auto wrong = AccessVector::create(ref, gko::dim<2>{3, 4}); ASSERT_THROW(op->apply(alpha, wrong, beta, x), gko::DimensionMismatch); } @@ -218,7 +237,7 @@ TEST_F(LinOpApply, ExtendedFailsOnWrongBSize) TEST_F(LinOpApply, ExtendedFailsOnWrongSolutionRows) { - auto wrong = DummyLinOp::create(ref, gko::dim<2>{5, 4}); + auto wrong = AccessVector::create(ref, gko::dim<2>{5, 4}); ASSERT_THROW(op->apply(alpha, b, beta, wrong), gko::DimensionMismatch); } @@ -226,7 +245,7 @@ TEST_F(LinOpApply, ExtendedFailsOnWrongSolutionRows) TEST_F(LinOpApply, ExtendedFailsOnWrongSolutionColumns) { - auto wrong = DummyLinOp::create(ref, gko::dim<2>{3, 5}); + auto wrong = AccessVector::create(ref, gko::dim<2>{3, 5}); ASSERT_THROW(op->apply(alpha, b, beta, wrong), gko::DimensionMismatch); } @@ -234,7 +253,7 @@ TEST_F(LinOpApply, ExtendedFailsOnWrongSolutionColumns) TEST_F(LinOpApply, ExtendedFailsOnWrongAlphaDimension) { - auto wrong = DummyLinOp::create(ref, gko::dim<2>{2, 5}); + auto wrong = AccessVector::create(ref, gko::dim<2>{2, 5}); ASSERT_THROW(op->apply(wrong, b, beta, x), gko::DimensionMismatch); } @@ -242,7 +261,7 @@ TEST_F(LinOpApply, ExtendedFailsOnWrongAlphaDimension) TEST_F(LinOpApply, ExtendedFailsOnWrongBetaDimension) { - auto wrong = DummyLinOp::create(ref, gko::dim<2>{2, 5}); + auto wrong = AccessVector::create(ref, gko::dim<2>{2, 5}); ASSERT_THROW(op->apply(alpha, b, wrong, x), gko::DimensionMismatch); } @@ -365,7 +384,7 @@ class DummyLinOpWithFactory : public gko::LinOp { DummyLinOpWithFactory(const Factory* factory, std::shared_ptr op) - : gko::LinOp(factory->get_executor()), + : gko::LinOp(factory->get_executor(), op->get_size()), parameters_{factory->get_parameters()}, op_{op} {} @@ -373,10 +392,14 @@ class DummyLinOpWithFactory : public gko::LinOp { std::shared_ptr op_; protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; @@ -443,8 +466,9 @@ TEST_F(LinOpFactory, WithLoggersWorksAndPropagates) auto before_logger = *logger; auto factory = DummyLinOpWithFactory<>::build().with_loggers(logger).on(ref); - auto op = factory->generate(DummyLinOp::create(ref, gko::dim<2>{3, 5})); - op->apply(op, op); + auto op = factory->generate(DummyLinOp::create(ref, gko::dim<2>{3, 3})); + auto vec = AccessVector::create(ref, gko::dim<2>{3, 3}); + op->apply(vec, vec); ASSERT_EQ(logger->linop_factory_generate_started, before_logger.linop_factory_generate_started + 1); @@ -501,10 +525,14 @@ class DummyLinOpWithType Type get_value() const { return value_; } protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} private: diff --git a/core/test/base/mtx_io.cpp b/core/test/base/mtx_io.cpp index dc8e4919d74..99990432cb1 100644 --- a/core/test/base/mtx_io.cpp +++ b/core/test/base/mtx_io.cpp @@ -960,10 +960,14 @@ class DummyLinOp void write(mat_data& data) const override { data = data_; } protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} explicit DummyLinOp(std::shared_ptr exec) diff --git a/core/test/base/multivector.cpp b/core/test/base/multivector.cpp index 21ee01926f9..cbac8cefe14 100644 --- a/core/test/base/multivector.cpp +++ b/core/test/base/multivector.cpp @@ -14,15 +14,6 @@ #include "core/test/utils/dummy_vector.hpp" -class ScalarVector : public DummyVector, public gko::matrix::MultiVector<> { -public: - ScalarVector(std::shared_ptr exec, - gko::dim<2> size = {}) - : DummyVector(exec, size), MultiVector<>(exec, size) - {} -}; - - class TrackingVector : public AbstractDummyVector { public: enum class function { @@ -276,12 +267,14 @@ class TrackingVector : public AbstractDummyVector { class AbstractMultiVector : public ::testing::Test { protected: + using ScalarVector = gko::matrix::MultiVector<>; + AbstractMultiVector() : exec(gko::ReferenceExecutor::create()), vector(TrackingVector::create(exec, {2, 3}, gko::precision::fp64)), other(DummyVector::create(exec, {2, 3}, gko::precision::fp64)), result(DummyVector::create(exec, {1, 3}, gko::precision::fp64)), - alpha(std::make_unique(exec, gko::dim<2>{1, 1})), + alpha(ScalarVector::create(exec, gko::dim<2>{1, 1})), tmp(exec) {} @@ -316,7 +309,6 @@ TEST_F(AbstractMultiVector, DispatchToImplementations) vector->get_imag(other); vector->fill(1.0); - auto alpha = std::make_shared(exec, gko::dim<2>{1, 1}); vector->scale(alpha); vector->inv_scale(alpha); vector->add_scaled(alpha, other); @@ -393,53 +385,40 @@ TEST_F(AbstractMultiVector, CreateWithTypeOfThrows) TEST_F(AbstractMultiVector, ComputeAbsoluteThrows) { auto wrong_vector = DummyVector::create(exec, {4, 3}); - auto wrong_precision = - DummyVector::create(exec, vector->get_size(), gko::precision::fp32); EXPECT_THROW(vector->compute_absolute(wrong_vector), gko::DimensionMismatch); - EXPECT_THROW(vector->compute_absolute(wrong_precision), - gko::PrecisionError); } TEST_F(AbstractMultiVector, MakeComplexThrows) { auto wrong_vector = DummyVector::create(exec, {4, 3}); - auto wrong_precision = - DummyVector::create(exec, vector->get_size(), gko::precision::fp32); EXPECT_THROW(vector->make_complex(wrong_vector), gko::DimensionMismatch); - EXPECT_THROW(vector->make_complex(wrong_precision), gko::PrecisionError); } TEST_F(AbstractMultiVector, GetRealThrows) { auto wrong_vector = DummyVector::create(exec, {4, 3}); - auto wrong_precision = - DummyVector::create(exec, vector->get_size(), gko::precision::fp32); EXPECT_THROW(vector->get_real(wrong_vector), gko::DimensionMismatch); - EXPECT_THROW(vector->get_real(wrong_precision), gko::PrecisionError); } TEST_F(AbstractMultiVector, GetImagThrows) { auto wrong_vector = DummyVector::create(exec, {4, 3}); - auto wrong_precision = - DummyVector::create(exec, vector->get_size(), gko::precision::fp32); EXPECT_THROW(vector->get_imag(wrong_vector), gko::DimensionMismatch); - EXPECT_THROW(vector->get_imag(wrong_precision), gko::PrecisionError); } TEST_F(AbstractMultiVector, ScaleThrows) { - auto wrong_cols = std::make_shared(exec, gko::dim<2>{1, 4}); - auto wrong_rows = std::make_shared(exec, gko::dim<2>{2, 1}); + auto wrong_cols = ScalarVector::create(exec, gko::dim<2>{1, 4}); + auto wrong_rows = ScalarVector::create(exec, gko::dim<2>{2, 1}); auto wrong_alpha = DummyVector::create(exec, gko::dim<2>{2, 1}); EXPECT_THROW(vector->scale(wrong_cols), gko::DimensionMismatch); @@ -450,8 +429,8 @@ TEST_F(AbstractMultiVector, ScaleThrows) TEST_F(AbstractMultiVector, InvScaleThrows) { - auto wrong_cols = std::make_shared(exec, gko::dim<2>{1, 4}); - auto wrong_rows = std::make_shared(exec, gko::dim<2>{2, 1}); + auto wrong_cols = ScalarVector::create(exec, gko::dim<2>{1, 4}); + auto wrong_rows = ScalarVector::create(exec, gko::dim<2>{2, 1}); auto wrong_alpha = DummyVector::create(exec, gko::dim<2>{2, 1}); EXPECT_THROW(vector->inv_scale(wrong_cols), gko::DimensionMismatch); @@ -462,8 +441,8 @@ TEST_F(AbstractMultiVector, InvScaleThrows) TEST_F(AbstractMultiVector, AddScaledThrows) { - auto wrong_cols = std::make_shared(exec, gko::dim<2>{1, 4}); - auto wrong_rows = std::make_shared(exec, gko::dim<2>{2, 1}); + auto wrong_cols = ScalarVector::create(exec, gko::dim<2>{1, 4}); + auto wrong_rows = ScalarVector::create(exec, gko::dim<2>{2, 1}); auto wrong_alpha = DummyVector::create(exec, gko::dim<2>{2, 1}); auto wrong_other = DummyVector::create(exec, gko::dim<2>{3, 2}); @@ -477,8 +456,8 @@ TEST_F(AbstractMultiVector, AddScaledThrows) TEST_F(AbstractMultiVector, SubScaledThrows) { - auto wrong_cols = std::make_shared(exec, gko::dim<2>{1, 4}); - auto wrong_rows = std::make_shared(exec, gko::dim<2>{2, 1}); + auto wrong_cols = ScalarVector::create(exec, gko::dim<2>{1, 4}); + auto wrong_rows = ScalarVector::create(exec, gko::dim<2>{2, 1}); auto wrong_alpha = DummyVector::create(exec, gko::dim<2>{2, 1}); auto wrong_other = DummyVector::create(exec, gko::dim<2>{3, 2}); diff --git a/core/test/base/perturbation.cpp b/core/test/base/perturbation.cpp index 30f174e806d..47fad5fcc47 100644 --- a/core/test/base/perturbation.cpp +++ b/core/test/base/perturbation.cpp @@ -19,10 +19,14 @@ struct DummyOperator : public gko::LinOp { : gko::LinOp(exec, size) {} - void apply_impl(const LinOp* b, LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; @@ -33,10 +37,14 @@ struct TransposableDummyOperator : public gko::LinOp, public gko::Transposable { : gko::LinOp(exec, size) {} - void apply_impl(const LinOp* b, LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} std::unique_ptr transpose() const override diff --git a/core/test/base/temporary_conversion.cpp b/core/test/base/temporary_conversion.cpp index b711e662602..8dd99fc638b 100644 --- a/core/test/base/temporary_conversion.cpp +++ b/core/test/base/temporary_conversion.cpp @@ -35,9 +35,13 @@ class LinOpA : public gko::LinOp { {} protected: - void apply_impl(const LinOp* b, LinOp* x) const override {} - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} private: @@ -213,12 +217,11 @@ TEST_F(TemporaryConversion, ConstCreateChainFromDerivedType) { auto tmp = gko::temporary_conversion::create(vec.get()); - auto tmp_linop = - gko::temporary_conversion::create_from_derived( - std::move(tmp)); + auto tmp_linop = gko::temporary_conversion< + const gko::AbstractMultiVector>::create_from_derived(std::move(tmp)); - EXPECT_EQ(typeid(tmp_linop.get()), typeid(const gko::LinOp*)); - EXPECT_NE(dynamic_cast(tmp_linop.get()), nullptr); + EXPECT_EQ(typeid(tmp_linop.get()), typeid(const gko::AbstractMultiVector*)); + ASSERT_NE(dynamic_cast(tmp_linop.get()), nullptr); EXPECT_EQ(dynamic_cast(tmp_linop.get())->get_const_values(), vec->get_values()); EXPECT_EQ(log->count, 0); @@ -242,11 +245,11 @@ TEST_F(TemporaryConversion, CreateChainFromDerivedType) { auto tmp = gko::temporary_conversion::create(vec.get()); - auto tmp_linop = gko::temporary_conversion::create_from_derived( - std::move(tmp)); + auto tmp_linop = gko::temporary_conversion< + gko::AbstractMultiVector>::create_from_derived(std::move(tmp)); - EXPECT_EQ(typeid(tmp_linop.get()), typeid(gko::LinOp*)); - EXPECT_NE(dynamic_cast(tmp_linop.get()), nullptr); + EXPECT_EQ(typeid(tmp_linop.get()), typeid(gko::AbstractMultiVector*)); + ASSERT_NE(dynamic_cast(tmp_linop.get()), nullptr); EXPECT_EQ(dynamic_cast(tmp_linop.get())->get_values(), vec->get_values()); EXPECT_EQ(log->count, 0); diff --git a/core/test/log/logger.cpp b/core/test/log/logger.cpp index bf408507200..4a18a34147b 100644 --- a/core/test/log/logger.cpp +++ b/core/test/log/logger.cpp @@ -142,8 +142,9 @@ struct DummyLogger : gko::log::Logger { void on_iteration_complete( const gko::LinOp* solver, const gko::size_type& num_iterations, - const gko::LinOp* residual, const gko::LinOp* solution = nullptr, - const gko::LinOp* residual_norm = nullptr) const override + const gko::AbstractMultiVector* residual, + const gko::AbstractMultiVector* solution = nullptr, + const gko::AbstractMultiVector* residual_norm = nullptr) const override { this->num_iterations_ = num_iterations; } @@ -247,24 +248,29 @@ struct DummyLoggerExtended : gko::log::Logger { void on_iteration_complete( const gko::LinOp* solver, const gko::size_type& num_iterations, - const gko::LinOp* residual, const gko::LinOp* solution = nullptr, - const gko::LinOp* residual_norm = nullptr) const override + const gko::AbstractMultiVector* residual, + const gko::AbstractMultiVector* solution = nullptr, + const gko::AbstractMultiVector* residual_norm = nullptr) const override { this->logged_deprecated_1 = true; } - void on_iteration_complete(const gko::LinOp* solver, - const gko::size_type& it, const gko::LinOp* r, - const gko::LinOp* x, const gko::LinOp* tau, - const gko::LinOp* implicit_tau_sq) const override + void on_iteration_complete( + const gko::LinOp* solver, const gko::size_type& it, + const gko::AbstractMultiVector* r, const gko::AbstractMultiVector* x, + const gko::AbstractMultiVector* tau, + const gko::AbstractMultiVector* implicit_tau_sq) const override { this->logged_deprecated_2 = true; } - void on_iteration_complete(const gko::LinOp* solver, const gko::LinOp* b, - const gko::LinOp* x, const gko::size_type& it, - const gko::LinOp* r, const gko::LinOp* tau, - const gko::LinOp* implicit_tau_sq, + void on_iteration_complete(const gko::LinOp* solver, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* x, + const gko::size_type& it, + const gko::AbstractMultiVector* r, + const gko::AbstractMultiVector* tau, + const gko::AbstractMultiVector* implicit_tau_sq, const gko::array* status, bool stopped) const override { diff --git a/core/test/log/profiler_hook.cpp b/core/test/log/profiler_hook.cpp index 7eb4a27f7e3..65ea43a77bb 100644 --- a/core/test/log/profiler_hook.cpp +++ b/core/test/log/profiler_hook.cpp @@ -17,6 +17,7 @@ #include #include "core/test/utils.hpp" +#include "core/test/utils/dummy_vector.hpp" #ifdef __APPLE__ // APPLE has different name for the dense> @@ -126,13 +127,16 @@ class DummyLinOp : public gko::LinOp, {} protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { this->get_executor()->run(DummyOperation{}); } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override { this->get_executor()->run(DummyOperation{}); } @@ -146,14 +150,14 @@ TEST(ProfilerHook, LogsPolymorphicObjectLinOp) "end:copy(obj,obj)", "begin:move(obj_copy,obj)", "end:move(obj_copy,obj)", - "begin:apply(obj * obj = obj)", + "begin:apply(obj * vec = vec)", "begin:op", "end:op", - "end:apply(obj * obj = obj)", - "begin:advanced_apply(DummyLinOp * obj * obj + DummyLinOp * obj)", + "end:apply(obj * vec = vec)", + "begin:advanced_apply(DummyVector * obj * vec + DummyVector * vec)", "begin:op", "end:op", - "end:advanced_apply(DummyLinOp * obj * obj + DummyLinOp * obj)", + "end:advanced_apply(DummyVector * obj * vec + DummyVector * vec)", "begin:generate(obj_factory)", "begin:op", "end:op", @@ -168,18 +172,20 @@ TEST(ProfilerHook, LogsPolymorphicObjectLinOp) auto linop = gko::share(DummyLinOp::create(exec)); auto linop_copy = linop->clone(); auto factory = DummyLinOp::build().on(exec); - auto scalar = DummyLinOp::create(exec, gko::dim<2>{1, 1}); + auto vector = DummyVector::create(exec); + auto scalar = DummyVector::create(exec, gko::dim<2>{1, 1}); logger->set_object_name(linop, "obj"); logger->set_object_name(linop_copy, "obj_copy"); logger->set_object_name(factory, "obj_factory"); + logger->set_object_name(vector, "vec"); exec->add_logger(logger); linop->copy_from(linop); // self move-assignment is potentially illegal for std::vector in pre-C++23, // this would causes the libstdc++ debug mode to abort, so use the copy linop->move_from(linop_copy); - linop->apply(linop, linop); - linop->apply(scalar, linop, scalar, linop); + linop->apply(vector, vector); + linop->apply(scalar, vector, scalar, vector); factory->generate(linop); logger->on_criterion_check_started(nullptr, 0, nullptr, nullptr, nullptr, 0, false); @@ -205,14 +211,14 @@ TEST(ProfilerHook, LogsPolymorphicObjectLinOpApplyWithType) "begin:op", "end:op", "end:advanced_apply(" + dense_complex_double + " * obj * gko::matrix::MultiVector + gko::matrix::MultiVector * " + dense_complex_double + ")", - "begin:apply(obj * obj = " + dense_complex_double + ")", + "begin:apply(obj * vec = " + dense_complex_double + ")", "begin:op", "end:op", - "end:apply(obj * obj = " + dense_complex_double + ")", - "begin:advanced_apply(" + dense_complex_double + " * obj * gko::matrix::MultiVector + DummyLinOp * obj)", + "end:apply(obj * vec = " + dense_complex_double + ")", + "begin:advanced_apply(" + dense_complex_double + " * obj * gko::matrix::MultiVector + DummyVector * vec)", "begin:op", "end:op", - "end:advanced_apply(" + dense_complex_double + " * obj * gko::matrix::MultiVector + DummyLinOp * obj)"}; + "end:advanced_apply(" + dense_complex_double + " * obj * gko::matrix::MultiVector + DummyVector * vec)"}; // clang-format on std::vector output; auto hooks = make_hooks(output); @@ -220,6 +226,7 @@ TEST(ProfilerHook, LogsPolymorphicObjectLinOpApplyWithType) auto logger = gko::log::ProfilerHook::create_custom( std::move(hooks.first), std::move(hooks.second)); auto linop = gko::share(DummyLinOp::create(exec)); + auto vector = gko::share(DummyVector::create(exec)); auto alpha = gko::share(gko::matrix::MultiVector>::create( exec, gko::dim<2>{1, 1})); @@ -228,14 +235,15 @@ TEST(ProfilerHook, LogsPolymorphicObjectLinOpApplyWithType) auto invec = gko::share(gko::matrix::MultiVector::create(exec)); auto outvec = gko::share( gko::matrix::MultiVector>::create(exec)); - auto scalar = DummyLinOp::create(exec, gko::dim<2>{1, 1}); + auto scalar = DummyVector::create(exec, gko::dim<2>{1, 1}); logger->set_object_name(linop, "obj"); + logger->set_object_name(vector, "vec"); exec->add_logger(logger); linop->apply(invec, outvec); linop->apply(alpha, invec, beta, outvec); - linop->apply(linop, outvec); - linop->apply(alpha, invec, scalar, linop); + linop->apply(vector, outvec); + linop->apply(alpha, invec, scalar, vector); normalize_type_names(output); ASSERT_EQ(output, expected); @@ -245,23 +253,25 @@ TEST(ProfilerHook, LogsPolymorphicObjectLinOpApplyWithType) TEST(ProfilerHook, LogsIteration) { using Vec = gko::matrix::MultiVector<>; + using Dense = gko::matrix::Dense<>; // clang-format off std::vector expected{ - "begin:apply(solver * mtx = mtx)", + "begin:apply(solver * vec = vec)", "begin:iteration", "end:iteration", - "end:apply(solver * mtx = mtx)", - "begin:advanced_apply(gko::matrix::MultiVector * solver * mtx + gko::matrix::MultiVector * mtx)", + "end:apply(solver * vec = vec)", + "begin:advanced_apply(gko::matrix::MultiVector * solver * vec + gko::matrix::MultiVector * vec)", "begin:iteration", "end:iteration", - "end:advanced_apply(gko::matrix::MultiVector * solver * mtx + gko::matrix::MultiVector * mtx)"}; + "end:advanced_apply(gko::matrix::MultiVector * solver * vec + gko::matrix::MultiVector * vec)"}; // clang-format on std::vector output; auto hooks = make_hooks(output); auto exec = gko::ReferenceExecutor::create(); auto logger = gko::log::ProfilerHook::create_custom( std::move(hooks.first), std::move(hooks.second)); - auto mtx = gko::share(Vec::create(exec)); + auto mtx = gko::share(Dense::create(exec)); + auto vector = gko::share(gko::matrix::MultiVector<>::create(exec)); auto alpha = gko::share(gko::initialize({1.0}, exec)); auto solver = gko::solver::Ir<>::build() @@ -270,10 +280,11 @@ TEST(ProfilerHook, LogsIteration) ->generate(mtx); logger->set_object_name(solver, "solver"); logger->set_object_name(mtx, "mtx"); + logger->set_object_name(vector, "vec"); solver->add_logger(logger); - solver->apply(mtx, mtx); - solver->apply(alpha, mtx, alpha, mtx); + solver->apply(vector, vector); + solver->apply(alpha, vector, alpha, vector); solver->remove_logger(logger); normalize_type_names(output); diff --git a/core/test/matrix/multivector.cpp b/core/test/matrix/multivector.cpp index 663597dd08d..75f16a2de8d 100644 --- a/core/test/matrix/multivector.cpp +++ b/core/test/matrix/multivector.cpp @@ -321,7 +321,8 @@ TYPED_TEST(MultiVector, CanCreateConstDeviceView) TYPED_TEST(MultiVector, CanCreateSubmatrix) { using value_type = typename TestFixture::value_type; - auto submtx = this->mtx->create_submatrix(gko::span{0, 1}, gko::span{1, 3}); + auto submtx = + this->mtx->create_subview(gko::local_span{0, 1}, gko::local_span{1, 3}); EXPECT_EQ(submtx->get_size(), gko::dim<2>(1, 2)); EXPECT_EQ(submtx->at(0, 0), value_type{2.0}); @@ -337,8 +338,8 @@ TYPED_TEST(MultiVector, CanCreateSubmatrixWithGlobalSize) { using value_type = typename TestFixture::value_type; auto submtx_orig = - this->mtx->create_submatrix(gko::span{0, 1}, gko::span{1, 3}); - auto submtx = this->mtx->create_submatrix( + this->mtx->create_subview(gko::local_span{0, 1}, gko::local_span{1, 3}); + auto submtx = this->mtx->create_subview( gko::local_span{0, 1}, gko::local_span{1, 3}, gko::dim<2>{1, 2}); GKO_ASSERT_MTX_NEAR(submtx_orig, submtx, 0.0); @@ -349,8 +350,8 @@ TYPED_TEST(MultiVector, CanCreateSubmatrixWithGlobalSize) TYPED_TEST(MultiVector, CreateSubmatrixWithGlobalSizeThrowsOnIncorrectSize) { EXPECT_THROW( - this->mtx->create_submatrix(gko::local_span{0, 1}, - gko::local_span{1, 3}, gko::dim<2>{1, 20}), + auto _ = this->mtx->create_subview( + gko::local_span{0, 1}, gko::local_span{1, 3}, gko::dim<2>{1, 20}), gko::DimensionMismatch); } @@ -358,34 +359,13 @@ TYPED_TEST(MultiVector, CreateSubmatrixWithGlobalSizeThrowsOnIncorrectSize) TYPED_TEST(MultiVector, CanCreateEmptySubmatrix) { using value_type = typename TestFixture::value_type; - auto submtx = this->mtx->create_submatrix(gko::span{0, 0}, gko::span{1, 1}); + auto submtx = + this->mtx->create_subview(gko::local_span{0, 0}, gko::local_span{1, 1}); EXPECT_EQ(submtx->get_size(), gko::dim<2>{}); } -TYPED_TEST(MultiVector, CanCreateSubmatrixWithStride) -{ - using value_type = typename TestFixture::value_type; - auto submtx = - this->mtx->create_submatrix(gko::span{0, 2}, gko::span{0, 2}, 3); - - EXPECT_EQ(submtx->get_size(), gko::dim<2>(2, 2)); - EXPECT_EQ(submtx->get_stride(), 3); - // The entry submtx->at(1, 0) points to the strided data of this->mtx - // which means that it is undefined. Thus it is skipped in the tests - EXPECT_EQ(submtx->at(0, 0), value_type{1.0}); - EXPECT_EQ(submtx->at(0, 1), value_type{2.0}); - EXPECT_EQ(submtx->at(1, 1), value_type{1.5}); - EXPECT_EQ(submtx->get_num_stored_elements(), 6); - EXPECT_LT(std::distance(this->mtx->get_values(), submtx->get_values()), - this->mtx->get_num_stored_elements()); - EXPECT_EQ(&submtx->at(0, 0), &this->mtx->at(0, 0)); - EXPECT_EQ(&submtx->at(0, 1), &this->mtx->at(0, 1)); - EXPECT_EQ(&submtx->at(1, 1), &this->mtx->at(1, 0)); -} - - TYPED_TEST(MultiVector, CanCreateRealView) { using value_type = typename TestFixture::value_type; @@ -485,25 +465,32 @@ class CustomMultiVector : public gko::matrix::MultiVector<>, } protected: - [[nodiscard]] std::unique_ptr clone_impl( - std::shared_ptr exec) const override + [[nodiscard]] std::unique_ptr clone_impl() const override { - return create(exec, this->get_size(), this->data_); + return create(this->get_executor(), this->get_size(), this->data_); } -private: explicit CustomMultiVector(std::shared_ptr exec, gko::dim<2> size = {}, int data = 0) : gko::matrix::MultiVector<>(std::move(exec), size), data_(data) {} - std::unique_ptr> create_view_of_impl() override + [[nodiscard]] std::unique_ptr> create_subview_impl( + gko::local_span rows, gko::local_span columns) override + { + auto view = create(this->get_executor(), {}, this->get_data()); + MultiVector<>::create_subview_impl(rows, columns)->move_to(view); + return view; + } + [[nodiscard]] std::unique_ptr> create_subview_impl( + gko::local_span rows, gko::local_span columns) const override { auto view = create(this->get_executor(), {}, this->get_data()); - gko::matrix::MultiVector<>::create_view_of_impl()->move_to(view); + MultiVector<>::create_subview_impl(rows, columns)->convert_to(view); return view; } +private: int data_; }; diff --git a/core/test/mpi/distributed/CMakeLists.txt b/core/test/mpi/distributed/CMakeLists.txt index 8e7cb5555e2..ea0d63f561e 100644 --- a/core/test/mpi/distributed/CMakeLists.txt +++ b/core/test/mpi/distributed/CMakeLists.txt @@ -1,4 +1,3 @@ -ginkgo_create_test(helpers MPI_SIZE 1 LABELS distributed) ginkgo_create_test(matrix MPI_SIZE 3 LABELS distributed) ginkgo_create_test(collective_communicator MPI_SIZE 6 LABELS distributed) ginkgo_create_test(row_gatherer MPI_SIZE 6 LABELS distributed) diff --git a/core/test/mpi/distributed/helpers.cpp b/core/test/mpi/distributed/helpers.cpp deleted file mode 100644 index 6cb032257c6..00000000000 --- a/core/test/mpi/distributed/helpers.cpp +++ /dev/null @@ -1,106 +0,0 @@ -// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors -// -// SPDX-License-Identifier: BSD-3-Clause - -#include "core/distributed/helpers.hpp" - -#include - -#include - -#include "core/test/utils.hpp" - - -int run_function(gko::experimental::distributed::Vector<>*) { return 1; } - -int run_function(const gko::experimental::distributed::Vector<>*) { return 2; } - -int run_function(gko::matrix::MultiVector<>*) { return 3; } - -int run_function(const gko::matrix::MultiVector<>*) { return 4; } - - -class RunVector : public ::testing::Test { -public: - std::shared_ptr exec = - gko::ReferenceExecutor::create(); -}; - - -TEST_F(RunVector, PicksDistributedVectorCorrectly) -{ - std::unique_ptr dist_vector = - gko::experimental::distributed::Vector<>::create(exec, MPI_COMM_WORLD); - int result; - - gko::detail::vector_dispatch( - dist_vector.get(), [&](auto* dense) { result = run_function(dense); }); - - ASSERT_EQ(result, - run_function(gko::as>( - dist_vector.get()))); -} - - -TEST_F(RunVector, PicksConstDistributedVectorCorrectly) -{ - std::unique_ptr const_dist_vector = - gko::experimental::distributed::Vector<>::create(exec, MPI_COMM_WORLD); - int result; - - gko::detail::vector_dispatch( - const_dist_vector.get(), - [&](auto* dense) { result = run_function(dense); }); - - ASSERT_EQ( - result, - run_function(gko::as>( - const_dist_vector.get()))); -} - - -TEST_F(RunVector, PicksMultiVectorVectorCorrectly) -{ - std::unique_ptr dense_vector = - gko::matrix::MultiVector<>::create(exec); - int result; - - gko::detail::vector_dispatch( - dense_vector.get(), [&](auto* dense) { result = run_function(dense); }); - - ASSERT_EQ( - result, - run_function(gko::as>(dense_vector.get()))); -} - - -TEST_F(RunVector, PicksConstMultiVectorVectorCorrectly) -{ - std::unique_ptr const_dense_vector = - gko::matrix::MultiVector<>::create(exec); - int result; - - gko::detail::vector_dispatch( - const_dense_vector.get(), - [&](auto* dense) { result = run_function(dense); }); - - ASSERT_EQ(result, run_function(gko::as>( - const_dense_vector.get()))); -} - -TEST_F(RunVector, ThrowsIfWrongType) -{ - std::unique_ptr csr = gko::matrix::Csr<>::create(exec); - - ASSERT_THROW( - gko::detail::vector_dispatch(csr.get(), [&](auto* dense) {}), - gko::NotSupported); -} - - -TEST_F(RunVector, ThrowsIfNullptr) -{ - ASSERT_THROW(gko::detail::vector_dispatch( - static_cast(nullptr), [&](auto* dense) {}), - gko::NotSupported); -} diff --git a/core/test/mpi/distributed/matrix.cpp b/core/test/mpi/distributed/matrix.cpp index 8adf3b5c4a1..183f62c538f 100644 --- a/core/test/mpi/distributed/matrix.cpp +++ b/core/test/mpi/distributed/matrix.cpp @@ -43,10 +43,14 @@ class CustomLinOp {} protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/core/test/mpi/distributed/solver/multigrid.cpp b/core/test/mpi/distributed/solver/multigrid.cpp index 61f534d82db..97ea2c05e1a 100644 --- a/core/test/mpi/distributed/solver/multigrid.cpp +++ b/core/test/mpi/distributed/solver/multigrid.cpp @@ -36,10 +36,14 @@ class DummyLinOp : public gko::LinOp, {} protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; @@ -92,10 +96,14 @@ class DummyMultigridLevelWithFactory std::shared_ptr restrict_; std::shared_ptr prolong_; - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/core/test/preconditioner/isai.cpp b/core/test/preconditioner/isai.cpp index 2e2a40ed226..2a11091d3e6 100644 --- a/core/test/preconditioner/isai.cpp +++ b/core/test/preconditioner/isai.cpp @@ -24,10 +24,14 @@ struct DummyOperator : public gko::LinOp, : gko::LinOp(exec, size) {} - void apply_impl(const LinOp* b, LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, - LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/core/test/solver/multigrid.cpp b/core/test/solver/multigrid.cpp index 56e09d075ae..b2c65f148be 100644 --- a/core/test/solver/multigrid.cpp +++ b/core/test/solver/multigrid.cpp @@ -31,10 +31,14 @@ class DummyLinOp : public gko::LinOp, {} protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; @@ -84,10 +88,14 @@ class DummyLinOpWithFactory gko::size_type n_; protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/core/test/solver/workspace.cpp b/core/test/solver/workspace.cpp index 947cd58e8c6..155c0d76a43 100644 --- a/core/test/solver/workspace.cpp +++ b/core/test/solver/workspace.cpp @@ -11,55 +11,51 @@ #include #include "core/test/utils.hpp" +#include "core/test/utils/dummy_vector.hpp" -class DummyLinOp : public gko::LinOp, - public gko::EnableCreateMethod { +class Vector1 : public AbstractDummyVector, + public gko::EnableCreateMethod { public: - DummyLinOp(std::shared_ptr exec, gko::dim<2> size = {}, - gko::size_type stride = 0) - : LinOp(exec, size), stride_{stride} + using EnableCreateMethod::create; + + Vector1(std::shared_ptr exec, gko::dim<2> size = {}, + gko::size_type stride = 0) + : AbstractDummyVector(exec, size), stride_{stride} {} gko::size_type get_stride() { return stride_; } protected: gko::size_type stride_; - - void apply_impl(const gko::LinOp*, gko::LinOp*) const override {} - - void apply_impl(const gko::LinOp*, const gko::LinOp*, const gko::LinOp*, - gko::LinOp*) const override - {} }; -class DummyLinOp2 : public gko::LinOp, - public gko::EnableCreateMethod { +class Vector2 : public AbstractDummyVector, + public gko::EnableCreateMethod { public: - DummyLinOp2(std::shared_ptr exec, - gko::dim<2> size = {}, gko::size_type stride = 0) - : LinOp(exec, size), stride_{stride} + using EnableCreateMethod::create; + + Vector2(std::shared_ptr exec, gko::dim<2> size = {}, + gko::size_type stride = 0) + : AbstractDummyVector(exec, size), stride_{stride} {} gko::size_type get_stride() { return stride_; } protected: gko::size_type stride_; - - void apply_impl(const gko::LinOp*, gko::LinOp*) const override {} - - void apply_impl(const gko::LinOp*, const gko::LinOp*, const gko::LinOp*, - gko::LinOp*) const override - {} }; -class DerivedDummyLinOp : public DummyLinOp { +class DerivedVector : public Vector1, + public gko::EnableCreateMethod { public: - DerivedDummyLinOp(std::shared_ptr exec, - gko::dim<2> size = {}, gko::size_type stride = 0) - : DummyLinOp(exec, size, stride) + using EnableCreateMethod::create; + + DerivedVector(std::shared_ptr exec, + gko::dim<2> size = {}, gko::size_type stride = 0) + : Vector1(exec, size, stride) {} }; @@ -198,23 +194,23 @@ TEST_F(Workspace, CanCreateOperators) const gko::size_type stride1 = 3; const gko::size_type stride2 = 6; - auto op1 = ws.template create_or_get_op( - 1, [&] { return DummyLinOp::create(exec, size1, stride1); }, - typeid(DummyLinOp), size1, stride1); - auto op2 = ws.template create_or_get_op( - 0, [&] { return DummyLinOp2::create(exec, size2, stride2); }, - typeid(DummyLinOp2), size2, stride2); + auto op1 = ws.create_or_get_vector( + 1, [&] { return Vector1::create(exec, size1, stride1); }, + typeid(Vector1), size1); + auto op2 = ws.create_or_get_vector( + 0, [&] { return Vector2::create(exec, size2, stride2); }, + typeid(Vector2), size2); ASSERT_EQ(op1->get_executor(), exec); ASSERT_EQ(op2->get_executor(), exec); ASSERT_EQ(op1->get_size(), size1); ASSERT_EQ(op2->get_size(), size2); - ASSERT_EQ(op1->get_stride(), stride1); - ASSERT_EQ(op2->get_stride(), stride2); - GKO_ASSERT_DYNAMIC_TYPE(op1, DummyLinOp); - GKO_ASSERT_DYNAMIC_TYPE(op2, DummyLinOp2); - ASSERT_EQ(op1, ws.get_op(1)); - ASSERT_EQ(op2, ws.get_op(0)); + GKO_ASSERT_DYNAMIC_TYPE(op1, Vector1); + GKO_ASSERT_DYNAMIC_TYPE(op2, Vector2); + ASSERT_EQ(gko::as(op1)->get_stride(), stride1); + ASSERT_EQ(gko::as(op2)->get_stride(), stride2); + ASSERT_EQ(op1, ws.get_vector(1)); + ASSERT_EQ(op2, ws.get_vector(0)); } @@ -222,11 +218,11 @@ TEST_F(Workspace, CanReuseOperators) { gko::solver::detail::workspace ws{exec}; ws.set_size(1, 0); - auto op1 = ws.template create_or_get_op( - 0, [&] { return DummyLinOp::create(exec); }, typeid(DummyLinOp), {}, 0); + auto op1 = ws.create_or_get_vector(0, [&] { return Vector1::create(exec); }, + typeid(Vector1), {}); - auto op1_reuse = ws.template create_or_get_op( - 0, [&] { return DummyLinOp::create(exec); }, typeid(DummyLinOp), {}, 0); + auto op1_reuse = ws.create_or_get_vector( + 0, [&] { return Vector1::create(exec); }, typeid(Vector1), {}); ASSERT_EQ(op1, op1_reuse); } @@ -236,14 +232,14 @@ TEST_F(Workspace, ChecksExactOperatorType) { gko::solver::detail::workspace ws{exec}; ws.set_size(1, 0); - ws.template create_or_get_op( - 0, [&] { return DummyLinOp::create(exec); }, typeid(DummyLinOp), {}, 0); + ws.create_or_get_vector(0, [&] { return Vector1::create(exec); }, + typeid(Vector1), {}); - auto op1 = ws.template create_or_get_op( - 0, [&] { return std::make_unique(exec); }, - typeid(DerivedDummyLinOp), {}, 0); + auto op1 = ws.create_or_get_vector( + 0, [&] { return std::make_unique(exec); }, + typeid(DerivedVector), {}); - GKO_ASSERT_DYNAMIC_TYPE(op1, DerivedDummyLinOp); + GKO_ASSERT_DYNAMIC_TYPE(op1, DerivedVector); } @@ -252,42 +248,26 @@ TEST_F(Workspace, ChecksOperatorSize) gko::solver::detail::workspace ws{exec}; ws.set_size(1, 0); const gko::dim<2> size{1, 2}; - ws.template create_or_get_op( - 0, [&] { return DummyLinOp::create(exec); }, typeid(DummyLinOp), {}, 0); + ws.create_or_get_vector(0, [&] { return Vector1::create(exec); }, + typeid(Vector1), {}); - auto op1 = ws.template create_or_get_op( - 0, [&] { return DummyLinOp::create(exec, size); }, typeid(DummyLinOp), - size, 0); + auto op1 = ws.create_or_get_vector( + 0, [&] { return Vector1::create(exec, size); }, typeid(Vector1), size); ASSERT_EQ(op1->get_size(), size); } -TEST_F(Workspace, ChecksOperatorStride) -{ - gko::solver::detail::workspace ws{exec}; - ws.set_size(1, 0); - ws.template create_or_get_op( - 0, [&] { return DummyLinOp::create(exec); }, typeid(DummyLinOp), {}, 0); - - auto op1 = ws.template create_or_get_op( - 0, [&] { return DummyLinOp::create(exec, gko::dim<2>{}, 1); }, - typeid(DummyLinOp), {}, 1); - - ASSERT_EQ(op1->get_stride(), 1); -} - - TEST_F(Workspace, ClearResetsOperators) { gko::solver::detail::workspace ws{exec}; ws.set_size(1, 0); - auto op1 = ws.template create_or_get_op( - 0, [&] { return DummyLinOp::create(exec); }, typeid(DummyLinOp), {}, 0); + auto op1 = ws.create_or_get_vector(0, [&] { return Vector1::create(exec); }, + typeid(Vector1), {}); ws.clear(); - ASSERT_EQ(ws.get_op(0), nullptr); + ASSERT_EQ(ws.get_vector(0), nullptr); } @@ -295,10 +275,10 @@ TEST_F(Workspace, MoveResetsOperators) { gko::solver::detail::workspace ws{exec}; ws.set_size(1, 0); - auto op1 = ws.template create_or_get_op( - 0, [&] { return DummyLinOp::create(exec); }, typeid(DummyLinOp), {}, 0); + auto op1 = ws.create_or_get_vector(0, [&] { return Vector1::create(exec); }, + typeid(Vector1), {}); gko::solver::detail::workspace ws2{std::move(ws)}; - ASSERT_EQ(ws.get_op(0), nullptr); + ASSERT_EQ(ws.get_vector(0), nullptr); } diff --git a/core/test/stop/criterion.cpp b/core/test/stop/criterion.cpp index 9c7c04ab12e..c60c59ac9b6 100644 --- a/core/test/stop/criterion.cpp +++ b/core/test/stop/criterion.cpp @@ -13,22 +13,24 @@ namespace { struct DummyLogger : public gko::log::Logger { DummyLogger() : gko::log::Logger(gko::log::Logger::criterion_events_mask) {} - void on_criterion_check_started(const gko::stop::Criterion* criterion, - const gko::size_type& num_iterations, - const gko::LinOp* residual, - const gko::LinOp* residual_norm, - const gko::LinOp* solution, - const gko::uint8& stopping_id, - const bool& set_finalized) const override + void on_criterion_check_started( + const gko::stop::Criterion* criterion, + const gko::size_type& num_iterations, + const gko::AbstractMultiVector* residual, + const gko::AbstractMultiVector* residual_norm, + const gko::AbstractMultiVector* solution, const gko::uint8& stopping_id, + const bool& set_finalized) const override { criterion_check_started++; } void on_criterion_check_completed( const gko::stop::Criterion* criterion, - const gko::size_type& num_iterations, const gko::LinOp* residual, - const gko::LinOp* residual_norm, const gko::LinOp* solution, - const gko::uint8& stopping_id, const bool& set_finalized, + const gko::size_type& num_iterations, + const gko::AbstractMultiVector* residual, + const gko::AbstractMultiVector* residual_norm, + const gko::AbstractMultiVector* solution, const gko::uint8& stopping_id, + const bool& set_finalized, const gko::array* status, const bool& one_changed, const bool& all_converged) const override { diff --git a/cuda/test/base/lin_op.cpp b/cuda/test/base/lin_op.cpp index a662f386c0f..4ab9c78ce60 100644 --- a/cuda/test/base/lin_op.cpp +++ b/cuda/test/base/lin_op.cpp @@ -4,12 +4,37 @@ #include +#include "core/test/utils/dummy_vector.hpp" #include "cuda/test/utils.hpp" namespace { +class AccessVector : public AbstractDummyVector { +public: + using AbstractDummyVector::AbstractDummyVector; + using AbstractDummyVector::create; + + void access() const { last_access = this->get_executor(); } + + mutable std::shared_ptr last_access; + +protected: + [[nodiscard]] std::unique_ptr clone_impl( + std::shared_ptr exec) const override + { + return create(exec, this->get_size()); + } + + Cloneable* copy_from_impl(const Cloneable* other) override + { + this->last_access = gko::as(other)->last_access; + return this; + } +}; + + class DummyLinOp : public gko::LinOp, public gko::EnableCloneable, public gko::EnableCreateMethod { @@ -28,23 +53,26 @@ class DummyLinOp : public gko::LinOp, mutable std::shared_ptr last_beta_access; protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { this->access(); - static_cast(b)->access(); - static_cast(x)->access(); + dynamic_cast(b)->access(); + dynamic_cast(x)->access(); last_b_access = b->get_executor(); last_x_access = x->get_executor(); } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override { this->access(); - static_cast(alpha)->access(); - static_cast(b)->access(); - static_cast(beta)->access(); - static_cast(x)->access(); + dynamic_cast(alpha)->access(); + dynamic_cast(b)->access(); + dynamic_cast(beta)->access(); + dynamic_cast(x)->access(); last_alpha_access = alpha->get_executor(); last_b_access = b->get_executor(); last_beta_access = beta->get_executor(); @@ -57,17 +85,17 @@ class LinOp : public CudaTestFixture { protected: LinOp() : op{DummyLinOp::create(exec, gko::dim<2>{3, 5})}, - alpha{DummyLinOp::create(ref, gko::dim<2>{1})}, - beta{DummyLinOp::create(ref, gko::dim<2>{1})}, - b{DummyLinOp::create(ref, gko::dim<2>{5, 4})}, - x{DummyLinOp::create(ref, gko::dim<2>{3, 4})} + alpha{AccessVector::create(ref, gko::dim<2>{1})}, + beta{AccessVector::create(ref, gko::dim<2>{1})}, + b{AccessVector::create(ref, gko::dim<2>{5, 4})}, + x{AccessVector::create(ref, gko::dim<2>{3, 4})} {} std::unique_ptr op; - std::unique_ptr alpha; - std::unique_ptr beta; - std::unique_ptr b; - std::unique_ptr x; + std::unique_ptr alpha; + std::unique_ptr beta; + std::unique_ptr b; + std::unique_ptr x; }; diff --git a/examples/custom-logger/custom-logger.cpp b/examples/custom-logger/custom-logger.cpp index 9c2e9adfbc8..9898efa6d12 100644 --- a/examples/custom-logger/custom-logger.cpp +++ b/examples/custom-logger/custom-logger.cpp @@ -97,14 +97,14 @@ struct ResidualLogger : gko::log::Logger { // Customize the logging hook which is called everytime an iteration is // completed - void on_iteration_complete(const gko::LinOp* solver, const gko::LinOp* b, - const gko::LinOp* solution, - const gko::size_type& iteration, - const gko::LinOp* residual, - const gko::LinOp* residual_norm, - const gko::LinOp* implicit_sq_residual_norm, - const gko::array*, - bool) const override + void on_iteration_complete( + const gko::LinOp* solver, const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* solution, + const gko::size_type& iteration, + const gko::AbstractMultiVector* residual, + const gko::AbstractMultiVector* residual_norm, + const gko::AbstractMultiVector* implicit_sq_residual_norm, + const gko::array*, bool) const override { // If the solver shares a residual norm, log its value if (residual_norm) { diff --git a/examples/custom-matrix-format/custom-matrix-format.cpp b/examples/custom-matrix-format/custom-matrix-format.cpp index a570031cacd..31c13a53087 100644 --- a/examples/custom-matrix-format/custom-matrix-format.cpp +++ b/examples/custom-matrix-format/custom-matrix-format.cpp @@ -54,7 +54,8 @@ class StencilMatrix : public gko::LinOp, // For simplicity, we assume that there is always only one right hand side // and the stride of consecutive elements in the vectors is 1 (both of these // are always true in this example). - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { // we only implement the operator for dense RHS. // gko::as will throw an exception if its argument is not MultiVector. @@ -111,8 +112,10 @@ class StencilMatrix : public gko::LinOp, // x = alpha * A * b + beta * x. This function is commonly used and can // often be better optimized than implementing it using x = A * b. However, // for simplicity, we will implement it exactly like that in this example. - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override { auto dense_b = gko::as(b); auto dense_x = gko::as(x); diff --git a/examples/mixed-spmv/mixed-spmv.cpp b/examples/mixed-spmv/mixed-spmv.cpp index 3d9cebd2daf..120a8508ff0 100644 --- a/examples/mixed-spmv/mixed-spmv.cpp +++ b/examples/mixed-spmv/mixed-spmv.cpp @@ -188,7 +188,7 @@ int main(int argc, char* argv[]) // copy the data from host to device auto hp_b = share(gko::clone(exec, host_b)); auto lp_b = share(lp_vec::create(exec)); - lp_b->copy_from(hp_b); + hp_b->convert_to(lp_b); // create several result x vector in different precision auto hp_x = share(hp_vec::create(exec, x_dim)); diff --git a/examples/performance-debugging/performance-debugging.cpp b/examples/performance-debugging/performance-debugging.cpp index 533c5c7b544..08f442d5052 100644 --- a/examples/performance-debugging/performance-debugging.cpp +++ b/examples/performance-debugging/performance-debugging.cpp @@ -223,10 +223,11 @@ struct ResidualLogger : gko::log::Logger { // Depending on the available information, store the norm or compute it from // the residual. If the true residual norm could not be computed, store the // value `-1.0`. - void on_iteration_complete(const gko::LinOp*, const gko::size_type&, - const gko::LinOp* residual, - const gko::LinOp* solution, - const gko::LinOp* residual_norm) const override + void on_iteration_complete( + const gko::LinOp*, const gko::size_type&, + const gko::AbstractMultiVector* residual, + const gko::AbstractMultiVector* solution, + const gko::AbstractMultiVector* residual_norm) const override { if (residual_norm) { rec_res_norms.push_back(utils::get_first_element( diff --git a/reference/test/base/composition.cpp b/reference/test/base/composition.cpp index 4a014439c8e..4379da85953 100644 --- a/reference/test/base/composition.cpp +++ b/reference/test/base/composition.cpp @@ -27,10 +27,14 @@ class DummyLinOp : public gko::LinOp, bool apply_uses_initial_guess() const override { return true; } protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} explicit DummyLinOp(std::shared_ptr exec) diff --git a/reference/test/base/perturbation.cpp b/reference/test/base/perturbation.cpp index bba5ade38f8..e826d9c5ad3 100644 --- a/reference/test/base/perturbation.cpp +++ b/reference/test/base/perturbation.cpp @@ -140,28 +140,6 @@ TYPED_TEST(Perturbation, AppliesToComplexVector) } -TYPED_TEST(Perturbation, AppliesToMixedComplexVector) -{ - /* - cmp = I + 2 * [ 2 ] * [ 3 2 ] - [ 1 ] - */ - using value_type = gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto cmp = gko::Perturbation::create(this->scalar, this->basis, - this->projector); - auto x = gko::initialize( - {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec); - auto res = Mtx::create_with_config_of(x); - - cmp->apply(x, res); - - GKO_ASSERT_MTX_NEAR(res, - l({value_type{29.0, -58.0}, value_type{16.0, -32.0}}), - (r_mixed())); -} - - TYPED_TEST(Perturbation, AppliesLinearCombinationToVector) { /* @@ -229,32 +207,6 @@ TYPED_TEST(Perturbation, AppliesLinearCombinationToComplexVector) } -TYPED_TEST(Perturbation, AppliesLinearCombinationToMixedComplexVector) -{ - /* - cmp = I + 2 * [ 2 ] * [ 3 2 ] - [ 1 ] - */ - using MixedMultiVector = - gko::matrix::MultiVector>; - using MixedMultiVectorComplex = gko::to_complex; - using value_type = typename MixedMultiVectorComplex::value_type; - auto cmp = gko::Perturbation::create(this->scalar, this->basis, - this->projector); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize( - {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec); - auto res = gko::clone(x); - - cmp->apply(alpha, x, beta, res); - - GKO_ASSERT_MTX_NEAR(res, - l({value_type{86.0, -172.0}, value_type{46.0, -92.0}}), - (r_mixed())); -} - - TYPED_TEST(Perturbation, ConstructionByBasisAppliesToVector) { /* diff --git a/reference/test/factorization/ic_kernels.cpp b/reference/test/factorization/ic_kernels.cpp index 9cf1066a482..e3d9a7bc6c9 100644 --- a/reference/test/factorization/ic_kernels.cpp +++ b/reference/test/factorization/ic_kernels.cpp @@ -43,10 +43,14 @@ class DummyLinOp : public gko::LinOp, {} protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/reference/test/factorization/ilu_kernels.cpp b/reference/test/factorization/ilu_kernels.cpp index ad3b0033bdc..e22ec210b7d 100644 --- a/reference/test/factorization/ilu_kernels.cpp +++ b/reference/test/factorization/ilu_kernels.cpp @@ -43,10 +43,14 @@ class DummyLinOp : public gko::LinOp, {} protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/reference/test/factorization/par_ic_kernels.cpp b/reference/test/factorization/par_ic_kernels.cpp index 26c7c069f37..8dc706a6765 100644 --- a/reference/test/factorization/par_ic_kernels.cpp +++ b/reference/test/factorization/par_ic_kernels.cpp @@ -33,10 +33,14 @@ class DummyLinOp : public gko::LinOp, {} protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/reference/test/factorization/par_ict_kernels.cpp b/reference/test/factorization/par_ict_kernels.cpp index b6d2cc36d9d..7102cd60434 100644 --- a/reference/test/factorization/par_ict_kernels.cpp +++ b/reference/test/factorization/par_ict_kernels.cpp @@ -33,10 +33,14 @@ class DummyLinOp : public gko::LinOp, {} protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/reference/test/factorization/par_ilu_kernels.cpp b/reference/test/factorization/par_ilu_kernels.cpp index 1bdbb119368..82f4551e752 100644 --- a/reference/test/factorization/par_ilu_kernels.cpp +++ b/reference/test/factorization/par_ilu_kernels.cpp @@ -34,10 +34,14 @@ class DummyLinOp : public gko::LinOp, {} protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/reference/test/factorization/par_ilut_kernels.cpp b/reference/test/factorization/par_ilut_kernels.cpp index ec385e9dc7f..2f994bd1799 100644 --- a/reference/test/factorization/par_ilut_kernels.cpp +++ b/reference/test/factorization/par_ilut_kernels.cpp @@ -32,10 +32,14 @@ class DummyLinOp : public gko::LinOp, {} protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp index 61f8051bc4e..2a3832596fd 100644 --- a/reference/test/matrix/coo_kernels.cpp +++ b/reference/test/matrix/coo_kernels.cpp @@ -695,32 +695,6 @@ TYPED_TEST(Coo, AppliesToComplex) } -TYPED_TEST(Coo, AppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using Vec = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - // clang-format off - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec); - auto x = Vec::create(exec, gko::dim<2>{2,2}); - // clang-format on - - this->mtx->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{13.0, 14.0}, mixed_complex_type{19.0, 20.0}}, - {mixed_complex_type{10.0, 10.0}, mixed_complex_type{15.0, 15.0}}}), - 0.0); -} - - TYPED_TEST(Coo, AdvancedAppliesToComplex) { using value_type = typename TestFixture::value_type; @@ -751,38 +725,6 @@ TYPED_TEST(Coo, AdvancedAppliesToComplex) } -TYPED_TEST(Coo, AdvancedAppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using MixedMultiVector = gko::matrix::MultiVector; - using MixedMultiVectorComplex = - gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - // clang-format off - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec); - auto x = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, exec); - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({2.0}, this->exec); - // clang-format on - - this->mtx->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{-11.0, -14.0}, mixed_complex_type{-15.0, -18.0}}, - {mixed_complex_type{-6.0, -6.0}, mixed_complex_type{-9.0, -9.0}}}), - 0.0); -} - - TYPED_TEST(Coo, ApplyAddsToComplex) { using value_type = typename TestFixture::value_type; @@ -867,37 +809,6 @@ TYPED_TEST(Coo, ApplyAddsScaledToComplex) } -TYPED_TEST(Coo, ApplyAddsScaledToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using MixedMultiVector = gko::matrix::MultiVector; - using MixedMultiVectorComplex = - gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - // clang-format off - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec); - auto x = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, exec); - auto alpha = gko::initialize({-1.0}, this->exec); - // clang-format on - - this->mtx->apply2(alpha, b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{-12.0, -14.0}, mixed_complex_type{-17.0, -19.0}}, - {mixed_complex_type{-8.0, -8.0}, mixed_complex_type{-12.0, -12.0}}}), - 0.0); -} - - TYPED_TEST(Coo, Transpose) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp index 2b732802808..10288f3754d 100644 --- a/reference/test/matrix/csr_kernels.cpp +++ b/reference/test/matrix/csr_kernels.cpp @@ -2406,32 +2406,6 @@ TYPED_TEST(Csr, AppliesToComplex) } -TYPED_TEST(Csr, AppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using Vec = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - // clang-format off - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec); - auto x = Vec::create(exec, gko::dim<2>{2,2}); - // clang-format on - - this->mtx->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{13.0, 14.0}, mixed_complex_type{19.0, 20.0}}, - {mixed_complex_type{10.0, 10.0}, mixed_complex_type{15.0, 15.0}}}), - 0.0); -} - - TYPED_TEST(Csr, AdvancedAppliesToComplex) { using value_type = typename TestFixture::value_type; @@ -2462,38 +2436,6 @@ TYPED_TEST(Csr, AdvancedAppliesToComplex) } -TYPED_TEST(Csr, AdvancedAppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using MixedMultiVector = gko::matrix::MultiVector; - using MixedMultiVectorComplex = - gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - // clang-format off - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec); - auto x = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, exec); - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({2.0}, this->exec); - // clang-format on - - this->mtx->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{-11.0, -14.0}, mixed_complex_type{-15.0, -18.0}}, - {mixed_complex_type{-6.0, -6.0}, mixed_complex_type{-9.0, -9.0}}}), - 0.0); -} - - TYPED_TEST(Csr, ScalesData) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp index c7a1f80948c..3ad1c9a28ad 100644 --- a/reference/test/matrix/dense_kernels.cpp +++ b/reference/test/matrix/dense_kernels.cpp @@ -582,30 +582,6 @@ TYPED_TEST(Dense, AppliesToComplex) } -TYPED_TEST(Dense, AppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using Vec = gko::matrix::Dense; - auto exec = gko::ReferenceExecutor::create(); - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, - exec); - auto x = Vec::create(exec, gko::dim<2>{2, 2}); - - this->mtx1->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{14.0, 16.0}, mixed_complex_type{20.0, 22.0}}, - {mixed_complex_type{17.0, 19.0}, mixed_complex_type{24.5, 26.5}}}), - 0.0); -} - - TYPED_TEST(Dense, AdvancedAppliesToComplex) { using value_type = typename TestFixture::value_type; diff --git a/reference/test/matrix/diagonal_kernels.cpp b/reference/test/matrix/diagonal_kernels.cpp index 37446551596..5c93f126b54 100644 --- a/reference/test/matrix/diagonal_kernels.cpp +++ b/reference/test/matrix/diagonal_kernels.cpp @@ -573,33 +573,6 @@ TYPED_TEST(Diagonal, AppliesToComplex) } -TYPED_TEST(Diagonal, AppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using Vec = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - auto mdense1 = gko::initialize( - {{mixed_complex_type{1.0, 2.0}, mixed_complex_type{2.0, 4.0}, - mixed_complex_type{3.0, 6.0}}, - {mixed_complex_type{1.5, 3.0}, mixed_complex_type{2.5, 5.0}, - mixed_complex_type{3.5, 7.0}}}, - exec); - auto mdense2 = Vec::create(exec, gko::dim<2>{2, 3}); - - this->diag1->apply(mdense1, mdense2); - - GKO_ASSERT_MTX_NEAR( - mdense2, - l({{mixed_complex_type{2.0, 4.0}, mixed_complex_type{4.0, 8.0}, - mixed_complex_type{6.0, 12.0}}, - {mixed_complex_type{4.5, 9.0}, mixed_complex_type{7.5, 15.0}, - mixed_complex_type{10.5, 21.0}}}), - 0.0); -} - - TYPED_TEST(Diagonal, AppliesLinearCombinationToComplex) { using value_type = typename TestFixture::value_type; @@ -633,41 +606,6 @@ TYPED_TEST(Diagonal, AppliesLinearCombinationToComplex) } -TYPED_TEST(Diagonal, AppliesLinearCombinationToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using Vec = gko::matrix::MultiVector; - using Scalar = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - auto dense1 = gko::initialize( - {{mixed_complex_type{1.0, 2.0}, mixed_complex_type{2.0, 4.0}, - mixed_complex_type{3.0, 6.0}}, - {mixed_complex_type{1.5, 3.0}, mixed_complex_type{2.5, 5.0}, - mixed_complex_type{3.5, 7.0}}}, - exec); - auto dense2 = gko::initialize( - {{mixed_complex_type{1.0, 2.0}, mixed_complex_type{2.0, 4.0}, - mixed_complex_type{3.0, 6.0}}, - {mixed_complex_type{1.5, 3.0}, mixed_complex_type{2.5, 5.0}, - mixed_complex_type{3.5, 7.0}}}, - exec); - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({2.0}, this->exec); - - this->diag1->apply(alpha, dense1, beta, dense2); - - GKO_ASSERT_MTX_NEAR( - dense2, - l({{mixed_complex_type{0.0, 0.0}, mixed_complex_type{0.0, 0.0}, - mixed_complex_type{0.0, 0.0}}, - {mixed_complex_type{-1.5, -3.0}, mixed_complex_type{-2.5, -5.0}, - mixed_complex_type{-3.5, -7.0}}}), - 0.0); -} - - template class DiagonalComplex : public ::testing::Test { protected: diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp index 80919ff1bdc..6314fa14bf7 100644 --- a/reference/test/matrix/ell_kernels.cpp +++ b/reference/test/matrix/ell_kernels.cpp @@ -919,32 +919,6 @@ TYPED_TEST(Ell, AppliesToComplex) } -TYPED_TEST(Ell, AppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using Vec = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - // clang-format off - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec); - auto x = Vec::create(exec, gko::dim<2>{2,2}); - // clang-format on - - this->mtx1->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{13.0, 14.0}, mixed_complex_type{19.0, 20.0}}, - {mixed_complex_type{10.0, 10.0}, mixed_complex_type{15.0, 15.0}}}), - 0.0); -} - - TYPED_TEST(Ell, AdvancedAppliesToComplex) { using value_type = typename TestFixture::value_type; diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp index 8421ba478d6..1bd960f4728 100644 --- a/reference/test/matrix/hybrid_kernels.cpp +++ b/reference/test/matrix/hybrid_kernels.cpp @@ -702,32 +702,6 @@ TYPED_TEST(Hybrid, AppliesToComplex) } -TYPED_TEST(Hybrid, AppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using Vec = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - // clang-format off - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, exec); - auto x = Vec::create(exec, gko::dim<2>{2,2}); - // clang-format on - - this->mtx1->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{13.0, 14.0}, mixed_complex_type{19.0, 20.0}}, - {mixed_complex_type{10.0, 10.0}, mixed_complex_type{15.0, 15.0}}}), - 0.0); -} - - TYPED_TEST(Hybrid, AdvancedAppliesToComplex) { using value_type = typename TestFixture::value_type; @@ -759,39 +733,6 @@ TYPED_TEST(Hybrid, AdvancedAppliesToComplex) } -TYPED_TEST(Hybrid, AdvancedAppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using MixedMultiVector = gko::matrix::MultiVector; - using MixedMultiVectorComplex = - gko::matrix::MultiVector; - - // clang-format off - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, - this->exec); - auto x = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, - this->exec); - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({2.0}, this->exec); - // clang-format on - - this->mtx1->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{-11.0, -14.0}, mixed_complex_type{-15.0, -18.0}}, - {mixed_complex_type{-6.0, -6.0}, mixed_complex_type{-9.0, -9.0}}}), - 0.0); -} - - template class HybridComplex : public ::testing::Test { protected: diff --git a/reference/test/matrix/identity.cpp b/reference/test/matrix/identity.cpp index adddd3c04c7..4bde6824c3e 100644 --- a/reference/test/matrix/identity.cpp +++ b/reference/test/matrix/identity.cpp @@ -22,7 +22,6 @@ class Identity : public ::testing::Test { using Vec = gko::matrix::MultiVector; using MixedVec = gko::matrix::MultiVector>; using ComplexVec = gko::to_complex; - using MixedComplexVec = gko::to_complex; Identity() : exec(gko::ReferenceExecutor::create()) {} @@ -142,20 +141,6 @@ TYPED_TEST(Identity, AppliesToComplex) } -TYPED_TEST(Identity, AppliesToMixedComplex) -{ - using Id = typename TestFixture::Id; - using MixedComplexVec = typename TestFixture::MixedComplexVec; - auto identity = Id::create(this->exec, 3); - auto x = gko::initialize({3.0, -1.0, 2.0}, this->exec); - auto b = gko::initialize({2.0, 1.0, 5.0}, this->exec); - - identity->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, l({2.0, 1.0, 5.0}), 0.0); -} - - TYPED_TEST(Identity, AppliesLinearCombinationToComplex) { using Id = typename TestFixture::Id; @@ -173,21 +158,4 @@ TYPED_TEST(Identity, AppliesLinearCombinationToComplex) } -TYPED_TEST(Identity, AppliesLinearCombinationToMixedComplex) -{ - using Id = typename TestFixture::Id; - using MixedVec = typename TestFixture::MixedVec; - using MixedComplexVec = typename TestFixture::MixedComplexVec; - auto identity = Id::create(this->exec, 3); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({1.0}, this->exec); - auto x = gko::initialize({3.0, -1.0, 2.0}, this->exec); - auto b = gko::initialize({2.0, 1.0, 5.0}, this->exec); - - identity->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, l({7.0, 1.0, 12.0}), 0.0); -} - - } // namespace diff --git a/reference/test/matrix/multivector_kernels.cpp b/reference/test/matrix/multivector_kernels.cpp index 82e81b48097..793316ab6d1 100644 --- a/reference/test/matrix/multivector_kernels.cpp +++ b/reference/test/matrix/multivector_kernels.cpp @@ -101,7 +101,9 @@ TYPED_TEST(MultiVector, TemporaryOutputCloneWorks) using value_type = typename TestFixture::value_type; auto other = gko::OmpExecutor::create(); auto m = - gko::initialize>({1.0, 2.0}, other); + gko::matrix::MultiVector::create(other, gko::dim<2>{2, 1}); + m->at(0) = 1.0; + m->at(1) = 2.0; { auto clone = gko::make_temporary_output_clone(this->exec, m); @@ -397,40 +399,6 @@ TYPED_TEST(MultiVector, AddScaledFailsOnWrongSizes) } -TYPED_TEST(MultiVector, AddsScaledDiag) -{ - using Mtx = typename TestFixture::Mtx; - using T = typename TestFixture::value_type; - auto alpha = gko::initialize({2.0}, this->exec); - auto diag = gko::matrix::Diagonal::create( - this->exec, 2, gko::array{this->exec, {3.0, 2.0}}); - - this->mtx2->add_scaled(alpha, diag); - - ASSERT_EQ(this->mtx2->at(0, 0), T{7.0}); - ASSERT_EQ(this->mtx2->at(0, 1), T{-1.0}); - ASSERT_EQ(this->mtx2->at(1, 0), T{-2.0}); - ASSERT_EQ(this->mtx2->at(1, 1), T{6.0}); -} - - -TYPED_TEST(MultiVector, SubtractsScaledDiag) -{ - using Mtx = typename TestFixture::Mtx; - using T = typename TestFixture::value_type; - auto alpha = gko::initialize({-2.0}, this->exec); - auto diag = gko::matrix::Diagonal::create( - this->exec, 2, gko::array{this->exec, {3.0, 2.0}}); - - this->mtx2->sub_scaled(alpha, diag); - - ASSERT_EQ(this->mtx2->at(0, 0), T{7.0}); - ASSERT_EQ(this->mtx2->at(0, 1), T{-1.0}); - ASSERT_EQ(this->mtx2->at(1, 0), T{-2.0}); - ASSERT_EQ(this->mtx2->at(1, 1), T{6.0}); -} - - TYPED_TEST(MultiVector, ComputesDot) { using Mtx = typename TestFixture::Mtx; @@ -453,7 +421,7 @@ TYPED_TEST(MultiVector, ComputesDotMixed) this->mtx3->convert_to(mmtx3); auto result = MixedMtx::create(this->exec, gko::dim<2>{1, 3}); - this->mtx1->compute_dot(this->mtx3, result); + this->mtx1->compute_dot(mmtx3, result); EXPECT_EQ(result->at(0, 0), MixedT{1.75}); EXPECT_EQ(result->at(0, 1), MixedT{7.75}); @@ -483,7 +451,7 @@ TYPED_TEST(MultiVector, ComputesConjDotMixed) this->mtx3->convert_to(mmtx3); auto result = MixedMtx::create(this->exec, gko::dim<2>{1, 3}); - this->mtx1->compute_conj_dot(this->mtx3, result); + this->mtx1->compute_conj_dot(mmtx3, result); EXPECT_EQ(result->at(0, 0), MixedT{1.75}); EXPECT_EQ(result->at(0, 1), MixedT{7.75}); @@ -747,7 +715,7 @@ TYPED_TEST(MultiVector, SquareSubmatrixIsTransposableIntoMultiVector) using T = typename TestFixture::value_type; auto trans = Mtx::create(this->exec, gko::dim<2>{2, 2}, 4); - this->mtx5->create_submatrix({0, 2}, {0, 2})->transpose(trans); + this->mtx5->create_subview({0, 2}, {0, 2})->transpose(trans); GKO_ASSERT_MTX_NEAR(trans, l({{1.0, -2.0}, {-1.0, 2.0}}), 0.0); ASSERT_EQ(trans->get_stride(), 4); @@ -793,7 +761,7 @@ TYPED_TEST(MultiVector, NonSquareSubmatrixIsTransposableIntoMultiVector) using T = typename TestFixture::value_type; auto trans = Mtx::create(this->exec, gko::dim<2>{2, 1}, 5); - this->mtx4->create_submatrix({0, 1}, {0, 2})->transpose(trans); + this->mtx4->create_subview({0, 1}, {0, 2})->transpose(trans); GKO_ASSERT_MTX_NEAR(trans, l({1.0, 3.0}), 0.0); ASSERT_EQ(trans->get_stride(), 5); @@ -825,7 +793,7 @@ TYPED_TEST(MultiVector, InplaceAbsolute) TYPED_TEST(MultiVector, InplaceAbsoluteSubMatrix) { using T = typename TestFixture::value_type; - auto mtx = this->mtx5->create_submatrix(gko::span{0, 2}, gko::span{0, 2}); + auto mtx = this->mtx5->create_subview(gko::span{0, 2}, gko::span{0, 2}); mtx->compute_absolute_inplace(); @@ -865,7 +833,7 @@ TYPED_TEST(MultiVector, OutplaceAbsoluteIntoMultiVector) TYPED_TEST(MultiVector, OutplaceAbsoluteSubMatrix) { using T = typename TestFixture::value_type; - auto mtx = this->mtx5->create_submatrix(gko::span{0, 2}, gko::span{0, 2}); + auto mtx = this->mtx5->create_subview(gko::span{0, 2}, gko::span{0, 2}); auto abs_mtx = mtx->compute_absolute(); @@ -878,7 +846,7 @@ TYPED_TEST(MultiVector, OutplaceSubmatrixAbsoluteIntoMultiVector) { using Mtx = typename TestFixture::Mtx; using T = typename TestFixture::value_type; - auto mtx = this->mtx5->create_submatrix(gko::span{0, 2}, gko::span{0, 2}); + auto mtx = this->mtx5->create_subview(gko::span{0, 2}, gko::span{0, 2}); auto abs_mtx = gko::remove_complex::create(this->exec, gko::dim<2>{2, 2}, 4); @@ -996,17 +964,17 @@ TYPED_TEST(MultiVector, GetImagIntoMultiVectorFailsForWrongDimensions) } -TYPED_TEST(MultiVector, MakeTemporaryConversionDoesntConvertOnMatch) +TYPED_TEST(MultiVector, AsPrecisionDoesntConvertOnMatch) { using Mtx = typename TestFixture::Mtx; using T = typename TestFixture::value_type; auto alpha = gko::initialize({8.0}, this->exec); - ASSERT_EQ(gko::make_temporary_conversion(alpha).get(), alpha.get()); + ASSERT_EQ(alpha->template as_precision().get(), alpha.get()); } -TYPED_TEST(MultiVector, MakeTemporaryConversionConvertsBack) +TYPED_TEST(MultiVector, AsPrecisionConvertsBack) { using MixedMtx = typename TestFixture::MixedMtx; using T = typename TestFixture::value_type; @@ -1014,7 +982,7 @@ TYPED_TEST(MultiVector, MakeTemporaryConversionConvertsBack) auto alpha = gko::initialize({8.0}, this->exec); { - auto conversion = gko::make_temporary_conversion(alpha); + auto conversion = alpha->template as_precision(); conversion->at(0, 0) = T{7.0}; } @@ -1022,7 +990,7 @@ TYPED_TEST(MultiVector, MakeTemporaryConversionConvertsBack) } -TYPED_TEST(MultiVector, MakeTemporaryConversionConstDoesntConvertBack) +TYPED_TEST(MultiVector, AsPrecisionConstDoesntConvertBack) { using MixedMtx = typename TestFixture::MixedMtx; using T = typename TestFixture::value_type; @@ -1030,8 +998,8 @@ TYPED_TEST(MultiVector, MakeTemporaryConversionConstDoesntConvertBack) auto alpha = gko::initialize({8.0}, this->exec); { - auto conversion = gko::make_temporary_conversion( - static_cast(alpha.get())); + auto conversion = static_cast(alpha.get()) + ->template as_precision(); alpha->at(0, 0) = MixedT{7.0}; } @@ -1126,7 +1094,7 @@ std::unique_ptr> ref_permute( { using gko::matrix::permute_mode; auto result = input->clone(); - auto permutation_multivector = + auto permutation_dense = gko::matrix::MultiVector::create(input->get_executor()); gko::matrix_data permutation_data; if ((mode & permute_mode::inverse) == permute_mode::inverse) { @@ -1134,17 +1102,17 @@ std::unique_ptr> ref_permute( } else { permutation->write(permutation_data); } - permutation_multivector->read(permutation_data); + permutation_dense->read(permutation_data); if ((mode & permute_mode::rows) == permute_mode::rows) { // compute P * A - permutation_multivector->apply(input, result); + permutation_dense->as_const_dense_view()->apply(input, result); } if ((mode & permute_mode::columns) == permute_mode::columns) { // compute A * P^T = (P * A^T)^T auto tmp = gko::share(result->transpose()); auto tmp2 = gko::as>( gko::as(tmp)->clone()); - permutation_multivector->apply(tmp, tmp2); + permutation_dense->as_const_dense_view()->apply(tmp, tmp2); tmp2->transpose(result); } return result; @@ -1159,9 +1127,9 @@ std::unique_ptr> ref_permute( { using gko::matrix::permute_mode; auto result = input->clone(); - auto row_permutation_multivector = + auto row_permutation_dense = gko::matrix::MultiVector::create(input->get_executor()); - auto col_permutation_multivector = + auto col_permutation_dense = gko::matrix::MultiVector::create(input->get_executor()); gko::matrix_data row_permutation_data; gko::matrix_data col_permutation_data; @@ -1172,13 +1140,13 @@ std::unique_ptr> ref_permute( row_permutation->write(row_permutation_data); col_permutation->write(col_permutation_data); } - row_permutation_multivector->read(row_permutation_data); - col_permutation_multivector->read(col_permutation_data); - row_permutation_multivector->apply(input, result); + row_permutation_dense->read(row_permutation_data); + col_permutation_dense->read(col_permutation_data); + row_permutation_dense->as_const_dense_view()->apply(input, result); auto tmp = gko::share(result->transpose()); auto tmp2 = gko::as>( gko::as(tmp)->clone()); - col_permutation_multivector->apply(tmp, tmp2); + col_permutation_dense->as_const_dense_view()->apply(tmp, tmp2); tmp2->transpose(result); return result; } @@ -1454,7 +1422,7 @@ TYPED_TEST(MultiVectorWithIndexType, gko::array permute_idxs{exec, {1, 0}}; auto row_collection = Mtx::create(exec, gko::dim<2>{2, 2}, 4); - this->mtx5->create_submatrix({0, 2}, {1, 3}) + this->mtx5->create_subview({0, 2}, {1, 3}) ->row_gather(&permute_idxs, row_collection); GKO_ASSERT_MTX_NEAR(row_collection, @@ -1562,7 +1530,7 @@ TYPED_TEST(MultiVectorWithIndexType, SquareSubmatrixIsPermutableIntoMultiVector) auto exec = this->mtx5->get_executor(); gko::array permute_idxs{exec, {1, 0}}; auto permuted = Mtx::create(exec, gko::dim<2>{2, 2}, 4); - auto mtx = this->mtx5->create_submatrix({0, 2}, {1, 3}); + auto mtx = this->mtx5->create_subview({0, 2}, {1, 3}); auto ref_permuted = gko::as(gko::as(mtx->row_permute(&permute_idxs)) @@ -1654,7 +1622,7 @@ TYPED_TEST(MultiVectorWithIndexType, auto exec = this->mtx5->get_executor(); gko::array permute_idxs{exec, {1, 0}}; auto permuted = Mtx::create(exec, gko::dim<2>{2, 2}, 4); - auto mtx = this->mtx5->create_submatrix({0, 2}, {1, 3}); + auto mtx = this->mtx5->create_subview({0, 2}, {1, 3}); auto ref_permuted = gko::as(gko::as(mtx->inverse_row_permute(&permute_idxs)) @@ -1768,7 +1736,7 @@ TYPED_TEST(MultiVectorWithIndexType, gko::array permute_idxs{exec, {1, 0}}; auto permuted = Mtx::create(exec, gko::dim<2>{2, 2}, 4); - this->mtx5->create_submatrix({0, 2}, {0, 2}) + this->mtx5->create_subview({0, 2}, {0, 2}) ->row_permute(&permute_idxs, permuted); GKO_ASSERT_MTX_NEAR(permuted, l({{-2.0, 2.0}, {1.0, -1.0}}), @@ -1864,7 +1832,7 @@ TYPED_TEST(MultiVectorWithIndexType, gko::array permute_idxs{exec, {1, 0}}; auto permuted = Mtx::create(exec, gko::dim<2>{2, 2}, 4); - this->mtx5->create_submatrix({0, 2}, {0, 2}) + this->mtx5->create_subview({0, 2}, {0, 2}) ->column_permute(&permute_idxs, permuted); GKO_ASSERT_MTX_NEAR(permuted, l({{-1.0, 1.0}, {2.0, -2.0}}), @@ -1963,7 +1931,7 @@ TYPED_TEST(MultiVectorWithIndexType, gko::array permute_idxs{exec, {1, 0}}; auto permuted = Mtx::create(exec, gko::dim<2>{2, 2}, 4); - this->mtx5->create_submatrix({0, 2}, {0, 2}) + this->mtx5->create_subview({0, 2}, {0, 2}) ->inverse_row_permute(&permute_idxs, permuted); GKO_ASSERT_MTX_NEAR(permuted, l({{-2.0, 2.0}, {1.0, -1.0}}), @@ -2064,7 +2032,7 @@ TYPED_TEST(MultiVectorWithIndexType, gko::array permute_idxs{exec, {1, 0}}; auto permuted = Mtx::create(exec, gko::dim<2>{2, 2}, 4); - this->mtx5->create_submatrix({0, 2}, {0, 2}) + this->mtx5->create_subview({0, 2}, {0, 2}) ->column_permute(&permute_idxs, permuted); GKO_ASSERT_MTX_NEAR(permuted, l({{-1.0, 1.0}, {2.0, -2.0}}), @@ -2110,7 +2078,7 @@ std::unique_ptr> ref_scaled_permute( { using gko::matrix::permute_mode; auto result = input->clone(); - auto permutation_multivector = + auto permutation_dense = gko::matrix::MultiVector::create(input->get_executor()); gko::matrix_data permutation_data; if ((mode & permute_mode::inverse) == permute_mode::inverse) { @@ -2118,17 +2086,17 @@ std::unique_ptr> ref_scaled_permute( } else { permutation->write(permutation_data); } - permutation_multivector->read(permutation_data); + permutation_dense->read(permutation_data); if ((mode & permute_mode::rows) == permute_mode::rows) { // compute P * A - permutation_multivector->apply(input, result); + permutation_dense->as_const_dense_view()->apply(input, result); } if ((mode & permute_mode::columns) == permute_mode::columns) { // compute A * P^T = (P * A^T)^T auto tmp = share(result->transpose()); auto tmp2 = gko::as>( gko::as(tmp)->clone()); - permutation_multivector->apply(tmp, tmp2); + permutation_dense->as_const_dense_view()->apply(tmp, tmp2); tmp2->transpose(result); } return result; @@ -2144,9 +2112,9 @@ std::unique_ptr> ref_scaled_permute( { using gko::matrix::permute_mode; auto result = input->clone(); - auto row_permutation_multivector = + auto row_permutation_dense = gko::matrix::MultiVector::create(input->get_executor()); - auto col_permutation_multivector = + auto col_permutation_dense = gko::matrix::MultiVector::create(input->get_executor()); gko::matrix_data row_permutation_data; gko::matrix_data col_permutation_data; @@ -2157,13 +2125,13 @@ std::unique_ptr> ref_scaled_permute( row_permutation->write(row_permutation_data); col_permutation->write(col_permutation_data); } - row_permutation_multivector->read(row_permutation_data); - col_permutation_multivector->read(col_permutation_data); - row_permutation_multivector->apply(input, result); + row_permutation_dense->read(row_permutation_data); + col_permutation_dense->read(col_permutation_data); + row_permutation_dense->as_const_dense_view()->apply(input, result); auto tmp = gko::share(result->transpose()); auto tmp2 = gko::as>( gko::as(tmp)->clone()); - col_permutation_multivector->apply(tmp, tmp2); + col_permutation_dense->as_const_dense_view()->apply(tmp, tmp2); tmp2->transpose(result); return result; } diff --git a/reference/test/matrix/sellp_kernels.cpp b/reference/test/matrix/sellp_kernels.cpp index 187b344d4ab..3bbc6cbda6b 100644 --- a/reference/test/matrix/sellp_kernels.cpp +++ b/reference/test/matrix/sellp_kernels.cpp @@ -679,31 +679,6 @@ TYPED_TEST(Sellp, AppliesToComplex) } -TYPED_TEST(Sellp, AppliesToMixedComplex) -{ - using value_type = typename TestFixture::value_type; - using complex_type = gko::to_complex; - using Vec = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - // clang-format off - auto b = gko::initialize( - {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}}, - {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}, - {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec); - auto x = Vec::create(exec, gko::dim<2>{2,2}); - // clang-format on - - this->mtx1->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{complex_type{13.0, 14.0}, complex_type{19.0, 20.0}}, - {complex_type{10.0, 10.0}, complex_type{15.0, 15.0}}}), - 0.0); -} - - TYPED_TEST(Sellp, AdvancedAppliesToComplex) { using value_type = typename TestFixture::value_type; @@ -734,36 +709,6 @@ TYPED_TEST(Sellp, AdvancedAppliesToComplex) } -TYPED_TEST(Sellp, AdvancedAppliesToMixedComplex) -{ - using value_type = typename TestFixture::value_type; - using complex_type = gko::to_complex; - using MixedMultiVector = gko::matrix::MultiVector; - using MixedMultiVectorComplex = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - // clang-format off - auto b = gko::initialize( - {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}}, - {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}, - {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, exec); - auto x = gko::initialize( - {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}}, - {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, exec); - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({2.0}, this->exec); - // clang-format on - - this->mtx1->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{complex_type{-11.0, -14.0}, complex_type{-15.0, -18.0}}, - {complex_type{-6.0, -6.0}, complex_type{-9.0, -9.0}}}), - 0.0); -} - - template class SellpComplex : public ::testing::Test { protected: diff --git a/reference/test/matrix/sparsity_csr_kernels.cpp b/reference/test/matrix/sparsity_csr_kernels.cpp index 87abe594509..e9532ada23a 100644 --- a/reference/test/matrix/sparsity_csr_kernels.cpp +++ b/reference/test/matrix/sparsity_csr_kernels.cpp @@ -259,22 +259,6 @@ TYPED_TEST(SparsityCsr, AppliesToComplex) } -TYPED_TEST(SparsityCsr, AppliesToMixedComplex) -{ - using T = - gko::next_precision>; - using Vec = gko::matrix::MultiVector; - auto x = gko::initialize({T{2.0, 4.0}, T{1.0, 2.0}, T{4.0, 8.0}}, - this->exec); - auto y = Vec::create(this->exec, gko::dim<2>{2, 1}); - - this->mtx->apply(x, y); - - EXPECT_EQ(y->at(0), T(7.0, 14.0)); - EXPECT_EQ(y->at(1), T(1.0, 2.0)); -} - - TYPED_TEST(SparsityCsr, AppliesLinearCombinationToComplex) { using Vec = typename TestFixture::Vec; @@ -294,26 +278,6 @@ TYPED_TEST(SparsityCsr, AppliesLinearCombinationToComplex) } -TYPED_TEST(SparsityCsr, AppliesLinearCombinationToMixedComplex) -{ - using Vec = gko::matrix::MultiVector< - gko::next_precision>; - using ComplexVec = gko::to_complex; - using T = typename ComplexVec::value_type; - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({2.0}, this->exec); - auto x = gko::initialize( - {T{2.0, 4.0}, T{1.0, 2.0}, T{4.0, 8.0}}, this->exec); - auto y = - gko::initialize({T{1.0, 2.0}, T{2.0, 4.0}}, this->exec); - - this->mtx->apply(alpha, x, beta, y); - - EXPECT_EQ(y->at(0), T(-5.0, -10.0)); - EXPECT_EQ(y->at(1), T(3.0, 6.0)); -} - - TYPED_TEST(SparsityCsr, ApplyFailsOnWrongInnerDimension) { using Vec = typename TestFixture::Vec; diff --git a/reference/test/preconditioner/ic.cpp b/reference/test/preconditioner/ic.cpp index d30acc4de74..b8241244149 100644 --- a/reference/test/preconditioner/ic.cpp +++ b/reference/test/preconditioner/ic.cpp @@ -253,25 +253,6 @@ TYPED_TEST(Ic, SolvesSingleRhsComplex) } -TYPED_TEST(Ic, SolvesSingleRhsComplexMixed) -{ - using ic_prec_type = typename TestFixture::ic_type; - using Vec = gko::matrix::MultiVector< - gko::next_precision>>; - using T = typename Vec::value_type; - const auto b = gko::initialize( - {T{1.0, 2.0}, T{3.0, 6.0}, T{6.0, 12.0}}, this->exec); - auto x = Vec::create(this->exec, gko::dim<2>{3, 1}); - auto preconditioner = - ic_prec_type::build().on(this->exec)->generate(this->mtx); - - preconditioner->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, l({T{3.0, 6.0}, T{-2.0, -4.0}, T{4.0, 8.0}}), - this->tol); -} - - TYPED_TEST(Ic, AdvancedSolvesSingleRhs) { using ic_prec_type = typename TestFixture::ic_type; @@ -329,29 +310,6 @@ TYPED_TEST(Ic, AdvancedSolvesSingleRhsComplex) } -TYPED_TEST(Ic, AdvancedSolvesSingleRhsComplexMixed) -{ - using ic_prec_type = typename TestFixture::ic_type; - using MixedMultiVector = gko::matrix::MultiVector< - gko::next_precision>; - using MixedMultiVectorComplex = gko::to_complex; - using T = typename MixedMultiVectorComplex::value_type; - const auto b = gko::initialize( - {T{1.0, 2.0}, T{3.0, 6.0}, T{6.0, 12.0}}, this->exec); - const auto alpha = gko::initialize({2.0}, this->exec); - const auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize( - {T{1.0, 2.0}, T{2.0, 4.0}, T{3.0, 6.0}}, this->exec); - auto preconditioner = - ic_prec_type::build().on(this->exec)->generate(this->mtx); - - preconditioner->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, l({T{5.0, 10.0}, T{-6.0, -12.0}, T{5.0, 10.0}}), - this->tol); -} - - TYPED_TEST(Ic, SolvesMultipleRhs) { using ic_prec_type = typename TestFixture::ic_type; diff --git a/reference/test/preconditioner/ilu.cpp b/reference/test/preconditioner/ilu.cpp index b9fe38c8d21..512a6eeda42 100644 --- a/reference/test/preconditioner/ilu.cpp +++ b/reference/test/preconditioner/ilu.cpp @@ -310,25 +310,6 @@ TYPED_TEST(Ilu, SolvesSingleRhsWithComplexMtx) } -TYPED_TEST(Ilu, SolvesSingleRhsWithMixedComplexMtx) -{ - using Mtx = gko::matrix::MultiVector< - gko::to_complex>>; - using T = typename Mtx::value_type; - const auto b = gko::initialize( - {T{1.0, 2.0}, T{3.0, 6.0}, T{6.0, 12.0}}, this->exec); - auto x = Mtx::create(this->exec, gko::dim<2>{3, 1}); - x->copy_from(b); - - auto preconditioner = this->ilu_pre_factory->generate(this->mtx); - preconditioner->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, l({T{-0.125, -0.25}, T{0.25, 0.5}, T{1.0, 2.0}}), - (r_mixed()) * 1e+1); -} - - TYPED_TEST(Ilu, SolvesReverseSingleRhs) { using Mtx = typename TestFixture::Mtx; @@ -384,70 +365,6 @@ TYPED_TEST(Ilu, SolvesAdvancedSingleRhsMixed) } -TYPED_TEST(Ilu, SolvesAdvancedSingleRhsComplex) -{ - using value_type = typename TestFixture::value_type; - using complex_type = gko::to_complex; - using MultiVector = typename TestFixture::Mtx; - using MultiVectorComplex = gko::to_complex; - const value_type alpha{2.0}; - const auto alpha_linop = gko::initialize({alpha}, this->exec); - const value_type beta{-1}; - const auto beta_linop = gko::initialize({beta}, this->exec); - const auto b = gko::initialize( - {complex_type{-3.0, 6.0}, complex_type{6.0, -12.0}, - complex_type{9.0, -18.0}}, - this->exec); - auto x = gko::initialize( - {complex_type{1.0, -2.0}, complex_type{2.0, -4.0}, - complex_type{3.0, -6.0}}, - this->exec); - auto preconditioner = - this->ilu_pre_factory->generate(this->l_u_composition); - - preconditioner->apply(alpha_linop, b, beta_linop, x); - - GKO_ASSERT_MTX_NEAR(x, - l({complex_type{-7.0, 14.0}, complex_type{2.0, -4.0}, - complex_type{-1.0, 2.0}}), - r::value * 2.0); -} - - -TYPED_TEST(Ilu, SolvesAdvancedSingleRhsMixedComplex) -{ - using value_type = gko::next_precision; - using complex_type = gko::to_complex; - using MixedMultiVector = gko::matrix::MultiVector; - using MixedMultiVectorComplex = gko::to_complex; - const value_type alpha{2.0}; - const auto alpha_linop = - gko::initialize({alpha}, this->exec); - const value_type beta{-1}; - const auto beta_linop = - gko::initialize({beta}, this->exec); - const auto b = gko::initialize( - {complex_type{-3.0, 6.0}, complex_type{6.0, -12.0}, - complex_type{9.0, -18.0}}, - this->exec); - auto x = gko::initialize( - {complex_type{1.0, -2.0}, complex_type{2.0, -4.0}, - complex_type{3.0, -6.0}}, - this->exec); - auto preconditioner = - this->ilu_pre_factory->generate(this->l_u_composition); - - preconditioner->apply(alpha_linop, b, beta_linop, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({complex_type{-7.0, 14.0}, complex_type{2.0, -4.0}, - complex_type{-1.0, 2.0}}), - (r_mixed()) * - 2.0); -} - - TYPED_TEST(Ilu, SolvesAdvancedReverseSingleRhs) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/preconditioner/jacobi_kernels.cpp b/reference/test/preconditioner/jacobi_kernels.cpp index 82836ea690c..21fddd5a620 100644 --- a/reference/test/preconditioner/jacobi_kernels.cpp +++ b/reference/test/preconditioner/jacobi_kernels.cpp @@ -695,31 +695,6 @@ TYPED_TEST(Jacobi, AppliesToComplexVector) } -TYPED_TEST(Jacobi, AppliesToMixedComplexVector) -{ - using value_type = - gko::to_complex>; - using Vec = gko::matrix::MultiVector; - auto x = gko::initialize( - {value_type{1.0, 2.0}, value_type{-1.0, -2.0}, value_type{2.0, 4.0}, - value_type{-2.0, -4.0}, value_type{3.0, 6.0}}, - this->exec); - auto b = gko::initialize( - {value_type{4.0, 8.0}, value_type{-1.0, -2.0}, value_type{-2.0, -4.0}, - value_type{4.0, 8.0}, value_type{-1.0, -2.0}}, - this->exec); - auto bj = this->bj_factory->generate(this->mtx); - - bj->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({value_type{1.0, 2.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}, - value_type{1.0, 2.0}, value_type{0.0, 0.0}}), - (r_mixed())); -} - - TYPED_TEST(Jacobi, AppliesToVectorWithAdaptivePrecision) { using Vec = typename TestFixture::Vec; @@ -944,32 +919,6 @@ TYPED_TEST(Jacobi, AppliesLinearCombinationToComplexVector) } -TYPED_TEST(Jacobi, AppliesLinearCombinationToMixedComplexVector) -{ - using value_type = gko::next_precision; - using MixedMultiVector = gko::matrix::MultiVector; - using MixedMultiVectorComplex = gko::to_complex; - using T = gko::to_complex; - auto x = gko::initialize( - {T{1.0, 2.0}, T{-1.0, -2.0}, T{2.0, 4.0}, T{-2.0, -4.0}, T{3.0, 6.0}}, - this->exec); - auto b = gko::initialize( - {T{4.0, 8.0}, T{-1.0, -2.0}, T{-2.0, -4.0}, T{4.0, 8.0}, T{-1.0, -2.0}}, - this->exec); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto bj = this->bj_factory->generate(this->mtx); - - bj->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({T{1.0, 2.0}, T{1.0, 2.0}, T{-2.0, -4.0}, T{4.0, 8.0}, - T{-3.0, -6.0}}), - (r_mixed())); -} - - TYPED_TEST(Jacobi, AppliesLinearCombinationToVectorWithAdaptivePrecision) { using Vec = typename TestFixture::Vec; diff --git a/reference/test/solver/bicg_kernels.cpp b/reference/test/solver/bicg_kernels.cpp index 77aec032d0b..7f2c4d76169 100644 --- a/reference/test/solver/bicg_kernels.cpp +++ b/reference/test/solver/bicg_kernels.cpp @@ -316,28 +316,6 @@ TYPED_TEST(Bicg, SolvesStencilSystemComplex) } -TYPED_TEST(Bicg, SolvesStencilSystemMixedComplex) -{ - using value_type = - gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto solver = this->bicg_factory->generate(this->mtx); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.0, -2.0}, value_type{3.0, -6.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(Bicg, SolvesMultipleStencilSystems) { using Mtx = typename TestFixture::Mtx; @@ -413,31 +391,6 @@ TYPED_TEST(Bicg, SolvesStencilSystemUsingAdvancedApplyComplex) } -TYPED_TEST(Bicg, SolvesStencilSystemUsingAdvancedApplyMixedComplex) -{ - using Scalar = gko::matrix::MultiVector< - gko::next_precision>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto solver = this->bicg_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}}, - this->exec); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.5, -3.0}, value_type{5.0, -10.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(Bicg, SolvesMultipleStencilSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/bicgstab_kernels.cpp b/reference/test/solver/bicgstab_kernels.cpp index 62ebc95275e..91a261db75a 100644 --- a/reference/test/solver/bicgstab_kernels.cpp +++ b/reference/test/solver/bicgstab_kernels.cpp @@ -459,28 +459,6 @@ TYPED_TEST(Bicgstab, SolvesMultiVectorSystemComplex) } -TYPED_TEST(Bicgstab, SolvesMultiVectorSystemMixedComplex) -{ - using value_type = - gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto solver = this->bicgstab_factory->generate(this->mtx); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{-4.0, 8.0}, value_type{-1.0, 2.0}, - value_type{4.0, -8.0}}), - (r_mixed())); -} - - TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystems) { using Vec = typename TestFixture::Vec; @@ -576,31 +554,6 @@ TYPED_TEST(Bicgstab, SolvesMultiVectorSystemUsingAdvancedApplyComplex) } -TYPED_TEST(Bicgstab, SolvesMultiVectorSystemUsingAdvancedApplyMixedComplex) -{ - using Scalar = gko::matrix::MultiVector< - gko::next_precision>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto solver = this->bicgstab_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.5, -0.5}, value_type{1.0, 0.5}, value_type{2.0, -1.0}}, - this->exec); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{-8.5, 16.5}, value_type{-3.0, 3.5}, - value_type{6.0, -15.0}}), - (r_mixed())); -} - - TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystemsUsingAdvancedApply) { using Vec = typename TestFixture::Vec; diff --git a/reference/test/solver/cb_gmres_kernels.cpp b/reference/test/solver/cb_gmres_kernels.cpp index f1009d0518a..6ef56ca8332 100644 --- a/reference/test/solver/cb_gmres_kernels.cpp +++ b/reference/test/solver/cb_gmres_kernels.cpp @@ -199,30 +199,6 @@ TYPED_TEST(CbGmres, SolvesStencilSystemComplex) } -TYPED_TEST(CbGmres, SolvesStencilSystemMixedComplex) -{ - using value_type = gko::to_complex< - gko::next_precision_base>; - using Mtx = gko::matrix::MultiVector; - auto solver = this->cb_gmres_factory->generate(this->mtx); - auto b = - gko::initialize({value_type{13.0, -26.0}, value_type{7.0, -14.0}, - value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({value_type{1.0, -2.0}, value_type{3.0, -6.0}, - value_type{2.0, -4.0}}), - std::max(this->assert_precision(), r::value)); -} - - TYPED_TEST(CbGmres, SolvesStencilSystem2) { using Mtx = typename TestFixture::Mtx; @@ -325,33 +301,6 @@ TYPED_TEST(CbGmres, SolvesStencilSystemUsingAdvancedApplyComplex) } -TYPED_TEST(CbGmres, SolvesStencilSystemUsingAdvancedApplyMixedComplex) -{ - using Scalar = gko::matrix::MultiVector< - gko::next_precision_base>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto solver = this->cb_gmres_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = - gko::initialize({value_type{13.0, -26.0}, value_type{7.0, -14.0}, - value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}}, - this->exec); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({value_type{1.5, -3.0}, value_type{5.0, -10.0}, - value_type{2.0, -4.0}}), - std::max(this->assert_precision(), r::value)); -} - - TYPED_TEST(CbGmres, SolvesMultipleStencilSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/cg_kernels.cpp b/reference/test/solver/cg_kernels.cpp index 2364c5ae8b8..804fc9e4f08 100644 --- a/reference/test/solver/cg_kernels.cpp +++ b/reference/test/solver/cg_kernels.cpp @@ -277,28 +277,6 @@ TYPED_TEST(Cg, SolvesStencilSystemComplex) } -TYPED_TEST(Cg, SolvesStencilSystemMixedComplex) -{ - using value_type = - gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto solver = this->cg_factory->generate(this->mtx); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.0, -2.0}, value_type{3.0, -6.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(Cg, SolvesMultipleStencilSystems) { using Vec = typename TestFixture::Vec; @@ -374,31 +352,6 @@ TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApplyComplex) } -TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApplyMixedComplex) -{ - using Scalar = gko::matrix::MultiVector< - gko::next_precision>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto solver = this->cg_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}}, - this->exec); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.5, -3.0}, value_type{5.0, -10.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(Cg, SolvesMultipleStencilSystemsUsingAdvancedApply) { using Vec = typename TestFixture::Vec; diff --git a/reference/test/solver/cgs_kernels.cpp b/reference/test/solver/cgs_kernels.cpp index 1cbe929ba5d..2bbe037a99f 100644 --- a/reference/test/solver/cgs_kernels.cpp +++ b/reference/test/solver/cgs_kernels.cpp @@ -469,32 +469,6 @@ TYPED_TEST(Cgs, SolvesMultiVectorSystemUsingAdvancedApplyComplex) } -TYPED_TEST(Cgs, SolvesMultiVectorSystemUsingAdvancedApplyMixedComplex) -{ - using Scalar = gko::matrix::MultiVector< - gko::next_precision>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto solver = this->cgs_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{-2.0, 4.0}, value_type{-0.5, 1.0}, value_type{2.0, -4.0}}, - this->exec); - - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{-6.0, 12.0}, value_type{-1.5, 3.0}, - value_type{6.0, -12.0}}), - (r_mixed()) * 1e3); -} - - TYPED_TEST(Cgs, SolvesMultipleMultiVectorSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/chebyshev_kernels.cpp b/reference/test/solver/chebyshev_kernels.cpp index f83d8878b05..f9648bf512d 100644 --- a/reference/test/solver/chebyshev_kernels.cpp +++ b/reference/test/solver/chebyshev_kernels.cpp @@ -159,31 +159,6 @@ TYPED_TEST(Chebyshev, SolvesTriangularSystemComplex) } -TYPED_TEST(Chebyshev, SolvesTriangularSystemMixedComplex) -{ - using mixed_complex_type = - gko::to_complex>; - using MixedMtx = gko::matrix::MultiVector; - auto solver = this->chebyshev_factory->generate(this->mtx); - auto b = gko::initialize( - {mixed_complex_type{3.9, -7.8}, mixed_complex_type{9.0, -18.0}, - mixed_complex_type{2.2, -4.4}}, - this->exec); - auto x = gko::initialize( - {mixed_complex_type{0.0, 0.0}, mixed_complex_type{0.0, 0.0}, - mixed_complex_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({mixed_complex_type{1.0, -2.0}, mixed_complex_type{3.0, -6.0}, - mixed_complex_type{2.0, -4.0}}), - (r_mixed()) * 1e1); -} - - TYPED_TEST(Chebyshev, SolvesTriangularSystemWithIterativeInnerSolver) { using Mtx = typename TestFixture::Mtx; @@ -287,34 +262,6 @@ TYPED_TEST(Chebyshev, SolvesTriangularSystemUsingAdvancedApplyComplex) } -TYPED_TEST(Chebyshev, SolvesTriangularSystemUsingAdvancedApplyMixedComplex) -{ - using mixed_type = gko::next_precision; - using mixed_complex_type = gko::to_complex; - using Scalar = gko::matrix::MultiVector; - using MixedMtx = gko::matrix::MultiVector; - auto solver = this->chebyshev_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( - {mixed_complex_type{3.9, -7.8}, mixed_complex_type{9.0, -18.0}, - mixed_complex_type{2.2, -4.4}}, - this->exec); - auto x = gko::initialize( - {mixed_complex_type{0.5, -1.0}, mixed_complex_type{1.0, -2.0}, - mixed_complex_type{2.0, -4.0}}, - this->exec); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({mixed_complex_type{1.5, -3.0}, mixed_complex_type{5.0, -10.0}, - mixed_complex_type{2.0, -4.0}}), - (r_mixed()) * 1e1); -} - - TYPED_TEST(Chebyshev, SolvesMultipleStencilSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/fcg_kernels.cpp b/reference/test/solver/fcg_kernels.cpp index f223d0ee12f..a98e82748c9 100644 --- a/reference/test/solver/fcg_kernels.cpp +++ b/reference/test/solver/fcg_kernels.cpp @@ -291,28 +291,6 @@ TYPED_TEST(Fcg, SolvesStencilSystemComplex) } -TYPED_TEST(Fcg, SolvesStencilSystemMixedComplex) -{ - using value_type = - gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto solver = this->fcg_factory->generate(this->mtx); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.0, -2.0}, value_type{3.0, -6.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(Fcg, SolvesMultipleStencilSystems) { using Mtx = typename TestFixture::Mtx; @@ -388,31 +366,6 @@ TYPED_TEST(Fcg, SolvesStencilSystemUsingAdvancedApplyComplex) } -TYPED_TEST(Fcg, SolvesStencilSystemUsingAdvancedApplyMixedComplex) -{ - using Scalar = gko::matrix::MultiVector< - gko::next_precision>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto solver = this->fcg_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}}, - this->exec); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.5, -3.0}, value_type{5.0, -10.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(Fcg, SolvesMultipleStencilSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/gcr_kernels.cpp b/reference/test/solver/gcr_kernels.cpp index 1a72732c6ab..b933156220f 100644 --- a/reference/test/solver/gcr_kernels.cpp +++ b/reference/test/solver/gcr_kernels.cpp @@ -265,29 +265,6 @@ TYPED_TEST(Gcr, SolvesStencilSystemComplex) } -TYPED_TEST(Gcr, SolvesStencilSystemMixedComplex) -{ - using value_type = - gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto solver = this->gcr_factory->generate(this->mtx); - auto b = - gko::initialize({value_type{13.0, -26.0}, value_type{7.0, -14.0}, - value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.0, -2.0}, value_type{3.0, -6.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(Gcr, SolvesMultipleStencilSystems) { using Mtx = typename TestFixture::Mtx; @@ -364,32 +341,6 @@ TYPED_TEST(Gcr, SolvesStencilSystemUsingAdvancedApplyComplex) } -TYPED_TEST(Gcr, SolvesStencilSystemUsingAdvancedApplyMixedComplex) -{ - using Scalar = gko::matrix::MultiVector< - gko::next_precision>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto solver = this->gcr_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = - gko::initialize({value_type{13.0, -26.0}, value_type{7.0, -14.0}, - value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}}, - this->exec); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.5, -3.0}, value_type{5.0, -10.0}, - value_type{2.0, -4.0}}), - (r_mixed()) * 1e2); -} - - TYPED_TEST(Gcr, SolvesMultipleStencilSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/gmres_kernels.cpp b/reference/test/solver/gmres_kernels.cpp index b7f2b5acdea..59cda4c1f00 100644 --- a/reference/test/solver/gmres_kernels.cpp +++ b/reference/test/solver/gmres_kernels.cpp @@ -483,29 +483,6 @@ TYPED_TEST(Gmres, SolvesStencilSystemComplex) } -TYPED_TEST(Gmres, SolvesStencilSystemMixedComplex) -{ - using value_type = - gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto solver = this->gmres_factory->generate(this->mtx); - auto b = - gko::initialize({value_type{13.0, -26.0}, value_type{7.0, -14.0}, - value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.0, -2.0}, value_type{3.0, -6.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(Gmres, SolvesMultipleStencilSystems) { using Mtx = typename TestFixture::Mtx; @@ -582,32 +559,6 @@ TYPED_TEST(Gmres, SolvesStencilSystemUsingAdvancedApplyComplex) } -TYPED_TEST(Gmres, SolvesStencilSystemUsingAdvancedApplyMixedComplex) -{ - using Scalar = gko::matrix::MultiVector< - gko::next_precision>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto solver = this->gmres_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = - gko::initialize({value_type{13.0, -26.0}, value_type{7.0, -14.0}, - value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}}, - this->exec); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.5, -3.0}, value_type{5.0, -10.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(Gmres, SolvesMultipleStencilSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp index 78b3dcedaa9..c61b2c505aa 100644 --- a/reference/test/solver/idr_kernels.cpp +++ b/reference/test/solver/idr_kernels.cpp @@ -117,28 +117,6 @@ TYPED_TEST(Idr, SolvesMultiVectorSystemComplex) } -TYPED_TEST(Idr, SolvesMultiVectorSystemMixedComplex) -{ - using T = typename TestFixture::value_type; - using value_type = gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto solver = this->idr_factory->generate(this->mtx); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{-4.0, 8.0}, value_type{-1.0, 2.0}, - value_type{4.0, -8.0}}), - (r_mixed()) * 1e1); -} - - TYPED_TEST(Idr, SolvesMultiVectorSystemWithComplexSubSpace) { using Mtx = typename TestFixture::Mtx; @@ -278,31 +256,6 @@ TYPED_TEST(Idr, SolvesMultiVectorSystemUsingAdvancedApplyComplex) } -TYPED_TEST(Idr, SolvesMultiVectorSystemUsingAdvancedApplyMixedComplex) -{ - using Scalar = gko::matrix::MultiVector< - gko::next_precision>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto solver = this->idr_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}}, - this->exec); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{-8.5, 17.0}, value_type{-3.0, 6.0}, - value_type{6.0, -12.0}}), - (r_mixed()) * 1e1); -} - - TYPED_TEST(Idr, SolvesMultipleMultiVectorSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/ir_kernels.cpp b/reference/test/solver/ir_kernels.cpp index 52c545e7d05..dca2a3e1d92 100644 --- a/reference/test/solver/ir_kernels.cpp +++ b/reference/test/solver/ir_kernels.cpp @@ -118,28 +118,6 @@ TYPED_TEST(Ir, SolvesTriangularSystemComplex) } -TYPED_TEST(Ir, SolvesTriangularSystemMixedComplex) -{ - using value_type = - gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto solver = this->ir_factory->generate(this->mtx); - auto b = gko::initialize( - {value_type{3.9, -7.8}, value_type{9.0, -18.0}, value_type{2.2, -4.4}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.0, -2.0}, value_type{3.0, -6.0}, - value_type{2.0, -4.0}}), - (r_mixed()) * 1e1); -} - - TYPED_TEST(Ir, SolvesTriangularSystemWithIterativeInnerSolver) { using Mtx = typename TestFixture::Mtx; @@ -244,34 +222,6 @@ TYPED_TEST(Ir, SolvesTriangularSystemUsingAdvancedApplyComplex) } -TYPED_TEST(Ir, SolvesTriangularSystemUsingAdvancedApplyMixedComplex) -{ - using mixed_type = gko::next_precision; - using mixed_complex_type = gko::to_complex; - using Scalar = gko::matrix::MultiVector; - using MixedMtx = gko::matrix::MultiVector; - auto solver = this->ir_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( - {mixed_complex_type{3.9, -7.8}, mixed_complex_type{9.0, -18.0}, - mixed_complex_type{2.2, -4.4}}, - this->exec); - auto x = gko::initialize( - {mixed_complex_type{0.5, -1.0}, mixed_complex_type{1.0, -2.0}, - mixed_complex_type{2.0, -4.0}}, - this->exec); - - solver->apply(alpha.get(), b.get(), beta.get(), x.get()); - - GKO_ASSERT_MTX_NEAR( - x, - l({mixed_complex_type{1.5, -3.0}, mixed_complex_type{5.0, -10.0}, - mixed_complex_type{2.0, -4.0}}), - (r_mixed()) * 1e1); -} - - TYPED_TEST(Ir, SolvesMultipleStencilSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/lower_trs_kernels.cpp b/reference/test/solver/lower_trs_kernels.cpp index ee705edd14f..2742b22410c 100644 --- a/reference/test/solver/lower_trs_kernels.cpp +++ b/reference/test/solver/lower_trs_kernels.cpp @@ -146,30 +146,6 @@ TYPED_TEST(LowerTrs, SolvesTriangularSystemComplex) } -TYPED_TEST(LowerTrs, SolvesTriangularSystemMixedComplex) -{ - using other_value_type = typename TestFixture::value_type; - using Scalar = - gko::matrix::MultiVector>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - std::shared_ptr b = gko::initialize( - {value_type{1.0, -2.0}, value_type{2.0, -4.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - auto solver = this->lower_trs_factory->generate(this->mtx); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.0, -2.0}, value_type{-1.0, 2.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(LowerTrs, SolvesMultipleTriangularSystems) { using Mtx = typename TestFixture::Mtx; @@ -260,32 +236,6 @@ TYPED_TEST(LowerTrs, SolvesTriangularSystemUsingAdvancedApplyComplex) } -TYPED_TEST(LowerTrs, SolvesTriangularSystemUsingAdvancedApplyMixedComplex) -{ - using other_value_type = typename TestFixture::value_type; - using Scalar = - gko::matrix::MultiVector>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - std::shared_ptr b = gko::initialize( - {value_type{1.0, -2.0}, value_type{2.0, -4.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{1.0, -2.0}, value_type{-1.0, 2.0}, value_type{1.0, -2.0}}, - this->exec); - auto solver = this->lower_trs_factory->generate(this->mtx); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.0, -2.0}, value_type{-1.0, 2.0}, - value_type{3.0, -6.0}}), - (r_mixed())); -} - - TYPED_TEST(LowerTrs, SolvesMultipleTriangularSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/multigrid_kernels.cpp b/reference/test/solver/multigrid_kernels.cpp index af021b1de95..885301bf68d 100644 --- a/reference/test/solver/multigrid_kernels.cpp +++ b/reference/test/solver/multigrid_kernels.cpp @@ -54,13 +54,16 @@ class DummyLinOp : public gko::LinOp, bool apply_uses_initial_guess() const override { return true; } protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { global_step++; } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; @@ -77,14 +80,17 @@ class DummyRestrictOp : public gko::LinOp, bool apply_uses_initial_guess() const override { return true; } protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { rstr_step.push_back(global_step); global_step++; } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} mutable std::vector rstr_step; @@ -104,10 +110,14 @@ class DummyProlongOp : public gko::LinOp, bool apply_uses_initial_guess() const override { return true; } protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override { prlg_step.push_back(global_step); global_step++; @@ -142,14 +152,17 @@ class DummyLinOpWithFactory : public gko::LinOp { std::shared_ptr op_; protected: - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override { step.push_back(global_step); global_step++; } - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override { auto alpha_value = gko::as>(alpha)->at(0, 0); @@ -208,10 +221,14 @@ class DummyMultigridLevelWithFactory std::shared_ptr restrict_; std::shared_ptr prolong_; - void apply_impl(const gko::LinOp* b, gko::LinOp* x) const override {} + void apply_impl(const gko::AbstractMultiVector* b, + gko::AbstractMultiVector* x) const override + {} - void apply_impl(const gko::LinOp* alpha, const gko::LinOp* b, - const gko::LinOp* beta, gko::LinOp* x) const override + void apply_impl(const gko::AbstractMultiVector* alpha, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* beta, + gko::AbstractMultiVector* x) const override {} }; diff --git a/reference/test/solver/pipe_cg_kernels.cpp b/reference/test/solver/pipe_cg_kernels.cpp index c7d5d427555..6295b34322b 100644 --- a/reference/test/solver/pipe_cg_kernels.cpp +++ b/reference/test/solver/pipe_cg_kernels.cpp @@ -414,28 +414,6 @@ TYPED_TEST(PipeCg, SolvesStencilSystemComplex) } -TYPED_TEST(PipeCg, SolvesStencilSystemMixedComplex) -{ - using value_type = - gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto solver = this->pipe_cg_factory->generate(this->mtx); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.0, -2.0}, value_type{3.0, -6.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(PipeCg, SolvesMultipleStencilSystems) { using Mtx = typename TestFixture::Mtx; @@ -511,31 +489,6 @@ TYPED_TEST(PipeCg, SolvesStencilSystemUsingAdvancedApplyComplex) } -TYPED_TEST(PipeCg, SolvesStencilSystemUsingAdvancedApplyMixedComplex) -{ - using Scalar = gko::matrix::MultiVector< - gko::next_precision>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto solver = this->pipe_cg_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( - {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}}, - this->exec); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{1.5, -3.0}, value_type{5.0, -10.0}, - value_type{2.0, -4.0}}), - (r_mixed())); -} - - TYPED_TEST(PipeCg, SolvesMultipleStencilSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/reference/test/solver/upper_trs_kernels.cpp b/reference/test/solver/upper_trs_kernels.cpp index f8561f2a86e..7a4155eb656 100644 --- a/reference/test/solver/upper_trs_kernels.cpp +++ b/reference/test/solver/upper_trs_kernels.cpp @@ -146,30 +146,6 @@ TYPED_TEST(UpperTrs, SolvesTriangularSystemComplex) } -TYPED_TEST(UpperTrs, SolvesTriangularSystemMixedComplex) -{ - using other_value_type = typename TestFixture::value_type; - using Scalar = - gko::matrix::MultiVector>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - std::shared_ptr b = gko::initialize( - {value_type{4.0, -8.0}, value_type{2.0, -4.0}, value_type{3.0, -6.0}}, - this->exec); - auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, - this->exec); - auto solver = this->upper_trs_factory->generate(this->mtx); - - solver->apply(b, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{13.0, -26.0}, value_type{-4.0, 8.0}, - value_type{3.0, -6.0}}), - (r_mixed())); -} - - TYPED_TEST(UpperTrs, SolvesMultipleTriangularSystems) { using Mtx = typename TestFixture::Mtx; @@ -261,32 +237,6 @@ TYPED_TEST(UpperTrs, SolvesTriangularSystemUsingAdvancedApplyComplex) } -TYPED_TEST(UpperTrs, SolvesTriangularSystemUsingAdvancedApplyMixedComplex) -{ - using other_value_type = typename TestFixture::value_type; - using Scalar = - gko::matrix::MultiVector>; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - std::shared_ptr b = gko::initialize( - {value_type{4.0, -8.0}, value_type{2.0, -4.0}, value_type{3.0, -6.0}}, - this->exec); - auto x = gko::initialize( - {value_type{1.0, -2.0}, value_type{-1.0, 2.0}, value_type{1.0, -2.0}}, - this->exec); - auto solver = this->upper_trs_factory->generate(this->mtx); - - solver->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR(x, - l({value_type{25.0, -50.0}, value_type{-7.0, 14.0}, - value_type{5.0, -10.0}}), - (r_mixed())); -} - - TYPED_TEST(UpperTrs, SolvesMultipleTriangularSystemsUsingAdvancedApply) { using Mtx = typename TestFixture::Mtx; diff --git a/test/matrix/csr_kernels2.cpp b/test/matrix/csr_kernels2.cpp index ea9e774a4c1..b14df3a0164 100644 --- a/test/matrix/csr_kernels2.cpp +++ b/test/matrix/csr_kernels2.cpp @@ -36,6 +36,7 @@ class Csr : public CommonTestFixture { protected: using Arr = gko::array; using Vec = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Mtx = gko::matrix::Csr; using ComplexVec = gko::matrix::MultiVector>; using ComplexMtx = gko::matrix::Csr>; @@ -80,9 +81,9 @@ class Csr : public CommonTestFixture { void set_up_apply_data(int num_vectors = 1) { mtx = Mtx::create(ref, strategy); - mtx->move_from(gen_mtx(mtx_size[0], mtx_size[1], 1)); + mtx->move_from(gen_mtx(mtx_size[0], mtx_size[1], 1)); square_mtx = Mtx::create(ref, strategy); - square_mtx->move_from(gen_mtx(mtx_size[0], mtx_size[0], 1)); + square_mtx->move_from(gen_mtx(mtx_size[0], mtx_size[0], 1)); expected = gen_mtx(mtx_size[0], num_vectors, 1); y = gen_mtx(mtx_size[1], num_vectors, 1); alpha = gko::initialize({2.0}, ref); diff --git a/test/matrix/dense_kernels.cpp b/test/matrix/dense_kernels.cpp index 28b01ee604f..da1e90d5447 100644 --- a/test/matrix/dense_kernels.cpp +++ b/test/matrix/dense_kernels.cpp @@ -187,21 +187,6 @@ TEST_F(Dense, ApplyToComplexIsEquivalentToRef) } -TEST_F(Dense, ApplyToMixedComplexIsEquivalentToRef) -{ - set_up_apply_data(); - auto complex_b = gen_mtx(x->get_size()[1], 1); - auto dcomplex_b = gko::clone(exec, complex_b); - auto complex_x = gen_mtx(x->get_size()[0], 1); - auto dcomplex_x = gko::clone(exec, complex_x); - - x->apply(complex_b, complex_x); - dx->apply(dcomplex_b, dcomplex_x); - - GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 2e-7); -} - - TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef) { set_up_apply_data(); @@ -217,23 +202,6 @@ TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef) } -TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef) -{ - set_up_apply_data(); - auto complex_b = gen_mtx(x->get_size()[1], 1); - auto dcomplex_b = gko::clone(exec, complex_b); - auto complex_x = gen_mtx(x->get_size()[0], 1); - auto dcomplex_x = gko::clone(exec, complex_x); - - x->apply(convert(alpha), complex_b, convert(beta), - complex_x); - dx->apply(convert(dalpha), dcomplex_b, convert(dbeta), - dcomplex_x); - - GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 2e-7); -} - - TEST_F(Dense, IsTransposable) { set_up_apply_data(); @@ -599,7 +567,6 @@ TEST_F(Dense, MoveToHybridIsEquivalentToRef) GKO_ASSERT_MTX_NEAR(drmtx, domtx, 0); GKO_ASSERT_MTX_NEAR(srmtx, somtx, 0); - GKO_ASSERT_MTX_NEAR(domtx, omtx, 0); } diff --git a/test/matrix/matrix.cpp b/test/matrix/matrix.cpp index 10d3ce28960..35d2a78be14 100644 --- a/test/matrix/matrix.cpp +++ b/test/matrix/matrix.cpp @@ -792,20 +792,20 @@ class Matrix : public CommonTestFixture { SCOPED_TRACE("Single strided vector"); run_strided(mtx, 1, 2, 3, guarded_fn); } - if (!gko::is_complex()) { - // check application of real matrix to complex vector - // viewed as interleaved real/imag vector - using complex_vec = gko::to_complex; - using complex_out_vec = gko::to_complex; + // check application of real matrix to complex vector + // viewed as interleaved real/imag vector + using complex_vec = gko::to_complex; + if (!gko::is_complex() && + std::is_same_v && + std::is_same_v>) { if (Config::supports_strides()) { SCOPED_TRACE("Single strided complex vector"); - run_strided(mtx, 1, 2, 3, - guarded_fn); + run_strided(mtx, 1, 2, 3, guarded_fn); } if (Config::supports_strides()) { SCOPED_TRACE("Strided complex multivector with 2 columns"); - run_strided(mtx, 2, 3, 4, - guarded_fn); + run_strided(mtx, 2, 3, 4, guarded_fn); } } { diff --git a/test/matrix/multivector_kernels.cpp b/test/matrix/multivector_kernels.cpp index 7a6889ba752..601da6c9d08 100644 --- a/test/matrix/multivector_kernels.cpp +++ b/test/matrix/multivector_kernels.cpp @@ -256,114 +256,6 @@ TEST_F(MultiVector, MultipleVectorComputeNorm2IsEquivalentToRef) } -TEST_F(MultiVector, SimpleApplyIsEquivalentToRef) -{ - set_up_apply_data(); - - x->apply(y, result); - dx->apply(dy, dresult); - - GKO_ASSERT_MTX_NEAR(dresult, result, r::value); -} - - -TEST_F(MultiVector, SimpleApplyMixedIsEquivalentToRef) -{ - set_up_apply_data(); - - x->apply(convert(y), convert(result)); - dx->apply(convert(dy), convert(dresult)); - - GKO_ASSERT_MTX_NEAR(dresult, result, 1e-7); -} - - -TEST_F(MultiVector, AdvancedApplyIsEquivalentToRef) -{ - set_up_apply_data(); - - x->apply(alpha, y, beta, result); - dx->apply(dalpha, dy, dbeta, dresult); - - GKO_ASSERT_MTX_NEAR(dresult, result, r::value); -} - - -TEST_F(MultiVector, AdvancedApplyMixedIsEquivalentToRef) -{ - set_up_apply_data(); - - x->apply(convert(alpha), convert(y), - convert(beta), convert(result)); - dx->apply(convert(dalpha), convert(dy), - convert(dbeta), convert(dresult)); - - GKO_ASSERT_MTX_NEAR(dresult, result, 1e-7); -} - - -TEST_F(MultiVector, ApplyToComplexIsEquivalentToRef) -{ - set_up_apply_data(); - auto complex_b = gen_mtx(x->get_size()[1], 1); - auto dcomplex_b = gko::clone(exec, complex_b); - auto complex_x = gen_mtx(x->get_size()[0], 1); - auto dcomplex_x = gko::clone(exec, complex_x); - - x->apply(complex_b, complex_x); - dx->apply(dcomplex_b, dcomplex_x); - - GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r::value); -} - - -TEST_F(MultiVector, ApplyToMixedComplexIsEquivalentToRef) -{ - set_up_apply_data(); - auto complex_b = gen_mtx(x->get_size()[1], 1); - auto dcomplex_b = gko::clone(exec, complex_b); - auto complex_x = gen_mtx(x->get_size()[0], 1); - auto dcomplex_x = gko::clone(exec, complex_x); - - x->apply(complex_b, complex_x); - dx->apply(dcomplex_b, dcomplex_x); - - GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 2e-7); -} - - -TEST_F(MultiVector, AdvancedApplyToComplexIsEquivalentToRef) -{ - set_up_apply_data(); - auto complex_b = gen_mtx(x->get_size()[1], 1); - auto dcomplex_b = gko::clone(exec, complex_b); - auto complex_x = gen_mtx(x->get_size()[0], 1); - auto dcomplex_x = gko::clone(exec, complex_x); - - x->apply(alpha, complex_b, beta, complex_x); - dx->apply(dalpha, dcomplex_b, dbeta, dcomplex_x); - - GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r::value); -} - - -TEST_F(MultiVector, AdvancedApplyToMixedComplexIsEquivalentToRef) -{ - set_up_apply_data(); - auto complex_b = gen_mtx(x->get_size()[1], 1); - auto dcomplex_b = gko::clone(exec, complex_b); - auto complex_x = gen_mtx(x->get_size()[0], 1); - auto dcomplex_x = gko::clone(exec, complex_x); - - x->apply(convert(alpha), complex_b, convert(beta), - complex_x); - dx->apply(convert(dalpha), dcomplex_b, convert(dbeta), - dcomplex_x); - - GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 2e-7); -} - - TEST_F(MultiVector, ComputeDotComplexIsEquivalentToRef) { set_up_apply_data(); diff --git a/test/mpi/distributed/vector.cpp b/test/mpi/distributed/vector.cpp index bdc13bee80c..84e7eb51af5 100644 --- a/test/mpi/distributed/vector.cpp +++ b/test/mpi/distributed/vector.cpp @@ -1022,8 +1022,8 @@ TYPED_TEST(VectorLocalOps, CreateSubmatrixSameAsLocal) this->comm.all_reduce(this->exec, &global_cols, 1, MPI_SUM); gko::dim<2> global_size{global_rows, global_cols}; - auto rv = this->x->create_submatrix(rows, cols, global_size); - auto local_rv = this->local_x->create_submatrix(rows, cols); + auto rv = this->x->create_subview(rows, cols, global_size); + auto local_rv = this->local_x->create_subview(rows, cols); GKO_ASSERT_EQUAL_DIMENSIONS(rv, global_size); GKO_ASSERT_MTX_NEAR(rv->get_local_vector(), local_rv, 0.0); diff --git a/test/mpi/solver/solver.cpp b/test/mpi/solver/solver.cpp index 0f2ebeb6394..69c9e956d53 100644 --- a/test/mpi/solver/solver.cpp +++ b/test/mpi/solver/solver.cpp @@ -457,10 +457,12 @@ class Solver : public CommonMpiTestFixture { gen_out_vec(part, solver, 17, 21)); #endif } - if (!gko::is_complex()) { - // check application of real matrix to complex vector - // viewed as interleaved real/imag vector - using complex_vec = gko::to_complex; + // check application of real matrix to complex vector + // viewed as interleaved real/imag vector + using complex_vec = gko::to_complex; + if (!gko::is_complex() && + std::is_same_v>) { { SCOPED_TRACE("Single strided complex vector"); guarded_fn(gen_in_vec(part, solver, 1, 2), diff --git a/test/solver/solver.cpp b/test/solver/solver.cpp index 0a8a8b089d3..9069c074c5d 100644 --- a/test/solver/solver.cpp +++ b/test/solver/solver.cpp @@ -523,10 +523,13 @@ struct DummyLogger : gko::log::Logger { DummyLogger() : gko::log::Logger(gko::log::Logger::iteration_complete_mask) {} - void on_iteration_complete(const gko::LinOp* solver, const gko::LinOp* b, - const gko::LinOp* x, const gko::size_type& it, - const gko::LinOp* r, const gko::LinOp* tau, - const gko::LinOp* implicit_tau, + void on_iteration_complete(const gko::LinOp* solver, + const gko::AbstractMultiVector* b, + const gko::AbstractMultiVector* x, + const gko::size_type& it, + const gko::AbstractMultiVector* r, + const gko::AbstractMultiVector* tau, + const gko::AbstractMultiVector* implicit_tau, const gko::array* status, bool all_stopped) const override { @@ -822,10 +825,12 @@ class Solver : public CommonTestFixture { guarded_fn(gen_in_vec(op, 1, 2), gen_out_vec(op, 1, 3)); } - if (!gko::is_complex()) { - // check application of real matrix to complex vector - // viewed as interleaved real/imag vector - using complex_vec = gko::to_complex; + // check application of real matrix to complex vector + // viewed as interleaved real/imag vector + using complex_vec = gko::to_complex; + if (!gko::is_complex() && + std::is_same_v>) { { SCOPED_TRACE("Single strided complex vector"); guarded_fn(gen_in_vec(op, 1, 2), From f975caa6ca250934571700769e2eff65385f97a2 Mon Sep 17 00:00:00 2001 From: Marcel Koch Date: Tue, 18 Aug 2026 09:59:56 +0200 Subject: [PATCH 21/21] remove temporary_conversion::create overload This is unncessary, since it can be recreated by using the run function and the existing create functions. --- core/preconditioner/jacobi.cpp | 13 +-- .../ginkgo/core/base/temporary_conversion.hpp | 90 ------------------- 2 files changed, 8 insertions(+), 95 deletions(-) diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp index c7d329e7c28..bb11cc0caf5 100644 --- a/core/preconditioner/jacobi.cpp +++ b/core/preconditioner/jacobi.cpp @@ -424,11 +424,14 @@ void Jacobi::generate(const LinOp* system_matrix, diag = share(as(system_matrix) ->extract_diagonal_linop()); } - auto diag_vt = ::gko::detail:: - temporary_conversion>::template create< - matrix::Diagonal>, - matrix::Diagonal>, - matrix::Diagonal>>(diag.get()); + auto diag_vt = run, + matrix::Diagonal>, + matrix::Diagonal>, + matrix::Diagonal>>( + diag.get(), [](auto diag_c) { + return temporary_conversion< + matrix::Diagonal>::create(diag_c); + }); if (!diag_vt) { GKO_NOT_SUPPORTED(system_matrix); } diff --git a/include/ginkgo/core/base/temporary_conversion.hpp b/include/ginkgo/core/base/temporary_conversion.hpp index 667f33516ed..49e957a67c9 100644 --- a/include/ginkgo/core/base/temporary_conversion.hpp +++ b/include/ginkgo/core/base/temporary_conversion.hpp @@ -131,78 +131,6 @@ struct conversion_target_helper { }; -/** - * @internal - * - * Helper type that attempts to statically find the dynamic type of a given - * LinOp from a list of ConversionCandidates and, on the first match, converts - * it to TargetType with an appropriate convert_back_deleter. - * - * @tparam ConversionCandidates list of potential dynamic types of the input - * object to be checked. - */ -template -struct conversion_helper { - /** Dispatch convert_impl with the ConversionCandidates list */ - template - static std::unique_ptr> - convert(MaybeConstLinOp* obj) - { - return convert_impl(obj); - } - - /** - * Attempts to cast obj from the first ConversionCandidate and convert it to - * TargetType with a matching convert_back_deleter. If the cast fails, - * recursively tries the remaining conversion candidates. - */ - template - static std::unique_ptr> - convert_impl(MaybeConstLinOp* obj) - { - // make candidate_type conditionally const based on whether obj is const - using candidate_type = - std::conditional_t::value, - const FirstCandidate, FirstCandidate>; - candidate_type* cast_obj{}; - if ((cast_obj = dynamic_cast(obj))) { - // if the cast is successful, obj is of dynamic type candidate_type - // so we can convert from this type to TargetType - auto converted = conversion_target_helper< - std::remove_cv_t>::create_empty(cast_obj); - cast_obj->convert_to(converted); - // Make sure ConvertibleTo is available and symmetric - static_assert( - std::is_base_of>, - FirstCandidate>::value, - "ConvertibleTo not implemented"); - static_assert(std::is_base_of, - TargetType>::value, - "ConvertibleTo not symmetric"); - return {converted.release(), - convert_back_deleter{cast_obj}}; - } else { - // else try the remaining candidates - return conversion_helper::template convert< - TargetType>(obj); - } - } -}; - -template <> -struct conversion_helper<> { - template - static std::unique_ptr> convert( - MaybeConstLinOp* obj) - { - // return nullptr if no previous candidates matched - return {nullptr, null_deleter{}}; - } -}; - - } // namespace detail @@ -233,24 +161,6 @@ class temporary_conversion { using lin_op_type = std::conditional_t::value, const LinOp, LinOp>; - /** - * Create a temporary conversion for a non-temporary LinOp. - * - * @tparam ConversionCandidates list of potential dynamic types of ptr to - * try out for converting ptr to type T. - */ - template - static temporary_conversion create(ptr_param ptr) - { - T* cast_ptr{}; - if ((cast_ptr = dynamic_cast(ptr.get()))) { - return handle_type{cast_ptr, null_deleter{}}; - } else { - return detail::conversion_helper< - ConversionCandidates...>::template convert(ptr.get()); - } - } - /** * Create a temporary conversion from a bare pointer. *