diff --git a/benchmark/sparse_blas/operations.cpp b/benchmark/sparse_blas/operations.cpp index 9d4329048e0..2849c119620 100644 --- a/benchmark/sparse_blas/operations.cpp +++ b/benchmark/sparse_blas/operations.cpp @@ -9,6 +9,8 @@ #include +#include + #include "core/base/array_access.hpp" #include "core/factorization/elimination_forest.hpp" #include "core/factorization/factorization_kernels.hpp" diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake index 4d670240ec0..d4387edeb8f 100644 --- a/cmake/create_test.cmake +++ b/cmake/create_test.cmake @@ -296,11 +296,15 @@ function(ginkgo_create_cuda_test_internal test_name filename test_target_name) --expt-relaxed-constexpr> ) elseif(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA") + # 997: remove false positive warning about overloading virtual functions target_compile_options( ${test_target_name} PRIVATE $<$:--expt-extended-lambda - --expt-relaxed-constexpr> + --expt-relaxed-constexpr + --diag-suppress + 997 + > ) endif() ginkgo_set_test_target_properties(${test_target_name} "_cuda" ${ARGN}) diff --git a/common/cuda_hip/CMakeLists.txt b/common/cuda_hip/CMakeLists.txt index 3fd01b7cbc0..38c6e6fb7fc 100644 --- a/common/cuda_hip/CMakeLists.txt +++ b/common/cuda_hip/CMakeLists.txt @@ -30,6 +30,7 @@ set(CUDA_HIP_SOURCES matrix/batch_dense_kernels.cpp matrix/batch_ell_kernels.cpp matrix/coo_kernels.cpp + matrix/dense_kernels.cpp matrix/diagonal_kernels.cpp matrix/ell_kernels.cpp matrix/multivector_kernels.cpp diff --git a/common/cuda_hip/factorization/lu_kernels.cpp b/common/cuda_hip/factorization/lu_kernels.cpp index 10fbd3179f1..47860ca6d3a 100644 --- a/common/cuda_hip/factorization/lu_kernels.cpp +++ b/common/cuda_hip/factorization/lu_kernels.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "common/cuda_hip/base/thrust.hpp" #include "common/cuda_hip/base/types.hpp" diff --git a/common/cuda_hip/matrix/dense_kernels.cpp b/common/cuda_hip/matrix/dense_kernels.cpp new file mode 100644 index 00000000000..d677538d3ad --- /dev/null +++ b/common/cuda_hip/matrix/dense_kernels.cpp @@ -0,0 +1,708 @@ +// SPDX-FileCopyrightText: 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include "core/matrix/dense_kernels.hpp" + +#include + +#include "common/cuda_hip/base/blas_bindings.hpp" +#include "common/cuda_hip/base/config.hpp" +#include "common/cuda_hip/base/pointer_mode_guard.hpp" +#include "common/cuda_hip/components/cooperative_groups.hpp" +#include "common/cuda_hip/components/intrinsics.hpp" +#include "common/cuda_hip/components/reduction.hpp" +#include "common/cuda_hip/components/thread_ids.hpp" +#include "common/cuda_hip/components/uninitialized_array.hpp" +#include "core/base/utils.hpp" +#include "core/components/prefix_sum_kernels.hpp" +#include "core/matrix/multivector_kernels.hpp" + + +namespace gko { +namespace kernels { +namespace GKO_DEVICE_NAMESPACE { +/** + * @brief The dense matrix format namespace. + * + * @ingroup dense + */ +namespace dense { + + +constexpr int default_block_size = 512; + + +namespace kernel { + + +template +__global__ +__launch_bounds__(default_block_size) void count_nonzero_blocks_per_row( + size_type num_block_rows, size_type num_block_cols, size_type stride, + int block_size, const ValueType* __restrict__ source, + IndexType* __restrict__ block_row_nnz) +{ + const auto brow = + thread::get_subwarp_id_flat(); + + if (brow >= num_block_rows) { + return; + } + + const auto num_cols = num_block_cols * block_size; + auto warp = + group::tiled_partition(group::this_thread_block()); + const auto lane = static_cast(warp.thread_rank()); + constexpr auto full_mask = ~config::lane_mask_type{}; + constexpr auto one_mask = config::lane_mask_type{1}; + bool first_block_nonzero = false; + IndexType block_count{}; + for (IndexType base_col = 0; base_col < num_cols; + base_col += config::warp_size) { + const auto col = base_col + lane; + const auto block_local_col = col % block_size; + // which is the first column in the current block? + const auto block_base_col = col - block_local_col; + // collect nonzero bitmask + bool local_nonzero = false; + for (int local_row = 0; local_row < block_size; local_row++) { + const auto row = local_row + brow * block_size; + local_nonzero |= + col < num_cols && is_nonzero(source[row * stride + col]); + } + auto nonzero_mask = group::ballot(warp, local_nonzero) | + (first_block_nonzero ? 1u : 0u); + // only consider threads in the current block + const auto first_thread = block_base_col - base_col; + const auto last_thread = first_thread + block_size; + // HIP compiles these assertions in Release, traps unconditionally + // assert(first_thread < int(config::warp_size)); + // assert(last_thread >= 0); + // mask off everything below first_thread + const auto lower_mask = + first_thread < 0 ? full_mask : ~((one_mask << first_thread) - 1u); + // mask off everything from last_thread + const auto upper_mask = last_thread >= config::warp_size + ? full_mask + : ((one_mask << last_thread) - 1u); + const auto block_mask = upper_mask & lower_mask; + const auto local_mask = nonzero_mask & block_mask; + // last column in the block increments the counter + block_count += + (block_local_col == block_size - 1 && local_mask) ? 1 : 0; + // if we need to store something for the next iteration + if ((base_col + config::warp_size) % block_size != 0) { + // check whether the last block (incomplete) in this warp is nonzero + auto local_block_nonzero_mask = + group::ballot(warp, local_mask != 0u); + bool last_block_nonzero = + (local_block_nonzero_mask >> (config::warp_size - 1)) != 0u; + first_block_nonzero = last_block_nonzero; + } else { + first_block_nonzero = false; + } + } + block_count = reduce(warp, block_count, + [](IndexType a, IndexType b) { return a + b; }); + if (lane == 0) { + block_row_nnz[brow] = block_count; + } +} + + +template +__global__ __launch_bounds__(default_block_size) void convert_to_fbcsr( + size_type num_block_rows, size_type num_block_cols, size_type stride, + int block_size, const ValueType* __restrict__ source, + const IndexType* __restrict__ block_row_ptrs, + IndexType* __restrict__ block_cols, ValueType* __restrict__ blocks) +{ + const auto brow = + thread::get_subwarp_id_flat(); + + if (brow >= num_block_rows) { + return; + } + + const auto bs_sq = block_size * block_size; + const auto num_cols = num_block_cols * block_size; + auto warp = + group::tiled_partition(group::this_thread_block()); + const auto lane = static_cast(warp.thread_rank()); + constexpr auto full_mask = ~config::lane_mask_type{}; + constexpr auto one_mask = config::lane_mask_type{1}; + const auto lane_prefix_mask = (one_mask << warp.thread_rank()) - 1u; + bool first_block_nonzero = false; + auto block_base_nz = block_row_ptrs[brow]; + for (IndexType base_col = 0; base_col < num_cols; + base_col += config::warp_size) { + const auto col = base_col + lane; + const auto block_local_col = col % block_size; + // which is the first column in the current block? + const auto block_base_col = col - block_local_col; + // collect nonzero bitmask + bool local_nonzero = false; + for (int local_row = 0; local_row < block_size; local_row++) { + const auto row = local_row + brow * block_size; + local_nonzero |= + col < num_cols && is_nonzero(source[row * stride + col]); + } + auto nonzero_mask = group::ballot(warp, local_nonzero) | + (first_block_nonzero ? 1u : 0u); + // only consider threads in the current block + const auto first_thread = block_base_col - base_col; + const auto last_thread = first_thread + block_size; + // HIP compiles these assertions in Release, traps unconditionally + // assert(first_thread < int(config::warp_size)); + // assert(last_thread >= 0); + // mask off everything below first_thread + const auto lower_mask = + first_thread < 0 ? full_mask : ~((one_mask << first_thread) - 1u); + // mask off everything from last_thread + const auto upper_mask = last_thread >= config::warp_size + ? full_mask + : ((one_mask << last_thread) - 1u); + const auto block_mask = upper_mask & lower_mask; + const auto local_mask = nonzero_mask & block_mask; + const auto block_nonzero_mask = group::ballot( + warp, local_mask && (block_local_col == block_size - 1)); + + // count how many Fbcsr blocks come before the Fbcsr block handled by + // the local group of threads + const auto block_nz = + block_base_nz + popcnt(block_nonzero_mask & lane_prefix_mask); + // now in a second sweep, store the actual elements + if (local_mask) { + if (block_local_col == block_size - 1) { + block_cols[block_nz] = col / block_size; + } + // only if we encountered elements in this column + if (local_nonzero) { + for (int local_row = 0; local_row < block_size; local_row++) { + const auto row = local_row + brow * block_size; + blocks[local_row + block_local_col * block_size + + block_nz * bs_sq] = source[row * stride + col]; + } + } + } + // if we need to store something for the next iteration + if ((base_col + config::warp_size) % block_size != 0) { + // check whether the last block (incomplete) in this warp is nonzero + auto local_block_nonzero_mask = + group::ballot(warp, local_mask != 0u); + bool last_block_nonzero = + (local_block_nonzero_mask >> (config::warp_size - 1)) != 0u; + first_block_nonzero = last_block_nonzero; + } else { + first_block_nonzero = false; + } + // advance by the completed blocks + block_base_nz += popcnt(block_nonzero_mask); + } +} + + +template +__global__ __launch_bounds__(default_block_size) void fill_in_coo( + size_type num_rows, size_type num_cols, size_type stride, + const ValueType* __restrict__ source, const int64* __restrict__ row_ptrs, + IndexType* __restrict__ row_idxs, IndexType* __restrict__ col_idxs, + ValueType* __restrict__ values) +{ + const auto row = thread::get_subwarp_id_flat(); + + if (row < num_rows) { + auto warp = group::tiled_partition( + group::this_thread_block()); + auto lane_prefix_mask = + (config::lane_mask_type(1) << warp.thread_rank()) - 1; + auto base_out_idx = row_ptrs[row]; + for (size_type i = 0; i < num_cols; i += config::warp_size) { + const auto col = i + warp.thread_rank(); + const auto pred = + col < num_cols ? is_nonzero(source[stride * row + col]) : false; + const auto mask = group::ballot(warp, pred); + const auto out_idx = base_out_idx + popcnt(mask & lane_prefix_mask); + if (pred) { + values[out_idx] = source[stride * row + col]; + col_idxs[out_idx] = col; + row_idxs[out_idx] = row; + } + base_out_idx += popcnt(mask); + } + } +} + + +template +__global__ __launch_bounds__(default_block_size) void fill_in_csr( + size_type num_rows, size_type num_cols, size_type stride, + const ValueType* __restrict__ source, IndexType* __restrict__ row_ptrs, + IndexType* __restrict__ col_idxs, ValueType* __restrict__ values) +{ + const auto row = thread::get_subwarp_id_flat(); + + if (row < num_rows) { + auto warp = group::tiled_partition( + group::this_thread_block()); + auto lane_prefix_mask = + (config::lane_mask_type(1) << warp.thread_rank()) - 1; + auto base_out_idx = row_ptrs[row]; + for (size_type i = 0; i < num_cols; i += config::warp_size) { + const auto col = i + warp.thread_rank(); + const auto pred = + col < num_cols ? is_nonzero(source[stride * row + col]) : false; + const auto mask = group::ballot(warp, pred); + const auto out_idx = base_out_idx + popcnt(mask & lane_prefix_mask); + if (pred) { + values[out_idx] = source[stride * row + col]; + col_idxs[out_idx] = col; + } + base_out_idx += popcnt(mask); + } + } +} + + +template +__global__ __launch_bounds__(default_block_size) void fill_in_sparsity_csr( + size_type num_rows, size_type num_cols, size_type stride, + const ValueType* __restrict__ source, IndexType* __restrict__ row_ptrs, + IndexType* __restrict__ col_idxs) +{ + const auto row = thread::get_subwarp_id_flat(); + + if (row < num_rows) { + auto warp = group::tiled_partition( + group::this_thread_block()); + auto lane_prefix_mask = + (config::lane_mask_type(1) << warp.thread_rank()) - 1; + auto base_out_idx = row_ptrs[row]; + for (size_type i = 0; i < num_cols; i += config::warp_size) { + const auto col = i + warp.thread_rank(); + const auto pred = + col < num_cols ? is_nonzero(source[stride * row + col]) : false; + const auto mask = group::ballot(warp, pred); + const auto out_idx = base_out_idx + popcnt(mask & lane_prefix_mask); + if (pred) { + col_idxs[out_idx] = col; + } + base_out_idx += popcnt(mask); + } + } +} + + +template +__global__ __launch_bounds__(default_block_size) void fill_in_ell( + size_type num_rows, size_type num_cols, size_type source_stride, + const ValueType* __restrict__ source, size_type max_nnz_per_row, + size_type result_stride, IndexType* __restrict__ col_idxs, + ValueType* __restrict__ values) +{ + const auto row = thread::get_subwarp_id_flat(); + + if (row < num_rows) { + auto warp = group::tiled_partition( + group::this_thread_block()); + auto lane_prefix_mask = + (config::lane_mask_type(1) << warp.thread_rank()) - 1; + size_type base_out_idx{}; + for (size_type i = 0; i < num_cols; i += config::warp_size) { + const auto col = i + warp.thread_rank(); + const auto pred = + col < num_cols ? is_nonzero(source[source_stride * row + col]) + : false; + const auto mask = group::ballot(warp, pred); + const auto out_idx = + row + (base_out_idx + popcnt(mask & lane_prefix_mask)) * + result_stride; + if (pred) { + values[out_idx] = source[source_stride * row + col]; + col_idxs[out_idx] = col; + } + base_out_idx += popcnt(mask); + } + for (size_type i = base_out_idx + warp.thread_rank(); + i < max_nnz_per_row; i += config::warp_size) { + const auto out_idx = row + i * result_stride; + values[out_idx] = zero(); + col_idxs[out_idx] = invalid_index(); + } + } +} + + +template +__global__ __launch_bounds__(default_block_size) void fill_in_hybrid( + size_type num_rows, size_type num_cols, size_type source_stride, + const ValueType* __restrict__ source, size_type ell_max_nnz_per_row, + size_type ell_stride, IndexType* __restrict__ ell_col_idxs, + ValueType* __restrict__ ell_values, const int64* __restrict__ coo_row_ptrs, + IndexType* __restrict__ coo_row_idxs, IndexType* __restrict__ coo_col_idxs, + ValueType* __restrict__ coo_values) +{ + const auto row = thread::get_subwarp_id_flat(); + + if (row < num_rows) { + auto warp = group::tiled_partition( + group::this_thread_block()); + auto lane_prefix_mask = + (config::lane_mask_type(1) << warp.thread_rank()) - 1; + size_type base_out_idx{}; + const auto coo_out_begin = coo_row_ptrs[row]; + for (size_type i = 0; i < num_cols; i += config::warp_size) { + const auto col = i + warp.thread_rank(); + const auto pred = + col < num_cols ? is_nonzero(source[source_stride * row + col]) + : false; + const auto mask = group::ballot(warp, pred); + const auto cur_out_idx = + base_out_idx + popcnt(mask & lane_prefix_mask); + if (pred) { + if (cur_out_idx < ell_max_nnz_per_row) { + const auto out_idx = row + cur_out_idx * ell_stride; + ell_values[out_idx] = source[source_stride * row + col]; + ell_col_idxs[out_idx] = col; + } else { + const auto out_idx = + cur_out_idx - ell_max_nnz_per_row + coo_out_begin; + coo_values[out_idx] = source[source_stride * row + col]; + coo_col_idxs[out_idx] = col; + coo_row_idxs[out_idx] = row; + } + } + base_out_idx += popcnt(mask); + } + for (size_type i = base_out_idx + warp.thread_rank(); + i < ell_max_nnz_per_row; i += config::warp_size) { + const auto out_idx = row + i * ell_stride; + ell_values[out_idx] = zero(); + ell_col_idxs[out_idx] = invalid_index(); + } + } +} + + +template +__global__ __launch_bounds__(default_block_size) void fill_in_sellp( + size_type num_rows, size_type num_cols, size_type slice_size, + size_type stride, const ValueType* __restrict__ source, + size_type* __restrict__ slice_sets, IndexType* __restrict__ col_idxs, + ValueType* __restrict__ values) +{ + const auto row = thread::get_subwarp_id_flat(); + const auto local_row = row % slice_size; + const auto slice = row / slice_size; + + if (row < num_rows) { + auto warp = group::tiled_partition( + group::this_thread_block()); + const auto lane = warp.thread_rank(); + const auto prefix_mask = (config::lane_mask_type{1} << lane) - 1; + const auto slice_end = slice_sets[slice + 1] * slice_size; + auto base_idx = slice_sets[slice] * slice_size + local_row; + for (size_type i = 0; i < num_cols; i += config::warp_size) { + const auto col = i + lane; + const auto val = checked_load(source + stride * row, col, num_cols, + zero()); + const auto pred = is_nonzero(val); + const auto mask = group::ballot(warp, pred); + const auto idx = base_idx + popcnt(mask & prefix_mask) * slice_size; + if (pred) { + values[idx] = val; + col_idxs[idx] = col; + } + base_idx += popcnt(mask) * slice_size; + } + for (auto i = base_idx + lane * slice_size; i < slice_end; + i += config::warp_size * slice_size) { + values[i] = zero(); + col_idxs[i] = invalid_index(); + } + } +} + + +} // namespace kernel + + +template +void simple_apply(std::shared_ptr exec, + matrix::view::dense a, + matrix::view::dense b, + matrix::view::dense c) +{ + if (blas::is_supported::value) { + auto handle = exec->get_blas_handle(); + if (c.size[0] > 0 && c.size[1] > 0) { + if (a.size[1] > 0) { + blas::pointer_mode_guard pm_guard(handle); + auto alpha = one(); + auto beta = zero(); + blas::gemm(handle, BLAS_OP_N, BLAS_OP_N, c.size[1], c.size[0], + a.size[1], &alpha, b.values, b.stride, a.values, + a.stride, &beta, c.values, c.stride); + } else { + multivector::fill(exec, c, zero()); + } + } + } else { + GKO_NOT_IMPLEMENTED; + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL); + + +template +void apply(std::shared_ptr exec, + matrix::view::dense alpha, + matrix::view::dense a, + matrix::view::dense b, + matrix::view::dense beta, + matrix::view::dense c) +{ + if (blas::is_supported::value) { + if (c.size[0] > 0 && c.size[1] > 0) { + if (a.size[1] > 0) { + blas::gemm(exec->get_blas_handle(), BLAS_OP_N, BLAS_OP_N, + c.size[1], c.size[0], a.size[1], alpha.values, + b.values, b.stride, a.values, a.stride, beta.values, + c.values, c.stride); + } else { + multivector::scale(exec, beta, c); + } + } + } else { + GKO_NOT_IMPLEMENTED; + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL); + + +template +void convert_to_coo(std::shared_ptr exec, + matrix::view::dense source, + const int64* row_ptrs, + matrix::view::coo result) +{ + auto num_rows = result.size[0]; + auto num_cols = result.size[1]; + + auto row_idxs = result.row_idxs; + auto col_idxs = result.col_idxs; + auto values = result.values; + + auto stride = source.stride; + + const auto grid_dim = + ceildiv(num_rows, default_block_size / config::warp_size); + if (grid_dim > 0) { + kernel::fill_in_coo<<get_stream()>>>( + num_rows, num_cols, stride, as_device_type(source.values), row_ptrs, + row_idxs, col_idxs, as_device_type(values)); + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL); + + +template +void convert_to_csr(std::shared_ptr exec, + matrix::view::dense source, + matrix::Csr* result) +{ + auto num_rows = result->get_size()[0]; + auto num_cols = result->get_size()[1]; + + auto row_ptrs = result->get_row_ptrs(); + auto col_idxs = result->get_col_idxs(); + auto values = result->get_values(); + + auto stride = source.stride; + + const auto grid_dim = + ceildiv(num_rows, default_block_size / config::warp_size); + if (grid_dim > 0) { + kernel::fill_in_csr<<get_stream()>>>( + num_rows, num_cols, stride, as_device_type(source.values), + as_device_type(row_ptrs), as_device_type(col_idxs), + as_device_type(values)); + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL); + + +template +void convert_to_ell(std::shared_ptr exec, + matrix::view::dense source, + matrix::view::ell result) +{ + auto num_rows = result.size[0]; + auto num_cols = result.size[1]; + auto max_nnz_per_row = result.num_stored_elements_per_row; + + auto col_idxs = result.col_idxs; + auto values = result.values; + + auto source_stride = source.stride; + auto result_stride = result.stride; + + const auto grid_dim = + ceildiv(num_rows, default_block_size / config::warp_size); + if (grid_dim > 0) { + kernel::fill_in_ell<<get_stream()>>>( + num_rows, num_cols, source_stride, as_device_type(source.values), + max_nnz_per_row, result_stride, col_idxs, as_device_type(values)); + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL); + + +template +void convert_to_fbcsr(std::shared_ptr exec, + matrix::view::dense source, + matrix::Fbcsr* result) +{ + const auto num_block_rows = result->get_num_block_rows(); + if (num_block_rows > 0) { + const auto num_blocks = + ceildiv(num_block_rows, default_block_size / config::warp_size); + kernel::convert_to_fbcsr<<get_stream()>>>( + num_block_rows, result->get_num_block_cols(), source.stride, + result->get_block_size(), as_device_type(source.values), + result->get_const_row_ptrs(), result->get_col_idxs(), + as_device_type(result->get_values())); + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_FBCSR_KERNEL); + + +template +void count_nonzero_blocks_per_row(std::shared_ptr exec, + matrix::view::dense source, + int bs, IndexType* result) +{ + const auto num_block_rows = source.size[0] / bs; + const auto num_block_cols = source.size[1] / bs; + if (num_block_rows > 0) { + const auto num_blocks = + ceildiv(num_block_rows, default_block_size / config::warp_size); + kernel::count_nonzero_blocks_per_row<<get_stream()>>>( + num_block_rows, num_block_cols, source.stride, bs, + as_device_type(source.values), result); + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL); + + +template +void convert_to_hybrid(std::shared_ptr exec, + matrix::view::dense source, + const int64* coo_row_ptrs, + matrix::view::hybrid result) +{ + auto num_rows = result.size[0]; + auto num_cols = result.size[1]; + auto ell_max_nnz_per_row = result.ell_part.num_stored_elements_per_row; + const auto source_stride = source.stride; + const auto ell_stride = result.ell_part.stride; + auto ell_col_idxs = result.ell_part.col_idxs; + auto ell_values = result.ell_part.values; + auto coo_row_idxs = result.coo_part.row_idxs; + auto coo_col_idxs = result.coo_part.col_idxs; + auto coo_values = result.coo_part.values; + + auto grid_dim = ceildiv(num_rows, default_block_size / config::warp_size); + if (grid_dim > 0) { + kernel::fill_in_hybrid<<get_stream()>>>( + num_rows, num_cols, source_stride, as_device_type(source.values), + ell_max_nnz_per_row, ell_stride, ell_col_idxs, + as_device_type(ell_values), coo_row_ptrs, coo_row_idxs, + coo_col_idxs, as_device_type(coo_values)); + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL); + + +template +void convert_to_sellp(std::shared_ptr exec, + matrix::view::dense source, + matrix::view::sellp result) +{ + const auto stride = source.stride; + const auto num_rows = result.size[0]; + const auto num_cols = result.size[1]; + + auto vals = result.values; + auto col_idxs = result.col_idxs; + auto slice_sets = result.slice_sets; + + const auto slice_size = result.slice_size; + const auto stride_factor = result.stride_factor; + + auto grid_dim = ceildiv(num_rows, default_block_size / config::warp_size); + if (grid_dim > 0) { + kernel::fill_in_sellp<<get_stream()>>>( + num_rows, num_cols, slice_size, stride, + as_device_type(source.values), as_device_type(slice_sets), + as_device_type(col_idxs), as_device_type(vals)); + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL); + + +template +void convert_to_sparsity_csr(std::shared_ptr exec, + matrix::view::dense source, + matrix::SparsityCsr* result) +{ + auto num_rows = result->get_size()[0]; + auto num_cols = result->get_size()[1]; + + auto row_ptrs = result->get_row_ptrs(); + auto col_idxs = result->get_col_idxs(); + + auto stride = source.stride; + + const auto grid_dim = + ceildiv(num_rows, default_block_size / config::warp_size); + if (grid_dim > 0) { + kernel::fill_in_sparsity_csr<<get_stream()>>>( + num_rows, num_cols, stride, as_device_type(source.values), + as_device_type(row_ptrs), as_device_type(col_idxs)); + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL); + + +} // namespace dense +} // namespace GKO_DEVICE_NAMESPACE +} // namespace kernels +} // namespace gko diff --git a/common/cuda_hip/matrix/multivector_kernels.cpp b/common/cuda_hip/matrix/multivector_kernels.cpp index 5d10bb2ea5b..54e93e2f6c4 100644 --- a/common/cuda_hip/matrix/multivector_kernels.cpp +++ b/common/cuda_hip/matrix/multivector_kernels.cpp @@ -5,13 +5,6 @@ #include "core/matrix/multivector_kernels.hpp" #include -#include -#include -#include -#include -#include -#include -#include #include "common/cuda_hip/base/blas_bindings.hpp" #include "common/cuda_hip/base/config.hpp" @@ -23,7 +16,6 @@ #include "common/cuda_hip/components/thread_ids.hpp" #include "common/cuda_hip/components/uninitialized_array.hpp" #include "core/base/utils.hpp" -#include "core/components/prefix_sum_kernels.hpp" namespace gko { @@ -37,624 +29,6 @@ namespace GKO_DEVICE_NAMESPACE { namespace multivector { -constexpr int default_block_size = 512; - - -namespace kernel { - - -template -__global__ -__launch_bounds__(default_block_size) void count_nonzero_blocks_per_row( - size_type num_block_rows, size_type num_block_cols, size_type stride, - int block_size, const ValueType* __restrict__ source, - IndexType* __restrict__ block_row_nnz) -{ - const auto brow = - thread::get_subwarp_id_flat(); - - if (brow >= num_block_rows) { - return; - } - - const auto num_cols = num_block_cols * block_size; - auto warp = - group::tiled_partition(group::this_thread_block()); - const auto lane = static_cast(warp.thread_rank()); - constexpr auto full_mask = ~config::lane_mask_type{}; - constexpr auto one_mask = config::lane_mask_type{1}; - bool first_block_nonzero = false; - IndexType block_count{}; - for (IndexType base_col = 0; base_col < num_cols; - base_col += config::warp_size) { - const auto col = base_col + lane; - const auto block_local_col = col % block_size; - // which is the first column in the current block? - const auto block_base_col = col - block_local_col; - // collect nonzero bitmask - bool local_nonzero = false; - for (int local_row = 0; local_row < block_size; local_row++) { - const auto row = local_row + brow * block_size; - local_nonzero |= - col < num_cols && is_nonzero(source[row * stride + col]); - } - auto nonzero_mask = group::ballot(warp, local_nonzero) | - (first_block_nonzero ? 1u : 0u); - // only consider threads in the current block - const auto first_thread = block_base_col - base_col; - const auto last_thread = first_thread + block_size; - // HIP compiles these assertions in Release, traps unconditionally - // assert(first_thread < int(config::warp_size)); - // assert(last_thread >= 0); - // mask off everything below first_thread - const auto lower_mask = - first_thread < 0 ? full_mask : ~((one_mask << first_thread) - 1u); - // mask off everything from last_thread - const auto upper_mask = last_thread >= config::warp_size - ? full_mask - : ((one_mask << last_thread) - 1u); - const auto block_mask = upper_mask & lower_mask; - const auto local_mask = nonzero_mask & block_mask; - // last column in the block increments the counter - block_count += - (block_local_col == block_size - 1 && local_mask) ? 1 : 0; - // if we need to store something for the next iteration - if ((base_col + config::warp_size) % block_size != 0) { - // check whether the last block (incomplete) in this warp is nonzero - auto local_block_nonzero_mask = - group::ballot(warp, local_mask != 0u); - bool last_block_nonzero = - (local_block_nonzero_mask >> (config::warp_size - 1)) != 0u; - first_block_nonzero = last_block_nonzero; - } else { - first_block_nonzero = false; - } - } - block_count = reduce(warp, block_count, - [](IndexType a, IndexType b) { return a + b; }); - if (lane == 0) { - block_row_nnz[brow] = block_count; - } -} - - -template -__global__ __launch_bounds__(default_block_size) void convert_to_fbcsr( - size_type num_block_rows, size_type num_block_cols, size_type stride, - int block_size, const ValueType* __restrict__ source, - const IndexType* __restrict__ block_row_ptrs, - IndexType* __restrict__ block_cols, ValueType* __restrict__ blocks) -{ - const auto brow = - thread::get_subwarp_id_flat(); - - if (brow >= num_block_rows) { - return; - } - - const auto bs_sq = block_size * block_size; - const auto num_cols = num_block_cols * block_size; - auto warp = - group::tiled_partition(group::this_thread_block()); - const auto lane = static_cast(warp.thread_rank()); - constexpr auto full_mask = ~config::lane_mask_type{}; - constexpr auto one_mask = config::lane_mask_type{1}; - const auto lane_prefix_mask = (one_mask << warp.thread_rank()) - 1u; - bool first_block_nonzero = false; - auto block_base_nz = block_row_ptrs[brow]; - for (IndexType base_col = 0; base_col < num_cols; - base_col += config::warp_size) { - const auto col = base_col + lane; - const auto block_local_col = col % block_size; - // which is the first column in the current block? - const auto block_base_col = col - block_local_col; - // collect nonzero bitmask - bool local_nonzero = false; - for (int local_row = 0; local_row < block_size; local_row++) { - const auto row = local_row + brow * block_size; - local_nonzero |= - col < num_cols && is_nonzero(source[row * stride + col]); - } - auto nonzero_mask = group::ballot(warp, local_nonzero) | - (first_block_nonzero ? 1u : 0u); - // only consider threads in the current block - const auto first_thread = block_base_col - base_col; - const auto last_thread = first_thread + block_size; - // HIP compiles these assertions in Release, traps unconditionally - // assert(first_thread < int(config::warp_size)); - // assert(last_thread >= 0); - // mask off everything below first_thread - const auto lower_mask = - first_thread < 0 ? full_mask : ~((one_mask << first_thread) - 1u); - // mask off everything from last_thread - const auto upper_mask = last_thread >= config::warp_size - ? full_mask - : ((one_mask << last_thread) - 1u); - const auto block_mask = upper_mask & lower_mask; - const auto local_mask = nonzero_mask & block_mask; - const auto block_nonzero_mask = group::ballot( - warp, local_mask && (block_local_col == block_size - 1)); - - // count how many Fbcsr blocks come before the Fbcsr block handled by - // the local group of threads - const auto block_nz = - block_base_nz + popcnt(block_nonzero_mask & lane_prefix_mask); - // now in a second sweep, store the actual elements - if (local_mask) { - if (block_local_col == block_size - 1) { - block_cols[block_nz] = col / block_size; - } - // only if we encountered elements in this column - if (local_nonzero) { - for (int local_row = 0; local_row < block_size; local_row++) { - const auto row = local_row + brow * block_size; - blocks[local_row + block_local_col * block_size + - block_nz * bs_sq] = source[row * stride + col]; - } - } - } - // if we need to store something for the next iteration - if ((base_col + config::warp_size) % block_size != 0) { - // check whether the last block (incomplete) in this warp is nonzero - auto local_block_nonzero_mask = - group::ballot(warp, local_mask != 0u); - bool last_block_nonzero = - (local_block_nonzero_mask >> (config::warp_size - 1)) != 0u; - first_block_nonzero = last_block_nonzero; - } else { - first_block_nonzero = false; - } - // advance by the completed blocks - block_base_nz += popcnt(block_nonzero_mask); - } -} - - -template -__global__ __launch_bounds__(default_block_size) void fill_in_coo( - size_type num_rows, size_type num_cols, size_type stride, - const ValueType* __restrict__ source, const int64* __restrict__ row_ptrs, - IndexType* __restrict__ row_idxs, IndexType* __restrict__ col_idxs, - ValueType* __restrict__ values) -{ - const auto row = thread::get_subwarp_id_flat(); - - if (row < num_rows) { - auto warp = group::tiled_partition( - group::this_thread_block()); - auto lane_prefix_mask = - (config::lane_mask_type(1) << warp.thread_rank()) - 1; - auto base_out_idx = row_ptrs[row]; - for (size_type i = 0; i < num_cols; i += config::warp_size) { - const auto col = i + warp.thread_rank(); - const auto pred = - col < num_cols ? is_nonzero(source[stride * row + col]) : false; - const auto mask = group::ballot(warp, pred); - const auto out_idx = base_out_idx + popcnt(mask & lane_prefix_mask); - if (pred) { - values[out_idx] = source[stride * row + col]; - col_idxs[out_idx] = col; - row_idxs[out_idx] = row; - } - base_out_idx += popcnt(mask); - } - } -} - - -template -__global__ __launch_bounds__(default_block_size) void fill_in_csr( - size_type num_rows, size_type num_cols, size_type stride, - const ValueType* __restrict__ source, IndexType* __restrict__ row_ptrs, - IndexType* __restrict__ col_idxs, ValueType* __restrict__ values) -{ - const auto row = thread::get_subwarp_id_flat(); - - if (row < num_rows) { - auto warp = group::tiled_partition( - group::this_thread_block()); - auto lane_prefix_mask = - (config::lane_mask_type(1) << warp.thread_rank()) - 1; - auto base_out_idx = row_ptrs[row]; - for (size_type i = 0; i < num_cols; i += config::warp_size) { - const auto col = i + warp.thread_rank(); - const auto pred = - col < num_cols ? is_nonzero(source[stride * row + col]) : false; - const auto mask = group::ballot(warp, pred); - const auto out_idx = base_out_idx + popcnt(mask & lane_prefix_mask); - if (pred) { - values[out_idx] = source[stride * row + col]; - col_idxs[out_idx] = col; - } - base_out_idx += popcnt(mask); - } - } -} - - -template -__global__ __launch_bounds__(default_block_size) void fill_in_sparsity_csr( - size_type num_rows, size_type num_cols, size_type stride, - const ValueType* __restrict__ source, IndexType* __restrict__ row_ptrs, - IndexType* __restrict__ col_idxs) -{ - const auto row = thread::get_subwarp_id_flat(); - - if (row < num_rows) { - auto warp = group::tiled_partition( - group::this_thread_block()); - auto lane_prefix_mask = - (config::lane_mask_type(1) << warp.thread_rank()) - 1; - auto base_out_idx = row_ptrs[row]; - for (size_type i = 0; i < num_cols; i += config::warp_size) { - const auto col = i + warp.thread_rank(); - const auto pred = - col < num_cols ? is_nonzero(source[stride * row + col]) : false; - const auto mask = group::ballot(warp, pred); - const auto out_idx = base_out_idx + popcnt(mask & lane_prefix_mask); - if (pred) { - col_idxs[out_idx] = col; - } - base_out_idx += popcnt(mask); - } - } -} - - -template -__global__ __launch_bounds__(default_block_size) void fill_in_ell( - size_type num_rows, size_type num_cols, size_type source_stride, - const ValueType* __restrict__ source, size_type max_nnz_per_row, - size_type result_stride, IndexType* __restrict__ col_idxs, - ValueType* __restrict__ values) -{ - const auto row = thread::get_subwarp_id_flat(); - - if (row < num_rows) { - auto warp = group::tiled_partition( - group::this_thread_block()); - auto lane_prefix_mask = - (config::lane_mask_type(1) << warp.thread_rank()) - 1; - size_type base_out_idx{}; - for (size_type i = 0; i < num_cols; i += config::warp_size) { - const auto col = i + warp.thread_rank(); - const auto pred = - col < num_cols ? is_nonzero(source[source_stride * row + col]) - : false; - const auto mask = group::ballot(warp, pred); - const auto out_idx = - row + (base_out_idx + popcnt(mask & lane_prefix_mask)) * - result_stride; - if (pred) { - values[out_idx] = source[source_stride * row + col]; - col_idxs[out_idx] = col; - } - base_out_idx += popcnt(mask); - } - for (size_type i = base_out_idx + warp.thread_rank(); - i < max_nnz_per_row; i += config::warp_size) { - const auto out_idx = row + i * result_stride; - values[out_idx] = zero(); - col_idxs[out_idx] = invalid_index(); - } - } -} - - -template -__global__ __launch_bounds__(default_block_size) void fill_in_hybrid( - size_type num_rows, size_type num_cols, size_type source_stride, - const ValueType* __restrict__ source, size_type ell_max_nnz_per_row, - size_type ell_stride, IndexType* __restrict__ ell_col_idxs, - ValueType* __restrict__ ell_values, const int64* __restrict__ coo_row_ptrs, - IndexType* __restrict__ coo_row_idxs, IndexType* __restrict__ coo_col_idxs, - ValueType* __restrict__ coo_values) -{ - const auto row = thread::get_subwarp_id_flat(); - - if (row < num_rows) { - auto warp = group::tiled_partition( - group::this_thread_block()); - auto lane_prefix_mask = - (config::lane_mask_type(1) << warp.thread_rank()) - 1; - size_type base_out_idx{}; - const auto coo_out_begin = coo_row_ptrs[row]; - for (size_type i = 0; i < num_cols; i += config::warp_size) { - const auto col = i + warp.thread_rank(); - const auto pred = - col < num_cols ? is_nonzero(source[source_stride * row + col]) - : false; - const auto mask = group::ballot(warp, pred); - const auto cur_out_idx = - base_out_idx + popcnt(mask & lane_prefix_mask); - if (pred) { - if (cur_out_idx < ell_max_nnz_per_row) { - const auto out_idx = row + cur_out_idx * ell_stride; - ell_values[out_idx] = source[source_stride * row + col]; - ell_col_idxs[out_idx] = col; - } else { - const auto out_idx = - cur_out_idx - ell_max_nnz_per_row + coo_out_begin; - coo_values[out_idx] = source[source_stride * row + col]; - coo_col_idxs[out_idx] = col; - coo_row_idxs[out_idx] = row; - } - } - base_out_idx += popcnt(mask); - } - for (size_type i = base_out_idx + warp.thread_rank(); - i < ell_max_nnz_per_row; i += config::warp_size) { - const auto out_idx = row + i * ell_stride; - ell_values[out_idx] = zero(); - ell_col_idxs[out_idx] = invalid_index(); - } - } -} - - -template -__global__ __launch_bounds__(default_block_size) void fill_in_sellp( - size_type num_rows, size_type num_cols, size_type slice_size, - size_type stride, const ValueType* __restrict__ source, - size_type* __restrict__ slice_sets, IndexType* __restrict__ col_idxs, - ValueType* __restrict__ values) -{ - const auto row = thread::get_subwarp_id_flat(); - const auto local_row = row % slice_size; - const auto slice = row / slice_size; - - if (row < num_rows) { - auto warp = group::tiled_partition( - group::this_thread_block()); - const auto lane = warp.thread_rank(); - const auto prefix_mask = (config::lane_mask_type{1} << lane) - 1; - const auto slice_end = slice_sets[slice + 1] * slice_size; - auto base_idx = slice_sets[slice] * slice_size + local_row; - for (size_type i = 0; i < num_cols; i += config::warp_size) { - const auto col = i + lane; - const auto val = checked_load(source + stride * row, col, num_cols, - zero()); - const auto pred = is_nonzero(val); - const auto mask = group::ballot(warp, pred); - const auto idx = base_idx + popcnt(mask & prefix_mask) * slice_size; - if (pred) { - values[idx] = val; - col_idxs[idx] = col; - } - base_idx += popcnt(mask) * slice_size; - } - for (auto i = base_idx + lane * slice_size; i < slice_end; - i += config::warp_size * slice_size) { - values[i] = zero(); - col_idxs[i] = invalid_index(); - } - } -} - - -} // namespace kernel - - -template -void convert_to_coo(std::shared_ptr exec, - matrix::view::dense source, - const int64* row_ptrs, - matrix::view::coo result) -{ - auto num_rows = result.size[0]; - auto num_cols = result.size[1]; - - auto row_idxs = result.row_idxs; - auto col_idxs = result.col_idxs; - auto values = result.values; - - auto stride = source.stride; - - const auto grid_dim = - ceildiv(num_rows, default_block_size / config::warp_size); - if (grid_dim > 0) { - kernel::fill_in_coo<<get_stream()>>>( - num_rows, num_cols, stride, as_device_type(source.values), row_ptrs, - row_idxs, col_idxs, as_device_type(values)); - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_COO_KERNEL); - - -template -void convert_to_csr(std::shared_ptr exec, - matrix::view::dense source, - matrix::Csr* result) -{ - auto num_rows = result->get_size()[0]; - auto num_cols = result->get_size()[1]; - - auto row_ptrs = result->get_row_ptrs(); - auto col_idxs = result->get_col_idxs(); - auto values = result->get_values(); - - auto stride = source.stride; - - const auto grid_dim = - ceildiv(num_rows, default_block_size / config::warp_size); - if (grid_dim > 0) { - kernel::fill_in_csr<<get_stream()>>>( - num_rows, num_cols, stride, as_device_type(source.values), - as_device_type(row_ptrs), as_device_type(col_idxs), - as_device_type(values)); - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_CSR_KERNEL); - - -template -void convert_to_ell(std::shared_ptr exec, - matrix::view::dense source, - matrix::view::ell result) -{ - auto num_rows = result.size[0]; - auto num_cols = result.size[1]; - auto max_nnz_per_row = result.num_stored_elements_per_row; - - auto col_idxs = result.col_idxs; - auto values = result.values; - - auto source_stride = source.stride; - auto result_stride = result.stride; - - const auto grid_dim = - ceildiv(num_rows, default_block_size / config::warp_size); - if (grid_dim > 0) { - kernel::fill_in_ell<<get_stream()>>>( - num_rows, num_cols, source_stride, as_device_type(source.values), - max_nnz_per_row, result_stride, col_idxs, as_device_type(values)); - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_ELL_KERNEL); - - -template -void convert_to_fbcsr(std::shared_ptr exec, - matrix::view::dense source, - matrix::Fbcsr* result) -{ - const auto num_block_rows = result->get_num_block_rows(); - if (num_block_rows > 0) { - const auto num_blocks = - ceildiv(num_block_rows, default_block_size / config::warp_size); - kernel::convert_to_fbcsr<<get_stream()>>>( - num_block_rows, result->get_num_block_cols(), source.stride, - result->get_block_size(), as_device_type(source.values), - result->get_const_row_ptrs(), result->get_col_idxs(), - as_device_type(result->get_values())); - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_FBCSR_KERNEL); - - -template -void count_nonzero_blocks_per_row(std::shared_ptr exec, - matrix::view::dense source, - int bs, IndexType* result) -{ - const auto num_block_rows = source.size[0] / bs; - const auto num_block_cols = source.size[1] / bs; - if (num_block_rows > 0) { - const auto num_blocks = - ceildiv(num_block_rows, default_block_size / config::warp_size); - kernel::count_nonzero_blocks_per_row<<get_stream()>>>( - num_block_rows, num_block_cols, source.stride, bs, - as_device_type(source.values), result); - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL); - - -template -void convert_to_hybrid(std::shared_ptr exec, - matrix::view::dense source, - const int64* coo_row_ptrs, - matrix::view::hybrid result) -{ - const auto num_rows = result.size[0]; - const auto num_cols = result.size[1]; - const auto ell_max_nnz_per_row = - result.ell_part.num_stored_elements_per_row; - const auto source_stride = source.stride; - const auto ell_stride = result.ell_part.stride; - auto ell_col_idxs = result.ell_part.col_idxs; - auto ell_values = result.ell_part.values; - auto coo_row_idxs = result.coo_part.row_idxs; - auto coo_col_idxs = result.coo_part.col_idxs; - auto coo_values = result.coo_part.values; - - auto grid_dim = ceildiv(num_rows, default_block_size / config::warp_size); - if (grid_dim > 0) { - kernel::fill_in_hybrid<<get_stream()>>>( - num_rows, num_cols, source_stride, as_device_type(source.values), - ell_max_nnz_per_row, ell_stride, ell_col_idxs, - as_device_type(ell_values), coo_row_ptrs, coo_row_idxs, - coo_col_idxs, as_device_type(coo_values)); - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_HYBRID_KERNEL); - - -template -void convert_to_sellp(std::shared_ptr exec, - matrix::view::dense source, - matrix::view::sellp result) -{ - const auto stride = source.stride; - const auto num_rows = result.size[0]; - const auto num_cols = result.size[1]; - - auto vals = result.values; - auto col_idxs = result.col_idxs; - auto slice_sets = result.slice_sets; - - const auto slice_size = result.slice_size; - const auto stride_factor = result.stride_factor; - - auto grid_dim = ceildiv(num_rows, default_block_size / config::warp_size); - if (grid_dim > 0) { - kernel::fill_in_sellp<<get_stream()>>>( - num_rows, num_cols, slice_size, stride, - as_device_type(source.values), as_device_type(slice_sets), - as_device_type(col_idxs), as_device_type(vals)); - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SELLP_KERNEL); - - -template -void convert_to_sparsity_csr(std::shared_ptr exec, - matrix::view::dense source, - matrix::SparsityCsr* result) -{ - auto num_rows = result->get_size()[0]; - auto num_cols = result->get_size()[1]; - - auto row_ptrs = result->get_row_ptrs(); - auto col_idxs = result->get_col_idxs(); - - auto stride = source.stride; - - const auto grid_dim = - ceildiv(num_rows, default_block_size / config::warp_size); - if (grid_dim > 0) { - kernel::fill_in_sparsity_csr<<get_stream()>>>( - num_rows, num_cols, stride, as_device_type(source.values), - as_device_type(row_ptrs), as_device_type(col_idxs)); - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SPARSITY_CSR_KERNEL); - - template void compute_dot_dispatch(std::shared_ptr exec, matrix::view::dense x, @@ -725,62 +99,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( GKO_DECLARE_MULTIVECTOR_COMPUTE_NORM2_DISPATCH_KERNEL); -template -void simple_apply(std::shared_ptr exec, - matrix::view::dense a, - matrix::view::dense b, - matrix::view::dense c) -{ - if (blas::is_supported::value) { - auto handle = exec->get_blas_handle(); - if (c.size[0] > 0 && c.size[1] > 0) { - if (a.size[1] > 0) { - blas::pointer_mode_guard pm_guard(handle); - auto alpha = one(); - auto beta = zero(); - blas::gemm(handle, BLAS_OP_N, BLAS_OP_N, c.size[1], c.size[0], - a.size[1], &alpha, b.values, b.stride, a.values, - a.stride, &beta, c.values, c.stride); - } else { - multivector::fill(exec, c, zero()); - } - } - } else { - GKO_NOT_IMPLEMENTED; - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_SIMPLE_APPLY_KERNEL); - - -template -void apply(std::shared_ptr exec, - matrix::view::dense alpha, - matrix::view::dense a, - matrix::view::dense b, - matrix::view::dense beta, - matrix::view::dense c) -{ - if (blas::is_supported::value) { - if (c.size[0] > 0 && c.size[1] > 0) { - if (a.size[1] > 0) { - blas::gemm(exec->get_blas_handle(), BLAS_OP_N, BLAS_OP_N, - c.size[1], c.size[0], a.size[1], alpha.values, - b.values, b.stride, a.values, a.stride, beta.values, - c.values, c.stride); - } else { - multivector::scale(exec, beta, c); - } - } - } else { - GKO_NOT_IMPLEMENTED; - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_APPLY_KERNEL); - - template void transpose(std::shared_ptr exec, matrix::view::dense orig, diff --git a/common/unified/CMakeLists.txt b/common/unified/CMakeLists.txt index 617b275db5a..f04677ce02a 100644 --- a/common/unified/CMakeLists.txt +++ b/common/unified/CMakeLists.txt @@ -12,6 +12,7 @@ set(UNIFIED_SOURCES distributed/partition_kernels.cpp matrix/coo_kernels.cpp matrix/csr_kernels.cpp + matrix/dense_kernels.cpp matrix/ell_kernels.cpp matrix/hybrid_kernels.cpp matrix/permutation_kernels.cpp diff --git a/common/unified/matrix/dense_kernels.cpp b/common/unified/matrix/dense_kernels.cpp new file mode 100644 index 00000000000..a7a9ddfd79a --- /dev/null +++ b/common/unified/matrix/dense_kernels.cpp @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include "core/matrix/dense_kernels.hpp" + +#include "common/unified/base/kernel_launch.hpp" +#include "common/unified/base/kernel_launch_reduction.hpp" +#include "core/base/array_access.hpp" +#include "core/components/prefix_sum_kernels.hpp" + + +namespace gko { +namespace kernels { +namespace GKO_DEVICE_NAMESPACE { +namespace dense { + + +template +void compute_max_nnz_per_row(std::shared_ptr exec, + matrix::view::dense source, + size_type& result) +{ + array partial{exec, source.size[0] + 1}; + count_nonzeros_per_row(exec, source, partial.get_data()); + run_kernel_reduction( + exec, [] GKO_KERNEL(auto i, auto partial) { return partial[i]; }, + GKO_KERNEL_REDUCE_MAX(size_type), partial.get_data() + source.size[0], + source.size[0], partial); + result = get_element(partial, source.size[0]); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( + GKO_DECLARE_DENSE_COMPUTE_MAX_NNZ_PER_ROW_KERNEL); + + +template +void compute_slice_sets(std::shared_ptr exec, + matrix::view::dense source, + size_type slice_size, size_type stride_factor, + size_type* slice_sets, size_type* slice_lengths) +{ + const auto num_rows = source.size[0]; + array row_nnz{exec, num_rows}; + count_nonzeros_per_row(exec, source, row_nnz.get_data()); + const auto num_slices = + static_cast(ceildiv(num_rows, slice_size)); + run_kernel_row_reduction( + exec, + [] GKO_KERNEL(auto slice, auto local_row, auto row_nnz, auto slice_size, + auto stride_factor, auto num_rows) { + const auto row = slice * slice_size + local_row; + return row < num_rows ? static_cast( + ceildiv(row_nnz[row], stride_factor) * + stride_factor) + : size_type{}; + }, + GKO_KERNEL_REDUCE_MAX(size_type), slice_lengths, 1, + gko::dim<2>{num_slices, slice_size}, row_nnz, slice_size, stride_factor, + num_rows); + exec->copy(num_slices, slice_lengths, slice_sets); + components::prefix_sum_nonnegative(exec, slice_sets, num_slices + 1); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( + GKO_DECLARE_DENSE_COMPUTE_SLICE_SETS_KERNEL); + + +template +void count_nonzeros_per_row(std::shared_ptr exec, + matrix::view::dense mtx, + IndexType* result) +{ + run_kernel_row_reduction( + exec, + [] GKO_KERNEL(auto i, auto j, auto mtx) { + return is_nonzero(mtx(i, j)) ? 1 : 0; + }, + GKO_KERNEL_REDUCE_SUM(IndexType), result, 1, mtx.size, mtx); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_COUNT_NONZEROS_PER_ROW_KERNEL); +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( + GKO_DECLARE_DENSE_COUNT_NONZEROS_PER_ROW_KERNEL_SIZE_T); + + +template +void extract_diagonal(std::shared_ptr exec, + matrix::view::dense orig, + matrix::Diagonal* diag) +{ + run_kernel( + exec, + [] GKO_KERNEL(auto i, auto orig, auto diag) { diag[i] = orig(i, i); }, + diag->get_size()[0], orig, diag->get_values()); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL); + + +template +void add_scaled_diag(std::shared_ptr exec, + matrix::view::dense alpha, + const matrix::Diagonal* x, + matrix::view::dense y) +{ + const auto diag_values = x->get_const_values(); + run_kernel( + exec, + [] GKO_KERNEL(auto i, auto alpha, auto diag, auto y) { + if (is_nonzero(alpha[0])) { + y(i, i) += alpha[0] * diag[i]; + } + }, + x->get_size()[0], alpha.values, x->get_const_values(), y); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL); + + +template +void sub_scaled_diag(std::shared_ptr exec, + matrix::view::dense alpha, + const matrix::Diagonal* x, + matrix::view::dense y) +{ + const auto diag_values = x->get_const_values(); + run_kernel( + exec, + [] GKO_KERNEL(auto i, auto alpha, auto diag, auto y) { + if (is_nonzero(alpha[0])) { + y(i, i) -= alpha[0] * diag[i]; + } + }, + x->get_size()[0], alpha.values, x->get_const_values(), y); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL); + + +template +void add_scaled_identity(std::shared_ptr exec, + matrix::view::dense alpha, + matrix::view::dense beta, + matrix::view::dense mtx) +{ + run_kernel( + exec, + [] GKO_KERNEL(auto row, auto col, auto alpha, auto beta, auto mtx) { + mtx(row, col) = beta[0] * mtx(row, col); + if (row == col) { + mtx(row, row) += alpha[0]; + } + }, + mtx.size, alpha.values, beta.values, mtx); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_SCALAR_TYPE( + GKO_DECLARE_DENSE_ADD_SCALED_IDENTITY_KERNEL); + + +} // namespace dense +} // namespace GKO_DEVICE_NAMESPACE +} // namespace kernels +} // namespace gko diff --git a/common/unified/matrix/multivector_kernels.instantiate.cpp b/common/unified/matrix/multivector_kernels.instantiate.cpp index 80c1f1e6915..fa3f65c487a 100644 --- a/common/unified/matrix/multivector_kernels.instantiate.cpp +++ b/common/unified/matrix/multivector_kernels.instantiate.cpp @@ -25,10 +25,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_SCALAR_TYPE( GKO_DECLARE_MULTIVECTOR_ADD_SCALED_KERNEL); GKO_INSTANTIATE_FOR_EACH_VALUE_AND_SCALAR_TYPE( GKO_DECLARE_MULTIVECTOR_SUB_SCALED_KERNEL); -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_ADD_SCALED_DIAG_KERNEL); -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_SUB_SCALED_DIAG_KERNEL); GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( GKO_DECLARE_MULTIVECTOR_COMPUTE_SQRT_KERNEL); GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( @@ -65,15 +61,11 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( GKO_DECLARE_MULTIVECTOR_COL_SCALE_PERMUTE_KERNEL); GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( GKO_DECLARE_MULTIVECTOR_INV_COL_SCALE_PERMUTE_KERNEL); -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_EXTRACT_DIAGONAL_KERNEL); GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL); GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL); GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL); GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_REAL_KERNEL); GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL); -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_SCALAR_TYPE( - GKO_DECLARE_MULTIVECTOR_ADD_SCALED_IDENTITY_KERNEL); // split GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_COMPUTE_DOT_KERNEL); // split @@ -86,17 +78,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( GKO_DECLARE_MULTIVECTOR_COMPUTE_NORM1_KERNEL); // split -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_COMPUTE_MAX_NNZ_PER_ROW_KERNEL); -// split -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_COMPUTE_SLICE_SETS_KERNEL); -// split -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZEROS_PER_ROW_KERNEL); -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZEROS_PER_ROW_KERNEL_SIZE_T); -// split GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( GKO_DECLARE_MULTIVECTOR_COMPUTE_SQUARED_NORM2_KERNEL); // split diff --git a/common/unified/matrix/multivector_kernels.template.cpp b/common/unified/matrix/multivector_kernels.template.cpp index 7c2f7e136b3..41c2c492256 100644 --- a/common/unified/matrix/multivector_kernels.template.cpp +++ b/common/unified/matrix/multivector_kernels.template.cpp @@ -9,7 +9,6 @@ #include "common/unified/base/kernel_launch.hpp" #include "common/unified/base/kernel_launch_reduction.hpp" -#include "core/base/array_access.hpp" #include "core/base/mixed_precision_types.hpp" #include "core/components/prefix_sum_kernels.hpp" @@ -184,42 +183,6 @@ void sub_scaled(std::shared_ptr exec, } -template -void add_scaled_diag(std::shared_ptr exec, - matrix::view::dense alpha, - const matrix::Diagonal* x, - matrix::view::dense y) -{ - const auto diag_values = x->get_const_values(); - run_kernel( - exec, - [] GKO_KERNEL(auto i, auto alpha, auto diag, auto y) { - if (is_nonzero(alpha[0])) { - y(i, i) += alpha[0] * diag[i]; - } - }, - x->get_size()[0], alpha.values, x->get_const_values(), y); -} - - -template -void sub_scaled_diag(std::shared_ptr exec, - matrix::view::dense alpha, - const matrix::Diagonal* x, - matrix::view::dense y) -{ - const auto diag_values = x->get_const_values(); - run_kernel( - exec, - [] GKO_KERNEL(auto i, auto alpha, auto diag, auto y) { - if (is_nonzero(alpha[0])) { - y(i, i) -= alpha[0] * diag[i]; - } - }, - x->get_size()[0], alpha.values, x->get_const_values(), y); -} - - template void compute_dot(std::shared_ptr exec, matrix::view::dense x, @@ -293,64 +256,6 @@ void compute_mean(std::shared_ptr exec, } -template -void compute_max_nnz_per_row(std::shared_ptr exec, - matrix::view::dense source, - size_type& result) -{ - array partial{exec, source.size[0] + 1}; - count_nonzeros_per_row(exec, source, partial.get_data()); - run_kernel_reduction( - exec, [] GKO_KERNEL(auto i, auto partial) { return partial[i]; }, - GKO_KERNEL_REDUCE_MAX(size_type), partial.get_data() + source.size[0], - source.size[0], partial); - result = get_element(partial, source.size[0]); -} - - -template -void compute_slice_sets(std::shared_ptr exec, - matrix::view::dense source, - size_type slice_size, size_type stride_factor, - size_type* slice_sets, size_type* slice_lengths) -{ - const auto num_rows = source.size[0]; - array row_nnz{exec, num_rows}; - count_nonzeros_per_row(exec, source, row_nnz.get_data()); - const auto num_slices = - static_cast(ceildiv(num_rows, slice_size)); - run_kernel_row_reduction( - exec, - [] GKO_KERNEL(auto slice, auto local_row, auto row_nnz, auto slice_size, - auto stride_factor, auto num_rows) { - const auto row = slice * slice_size + local_row; - return row < num_rows ? static_cast( - ceildiv(row_nnz[row], stride_factor) * - stride_factor) - : size_type{}; - }, - GKO_KERNEL_REDUCE_MAX(size_type), slice_lengths, 1, - gko::dim<2>{num_slices, slice_size}, row_nnz, slice_size, stride_factor, - num_rows); - exec->copy(num_slices, slice_lengths, slice_sets); - components::prefix_sum_nonnegative(exec, slice_sets, num_slices + 1); -} - - -template -void count_nonzeros_per_row(std::shared_ptr exec, - matrix::view::dense mtx, - IndexType* result) -{ - run_kernel_row_reduction( - exec, - [] GKO_KERNEL(auto i, auto j, auto mtx) { - return is_nonzero(mtx(i, j)) ? 1 : 0; - }, - GKO_KERNEL_REDUCE_SUM(IndexType), result, 1, mtx.size, mtx); -} - - template void compute_squared_norm2( std::shared_ptr exec, @@ -686,18 +591,6 @@ void inv_col_scale_permute(std::shared_ptr exec, } -template -void extract_diagonal(std::shared_ptr exec, - matrix::view::dense orig, - matrix::Diagonal* diag) -{ - run_kernel( - exec, - [] GKO_KERNEL(auto i, auto orig, auto diag) { diag[i] = orig(i, i); }, - diag->get_size()[0], orig, diag->get_values()); -} - - template void inplace_absolute_dense(std::shared_ptr exec, matrix::view::dense source) @@ -768,24 +661,6 @@ void get_imag(std::shared_ptr exec, } -template -void add_scaled_identity(std::shared_ptr exec, - matrix::view::dense alpha, - matrix::view::dense beta, - matrix::view::dense mtx) -{ - run_kernel( - exec, - [] GKO_KERNEL(auto row, auto col, auto alpha, auto beta, auto mtx) { - mtx(row, col) = beta[0] * mtx(row, col); - if (row == col) { - mtx(row, row) += alpha[0]; - } - }, - mtx.size, alpha.values, beta.values, mtx); -} - - } // namespace multivector } // namespace GKO_DEVICE_NAMESPACE } // namespace kernels diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index b198e6e5031..a2eed9a48d6 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -82,6 +82,7 @@ target_sources( matrix/coo.cpp matrix/csr.cpp matrix/csr_lookup.cpp + matrix/dense.cpp matrix/diagonal.cpp matrix/ell.cpp matrix/fbcsr.cpp diff --git a/core/base/combination.cpp b/core/base/combination.cpp index 2e95729d5a3..d9ea8011252 100644 --- a/core/base/combination.cpp +++ b/core/base/combination.cpp @@ -5,6 +5,7 @@ #include "ginkgo/core/base/combination.hpp" #include +#include #include diff --git a/core/base/perturbation.cpp b/core/base/perturbation.cpp index b92077d9076..5b48ba0ddb1 100644 --- a/core/base/perturbation.cpp +++ b/core/base/perturbation.cpp @@ -5,6 +5,7 @@ #include "ginkgo/core/base/perturbation.hpp" #include +#include namespace gko { diff --git a/core/device_hooks/common_kernels.inc.cpp b/core/device_hooks/common_kernels.inc.cpp index 15498903110..3bcb7dbd903 100644 --- a/core/device_hooks/common_kernels.inc.cpp +++ b/core/device_hooks/common_kernels.inc.cpp @@ -41,6 +41,7 @@ #include "core/matrix/batch_ell_kernels.hpp" #include "core/matrix/coo_kernels.hpp" #include "core/matrix/csr_kernels.hpp" +#include "core/matrix/dense_kernels.hpp" #include "core/matrix/diagonal_kernels.hpp" #include "core/matrix/ell_kernels.hpp" #include "core/matrix/fbcsr_kernels.hpp" @@ -447,21 +448,43 @@ GKO_STUB_VALUE_AND_INT32_TYPE(GKO_DECLARE_BATCH_ELL_ADD_SCALED_IDENTITY_KERNEL); } // namespace batch_ell +namespace dense { + + +GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL); +GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL); +GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL); +GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL); +GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL); +GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_CONVERT_TO_FBCSR_KERNEL); +GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL); +GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL); +GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL); +GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_MAX_NNZ_PER_ROW_KERNEL); +GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_COMPUTE_SLICE_SETS_KERNEL); +GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_DENSE_COUNT_NONZEROS_PER_ROW_KERNEL); +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( + GKO_DECLARE_DENSE_COUNT_NONZEROS_PER_ROW_KERNEL_SIZE_T); +GKO_STUB_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL); +GKO_STUB_VALUE_AND_SCALAR_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_IDENTITY_KERNEL); +GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL); +GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL); +GKO_STUB_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL); + + +} // namespace dense + + namespace multivector { -GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_SIMPLE_APPLY_KERNEL); -GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_APPLY_KERNEL); GKO_STUB_VALUE_CONVERSION_OR_COPY(GKO_DECLARE_MULTIVECTOR_COPY_KERNEL); GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_FILL_KERNEL); GKO_STUB_VALUE_AND_SCALAR_TYPE(GKO_DECLARE_MULTIVECTOR_SCALE_KERNEL); GKO_STUB_VALUE_AND_SCALAR_TYPE(GKO_DECLARE_MULTIVECTOR_INV_SCALE_KERNEL); GKO_STUB_VALUE_AND_SCALAR_TYPE(GKO_DECLARE_MULTIVECTOR_ADD_SCALED_KERNEL); GKO_STUB_VALUE_AND_SCALAR_TYPE(GKO_DECLARE_MULTIVECTOR_SUB_SCALED_KERNEL); -GKO_STUB_VALUE_AND_SCALAR_TYPE( - GKO_DECLARE_MULTIVECTOR_ADD_SCALED_IDENTITY_KERNEL); -GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_ADD_SCALED_DIAG_KERNEL); -GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_SUB_SCALED_DIAG_KERNEL); GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_COMPUTE_DOT_KERNEL); GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_COMPUTE_DOT_DISPATCH_KERNEL); GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_COMPUTE_CONJ_DOT_KERNEL); @@ -474,22 +497,6 @@ GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_COMPUTE_SQUARED_NORM2_KERNEL); GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_COMPUTE_SQRT_KERNEL); GKO_STUB_VALUE_AND_INDEX_TYPE( GKO_DECLARE_MULTIVECTOR_FILL_IN_MATRIX_DATA_KERNEL); -GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_MULTIVECTOR_CONVERT_TO_COO_KERNEL); -GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_MULTIVECTOR_CONVERT_TO_CSR_KERNEL); -GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_MULTIVECTOR_CONVERT_TO_ELL_KERNEL); -GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_MULTIVECTOR_CONVERT_TO_FBCSR_KERNEL); -GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_MULTIVECTOR_CONVERT_TO_HYBRID_KERNEL); -GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SELLP_KERNEL); -GKO_STUB_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SPARSITY_CSR_KERNEL); -GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_COMPUTE_MAX_NNZ_PER_ROW_KERNEL); -GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_COMPUTE_SLICE_SETS_KERNEL); -GKO_STUB_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZEROS_PER_ROW_KERNEL); -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZEROS_PER_ROW_KERNEL_SIZE_T); -GKO_STUB_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL); GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_TRANSPOSE_KERNEL); GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_CONJ_TRANSPOSE_KERNEL); GKO_STUB_VALUE_AND_INDEX_TYPE(GKO_DECLARE_MULTIVECTOR_SYMM_PERMUTE_KERNEL); @@ -518,7 +525,6 @@ GKO_STUB_VALUE_AND_INDEX_TYPE( GKO_DECLARE_MULTIVECTOR_NONSYMM_SCALE_PERMUTE_KERNEL); GKO_STUB_VALUE_AND_INDEX_TYPE( GKO_DECLARE_MULTIVECTOR_INV_NONSYMM_SCALE_PERMUTE_KERNEL); -GKO_STUB_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_EXTRACT_DIAGONAL_KERNEL); GKO_STUB_VALUE_TYPE(GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL); GKO_STUB_VALUE_TYPE(GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL); GKO_STUB_VALUE_TYPE(GKO_DECLARE_MAKE_COMPLEX_KERNEL); diff --git a/core/distributed/preconditioner/schwarz.cpp b/core/distributed/preconditioner/schwarz.cpp index 4d32fba7b14..a9b7e702f44 100644 --- a/core/distributed/preconditioner/schwarz.cpp +++ b/core/distributed/preconditioner/schwarz.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/core/distributed/vector.cpp b/core/distributed/vector.cpp index e5b527554a4..dac9e80d073 100644 --- a/core/distributed/vector.cpp +++ b/core/distributed/vector.cpp @@ -5,6 +5,7 @@ #include "ginkgo/core/distributed/vector.hpp" #include +#include #include "core/distributed/vector_kernels.hpp" #include "core/matrix/multivector_kernels.hpp" diff --git a/core/factorization/symbolic.cpp b/core/factorization/symbolic.cpp index 6ccb046cf59..b8c0256b53c 100644 --- a/core/factorization/symbolic.cpp +++ b/core/factorization/symbolic.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include diff --git a/core/log/convergence.cpp b/core/log/convergence.cpp index 732cc4a7b01..9931e4a1236 100644 --- a/core/log/convergence.cpp +++ b/core/log/convergence.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include diff --git a/core/log/stream.cpp b/core/log/stream.cpp index 6947fbd86b0..228c5ea720c 100644 --- a/core/log/stream.cpp +++ b/core/log/stream.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,13 @@ std::ostream& operator<<(std::ostream& os, } +template +std::ostream& operator<<(std::ostream& os, const matrix::Dense* mtx) +{ + return os << mtx->as_const_multivector_view().get(); +} + + std::ostream& operator<<(std::ostream& os, const stopping_status* status) { os << "[" << std::endl; @@ -252,7 +260,7 @@ void Stream::on_linop_apply_started(const LinOp* A, const LinOp* b, *os_ << prefix_ << "apply started on A " << demangle_name(A) << " with b " << demangle_name(b) << " and x " << demangle_name(x) << std::endl; if (verbose_) { - *os_ << demangle_name(A) << as>(A) + *os_ << demangle_name(A) << as>(A) << std::endl; *os_ << demangle_name(b) << as>(b) << std::endl; @@ -269,7 +277,7 @@ void Stream::on_linop_apply_completed(const LinOp* A, const LinOp* b, *os_ << prefix_ << "apply completed on A " << demangle_name(A) << " with b " << demangle_name(b) << " and x " << demangle_name(x) << std::endl; if (verbose_) { - *os_ << demangle_name(A) << as>(A) + *os_ << demangle_name(A) << as>(A) << std::endl; *os_ << demangle_name(b) << as>(b) << std::endl; @@ -291,7 +299,7 @@ void Stream::on_linop_advanced_apply_started(const LinOp* A, << " beta " << demangle_name(beta) << " and x " << demangle_name(x) << std::endl; if (verbose_) { - *os_ << demangle_name(A) << as>(A) + *os_ << demangle_name(A) << as>(A) << std::endl; *os_ << demangle_name(alpha) << as>(alpha) << std::endl; @@ -317,7 +325,7 @@ void Stream::on_linop_advanced_apply_completed(const LinOp* A, << " beta " << demangle_name(beta) << " and x " << demangle_name(x) << std::endl; if (verbose_) { - *os_ << demangle_name(A) << as>(A) + *os_ << demangle_name(A) << as>(A) << std::endl; *os_ << demangle_name(alpha) << as>(alpha) << std::endl; diff --git a/core/matrix/coo.cpp b/core/matrix/coo.cpp index fc583940fa0..4eb0067d740 100644 --- a/core/matrix/coo.cpp +++ b/core/matrix/coo.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "core/base/device_matrix_data_kernels.hpp" @@ -314,7 +315,7 @@ void Coo::move_to(Csr* result) template -void Coo::convert_to(MultiVector* result) const +void Coo::convert_to(Dense* result) const { auto exec = this->get_executor(); auto tmp_result = make_temporary_output_clone(exec, result); @@ -326,7 +327,7 @@ void Coo::convert_to(MultiVector* result) const template -void Coo::move_to(MultiVector* result) +void Coo::move_to(Dense* result) { this->convert_to(result); } diff --git a/core/matrix/csr.cpp b/core/matrix/csr.cpp index b8281f0fbc7..e70a3739692 100644 --- a/core/matrix/csr.cpp +++ b/core/matrix/csr.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -618,7 +619,7 @@ void Csr::move_to(Coo* result) template -void Csr::convert_to(MultiVector* result) const +void Csr::convert_to(Dense* result) const { auto exec = this->get_executor(); auto tmp_result = make_temporary_output_clone(exec, result); @@ -629,7 +630,7 @@ void Csr::convert_to(MultiVector* result) const template -void Csr::move_to(MultiVector* result) +void Csr::move_to(Dense* result) { this->convert_to(result); } diff --git a/core/matrix/dense.cpp b/core/matrix/dense.cpp new file mode 100644 index 00000000000..9b3a08a0b25 --- /dev/null +++ b/core/matrix/dense.cpp @@ -0,0 +1,1013 @@ +// SPDX-FileCopyrightText: 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/base/array_access.hpp" +#include "core/base/dispatch_helper.hpp" +#include "core/components/prefix_sum_kernels.hpp" +#include "core/matrix/dense_kernels.hpp" +#include "core/matrix/hybrid_kernels.hpp" +#include "core/matrix/multivector_kernels.hpp" +#include "ginkgo/core/matrix/fbcsr.hpp" + + +namespace gko { +namespace matrix { +namespace dense { + + +GKO_REGISTER_OPERATION(simple_apply, dense::simple_apply); +GKO_REGISTER_OPERATION(advanced_apply, dense::apply); +GKO_REGISTER_OPERATION(convert_to_coo, dense::convert_to_coo); +GKO_REGISTER_OPERATION(convert_to_csr, dense::convert_to_csr); +GKO_REGISTER_OPERATION(convert_to_ell, dense::convert_to_ell); +GKO_REGISTER_OPERATION(convert_to_fbcsr, dense::convert_to_fbcsr); +GKO_REGISTER_OPERATION(convert_to_hybrid, dense::convert_to_hybrid); +GKO_REGISTER_OPERATION(convert_to_sellp, dense::convert_to_sellp); +GKO_REGISTER_OPERATION(convert_to_sparsity_csr, dense::convert_to_sparsity_csr); +GKO_REGISTER_OPERATION(compute_max_nnz_per_row, dense::compute_max_nnz_per_row); +GKO_REGISTER_OPERATION(compute_hybrid_coo_row_ptrs, + hybrid::compute_coo_row_ptrs); +GKO_REGISTER_OPERATION(count_nonzeros_per_row, dense::count_nonzeros_per_row); +GKO_REGISTER_OPERATION(count_nonzero_blocks_per_row, + dense::count_nonzero_blocks_per_row); +GKO_REGISTER_OPERATION(prefix_sum_nonnegative, + components::prefix_sum_nonnegative); +GKO_REGISTER_OPERATION(compute_slice_sets, dense::compute_slice_sets); +GKO_REGISTER_OPERATION(extract_diagonal, dense::extract_diagonal); +GKO_REGISTER_OPERATION(add_scaled_diag, dense::add_scaled_diag); +GKO_REGISTER_OPERATION(sub_scaled_diag, dense::sub_scaled_diag); +GKO_REGISTER_OPERATION(add_scaled_identity, dense::add_scaled_identity); + + +} // namespace dense +namespace multivector { + + +GKO_REGISTER_OPERATION(copy, multivector::copy); +GKO_REGISTER_OPERATION(fill, multivector::fill); +GKO_REGISTER_OPERATION(fill_in_matrix_data, multivector::fill_in_matrix_data); + + +} // namespace multivector + + +template +void Dense::convert_to(MultiVector* result) const +{ + if (result->get_size() != this->get_size()) { + result->set_size(this->get_size()); + result->stride_ = stride_; + result->values_.resize_and_reset(result->get_size()[0] * + result->stride_); + } + auto exec = this->get_executor(); + exec->run(multivector::make_copy( + this->get_const_device_view(), + make_temporary_output_clone(exec, result)->get_device_view())); +} + + +template +void Dense::move_to(MultiVector* result) +{ + result->set_size(this->get_size()); + this->set_size(dim<2>{0, 0}); + result->stride_ = std::exchange(stride_, 0); + result->values_ = std::move(values_); +} + + +template +void Dense::convert_to( + Dense>* result) const +{ + if (result->get_size() != this->get_size()) { + result->set_size(this->get_size()); + result->stride_ = stride_; + result->values_.resize_and_reset(result->get_size()[0] * + result->stride_); + } + auto exec = this->get_executor(); + exec->run(multivector::make_copy( + this->get_const_device_view(), + make_temporary_output_clone(exec, result)->get_device_view())); +} + + +template +void Dense::move_to(Dense>* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +#if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16 +template +void Dense::convert_to( + Dense>* result) const +{ + if (result->get_size() != this->get_size()) { + result->set_size(this->get_size()); + result->stride_ = stride_; + result->values_.resize_and_reset(result->get_size()[0] * + result->stride_); + } + auto exec = this->get_executor(); + exec->run(multivector::make_copy( + this->get_const_device_view(), + make_temporary_output_clone(exec, result)->get_device_view())); +} + + +template +void Dense::move_to(Dense>* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} +#endif + + +#if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 +template +void Dense::convert_to( + Dense>* result) const +{ + if (result->get_size() != this->get_size()) { + result->set_size(this->get_size()); + result->stride_ = stride_; + result->values_.resize_and_reset(result->get_size()[0] * + result->stride_); + } + auto exec = this->get_executor(); + exec->run(multivector::make_copy( + this->get_const_device_view(), + make_temporary_output_clone(exec, result)->get_device_view())); +} + + +template +void Dense::move_to(Dense>* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} +#endif + + +template +template +void Dense::convert_impl(Coo* result) const +{ + auto exec = this->get_executor(); + const auto num_rows = this->get_size()[0]; + + array row_ptrs{exec, num_rows + 1}; + exec->run(dense::make_count_nonzeros_per_row(this->get_const_device_view(), + row_ptrs.get_data())); + exec->run( + dense::make_prefix_sum_nonnegative(row_ptrs.get_data(), num_rows + 1)); + const auto nnz = get_element(row_ptrs, num_rows); + result->resize(this->get_size(), nnz); + exec->run(dense::make_convert_to_coo( + this->get_const_device_view(), row_ptrs.get_const_data(), + make_temporary_clone(exec, result)->get_device_view())); +} + + +template +void Dense::convert_to(Coo* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Coo* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +void Dense::convert_to(Coo* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Coo* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +template +void Dense::convert_impl(Csr* result) const +{ + { + auto exec = this->get_executor(); + const auto num_rows = this->get_size()[0]; + auto tmp = make_temporary_clone(exec, result); + tmp->row_ptrs_.resize_and_reset(num_rows + 1); + exec->run(dense::make_count_nonzeros_per_row( + this->get_const_device_view(), tmp->get_row_ptrs())); + exec->run(dense::make_prefix_sum_nonnegative(tmp->get_row_ptrs(), + num_rows + 1)); + const auto nnz = + exec->copy_val_to_host(tmp->get_const_row_ptrs() + num_rows); + tmp->col_idxs_.resize_and_reset(nnz); + tmp->values_.resize_and_reset(nnz); + tmp->set_size(this->get_size()); + exec->run(dense::make_convert_to_csr(this->get_const_device_view(), + tmp.get())); + } + result->make_srow(); +} + + +template +void Dense::convert_to(Csr* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Csr* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +void Dense::convert_to(Csr* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Csr* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +template +void Dense::convert_impl(Fbcsr* result) const +{ + auto exec = this->get_executor(); + const auto bs = result->get_block_size(); + const auto row_blocks = detail::get_num_blocks(bs, this->get_size()[0]); + const auto col_blocks = detail::get_num_blocks(bs, this->get_size()[1]); + auto tmp = make_temporary_clone(exec, result); + tmp->row_ptrs_.resize_and_reset(row_blocks + 1); + exec->run(dense::make_count_nonzero_blocks_per_row( + this->get_const_device_view(), bs, tmp->get_row_ptrs())); + exec->run(dense::make_prefix_sum_nonnegative(tmp->get_row_ptrs(), + row_blocks + 1)); + const auto nnz_blocks = + exec->copy_val_to_host(tmp->get_const_row_ptrs() + row_blocks); + tmp->col_idxs_.resize_and_reset(nnz_blocks); + tmp->values_.resize_and_reset(nnz_blocks * bs * bs); + tmp->values_.fill(zero()); + tmp->set_size(this->get_size()); + exec->run( + dense::make_convert_to_fbcsr(this->get_const_device_view(), tmp.get())); +} + + +template +void Dense::convert_to(Fbcsr* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Fbcsr* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +void Dense::convert_to(Fbcsr* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Fbcsr* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +template +void Dense::convert_impl(Ell* result) const +{ + auto exec = this->get_executor(); + size_type num_stored_elements_per_row{}; + exec->run(dense::make_compute_max_nnz_per_row(this->get_const_device_view(), + num_stored_elements_per_row)); + result->resize(this->get_size(), num_stored_elements_per_row); + exec->run(dense::make_convert_to_ell( + this->get_const_device_view(), + make_temporary_clone(exec, result)->get_device_view())); +} + + +template +void Dense::convert_to(Ell* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Ell* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +void Dense::convert_to(Ell* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Ell* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +template +void Dense::convert_impl(Hybrid* result) const +{ + auto exec = this->get_executor(); + const auto num_rows = this->get_size()[0]; + const auto num_cols = this->get_size()[1]; + array row_nnz{exec, num_rows}; + array coo_row_ptrs{exec, num_rows + 1}; + exec->run(dense::make_count_nonzeros_per_row(this->get_const_device_view(), + row_nnz.get_data())); + size_type ell_lim{}; + size_type coo_nnz{}; + result->get_strategy()->compute_hybrid_config(row_nnz, &ell_lim, &coo_nnz); + if (ell_lim > num_cols) { + // TODO remove temporary fix after ELL gains true structural zeros + ell_lim = num_cols; + } + exec->run(dense::make_compute_hybrid_coo_row_ptrs(row_nnz, ell_lim, + coo_row_ptrs.get_data())); + coo_nnz = get_element(coo_row_ptrs, num_rows); + auto tmp = make_temporary_clone(exec, result); + tmp->resize(this->get_size(), ell_lim, coo_nnz); + exec->run(dense::make_convert_to_hybrid(this->get_const_device_view(), + coo_row_ptrs.get_const_data(), + tmp->get_device_view())); +} + + +template +void Dense::convert_to(Hybrid* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Hybrid* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +void Dense::convert_to(Hybrid* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Hybrid* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +template +void Dense::convert_impl(Sellp* result) const +{ + auto exec = this->get_executor(); + const auto num_rows = this->get_size()[0]; + const auto stride_factor = result->get_stride_factor(); + const auto slice_size = result->get_slice_size(); + const auto num_slices = ceildiv(num_rows, slice_size); + auto tmp = make_temporary_clone(exec, result); + tmp->stride_factor_ = stride_factor; + tmp->slice_size_ = slice_size; + tmp->slice_sets_.resize_and_reset(num_slices + 1); + tmp->slice_lengths_.resize_and_reset(num_slices); + exec->run(dense::make_compute_slice_sets( + this->get_const_device_view(), slice_size, stride_factor, + tmp->get_slice_sets(), tmp->get_slice_lengths())); + auto total_cols = + exec->copy_val_to_host(tmp->get_slice_sets() + num_slices); + tmp->col_idxs_.resize_and_reset(total_cols * slice_size); + tmp->values_.resize_and_reset(total_cols * slice_size); + tmp->set_size(this->get_size()); + exec->run(dense::make_convert_to_sellp(this->get_const_device_view(), + tmp->get_device_view())); +} + + +template +void Dense::convert_to(Sellp* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Sellp* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +void Dense::convert_to(Sellp* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(Sellp* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +template +void Dense::convert_impl( + SparsityCsr* result) const +{ + auto exec = this->get_executor(); + const auto num_rows = this->get_size()[0]; + auto tmp = make_temporary_clone(exec, result); + tmp->row_ptrs_.resize_and_reset(num_rows + 1); + exec->run(dense::make_count_nonzeros_per_row(this->get_const_device_view(), + tmp->row_ptrs_.get_data())); + exec->run(dense::make_prefix_sum_nonnegative(tmp->row_ptrs_.get_data(), + num_rows + 1)); + const auto nnz = get_element(tmp->row_ptrs_, num_rows); + tmp->col_idxs_.resize_and_reset(nnz); + tmp->value_.fill(one()); + tmp->set_size(this->get_size()); + exec->run(dense::make_convert_to_sparsity_csr(this->get_const_device_view(), + tmp.get())); +} + + +template +void Dense::add_scaled_identity_impl(const LinOp* a, const LinOp* b) +{ + precision_dispatch_real_complex( + [this](auto dense_alpha, auto dense_beta, auto dense_x) { + this->get_executor()->run(dense::make_add_scaled_identity( + dense_alpha->get_const_device_view(), + dense_beta->get_const_device_view(), + dense_x->get_device_view())); + }, + a, b, this); +} + + +template +void Dense::convert_to(SparsityCsr* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(SparsityCsr* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +void Dense::convert_to(SparsityCsr* result) const +{ + this->convert_impl(result); +} + + +template +void Dense::move_to(SparsityCsr* result) +{ + this->convert_to(result); + this->set_size(dim<2>{0, 0}); + this->stride_ = 0; + this->values_.resize_and_reset(0); +} + + +template +void Dense::read(const mat_data32& data) +{ + this->read(device_mat_data32::create_from_host(this->get_executor(), data)); +} + + +template +void Dense::read(const mat_data64& data) +{ + this->read(device_mat_data64::create_from_host(this->get_executor(), data)); +} + + +template +void Dense::read(const device_mat_data32& data) +{ + auto exec = this->get_executor(); + this->resize(data.get_size()); + this->fill(zero()); + exec->run(multivector::make_fill_in_matrix_data( + *make_temporary_clone(exec, &data), this->get_device_view())); +} + + +template +void Dense::read(const device_mat_data64& data) +{ + auto exec = this->get_executor(); + this->resize(data.get_size()); + this->fill(zero()); + exec->run(multivector::make_fill_in_matrix_data( + *make_temporary_clone(exec, &data), this->get_device_view())); +} + + +template +void Dense::read(device_mat_data32&& data) +{ + this->read(data); + data.empty_out(); +} + + +template +void Dense::read(device_mat_data64&& data) +{ + this->read(data); + data.empty_out(); +} + + +template +void Dense::write(matrix_data& data) const +{ + this->as_const_multivector_view()->write(data); +} + + +template +void Dense::write(matrix_data& data) const +{ + this->as_const_multivector_view()->write(data); +} + + +template +void Dense::fill(ValueType value) +{ + this->get_executor()->run( + multivector::make_fill(this->get_device_view(), value)); +} + + +template +void Dense::extract_diagonal( + ptr_param> output) const +{ + auto exec = this->get_executor(); + const auto diag_size = std::min(this->get_size()[0], this->get_size()[1]); + GKO_ASSERT_EQ(output->get_size()[0], diag_size); + + exec->run(dense::make_extract_diagonal( + this->get_const_device_view(), + make_temporary_output_clone(exec, output).get())); +} + + +template +std::unique_ptr> Dense::extract_diagonal() const +{ + const auto diag_size = std::min(this->get_size()[0], this->get_size()[1]); + auto diag = Diagonal::create(this->get_executor(), diag_size); + this->extract_diagonal(diag); + return diag; +} + + +template +std::unique_ptr Dense::transpose() const +{ + auto result = + Dense::create(this->get_executor(), gko::transpose(this->get_size())); + this->transpose(result); + return result; +} + + +template +std::unique_ptr Dense::conj_transpose() const +{ + auto result = + Dense::create(this->get_executor(), gko::transpose(this->get_size())); + this->conj_transpose(result); + return result; +} + + +template +void Dense::transpose(ptr_param output) const +{ + this->as_const_multivector_view()->transpose(output->as_multivector_view()); +} + + +template +void Dense::conj_transpose(ptr_param output) const +{ + this->as_const_multivector_view()->conj_transpose( + output->as_multivector_view()); +} + + +template +void Dense::add_scaled(ptr_param alpha, + ptr_param> diag) +{ + GKO_ASSERT_EQUAL_ROWS(alpha, dim<2>(1, 1)); + if (alpha->get_size()[1] != 1) { + // different alpha for each column + GKO_ASSERT_EQUAL_COLS(this, alpha); + } + GKO_ASSERT_EQUAL_DIMENSIONS(this, diag); + auto exec = this->get_executor(); + exec->run(dense::make_add_scaled_diag( + make_temporary_conversion(alpha)->get_const_device_view(), + diag.get(), this->get_device_view())); +} + + +template +void Dense::sub_scaled(ptr_param alpha, + ptr_param> diag) +{ + GKO_ASSERT_EQUAL_ROWS(alpha, dim<2>(1, 1)); + if (alpha->get_size()[1] != 1) { + // different alpha for each column + GKO_ASSERT_EQUAL_COLS(this, alpha); + } + GKO_ASSERT_EQUAL_DIMENSIONS(this, diag); + auto exec = this->get_executor(); + exec->run(dense::make_sub_scaled_diag( + make_temporary_conversion(alpha)->get_const_device_view(), + diag.get(), this->get_device_view())); +} + + +template +std::unique_ptr> Dense::create( + std::shared_ptr exec, const dim<2>& size, size_type stride) +{ + return std::unique_ptr{new Dense{std::move(exec), size, stride}}; +} + + +template +std::unique_ptr> Dense::create( + std::shared_ptr exec, const dim<2>& size, + array values, size_type stride) +{ + return std::unique_ptr{ + new Dense{std::move(exec), size, std::move(values), stride}}; +} + + +template +std::unique_ptr> Dense::create_const( + std::shared_ptr exec, const dim<2>& size, + ::gko::detail::const_array_view&& values, size_type stride) +{ + return std::unique_ptr{new Dense{ + exec, size, gko::detail::array_const_cast(std::move(values)), stride}}; +} + + +template +std::unique_ptr> Dense::create_subview(span rows, + span cols) +{ + row_major_range range_this{this->get_values(), this->get_size()[0], + this->get_size()[1], this->get_stride()}; + auto sub_range = range_this(rows, cols); + size_type storage_size = + rows.length() > 0 ? sub_range.length(1) + + (sub_range.length(0) - 1) * this->get_stride() + : 0; + return Dense::create( + this->get_executor(), dim<2>{sub_range.length(0), sub_range.length(1)}, + make_array_view(this->get_executor(), storage_size, sub_range->data), + this->get_stride()); +} + + +template +std::unique_ptr> Dense::create_subview( + span rows, span cols) const +{ + return const_cast(this)->create_subview(rows, cols); +} + + +template +std::unique_ptr> Dense::create_const_subview( + span rows, span cols) const +{ + return this->create_subview(rows, cols); +} + + +template +std::unique_ptr> +Dense::as_const_multivector_view() const +{ + return MultiVector::create_const( + this->get_executor(), this->get_size(), this->values_.as_const_view(), + stride_); +} + + +template +std::unique_ptr> Dense::as_multivector_view() +{ + return MultiVector::create(this->get_executor(), + this->get_size(), + this->values_.as_view(), stride_); +} + + +template +typename Dense::device_view Dense::get_device_view() +{ + return device_view{this->get_size(), this->stride_, + this->values_.get_data()}; +} + + +template +typename Dense::const_device_view +Dense::get_const_device_view() const +{ + return const_device_view{this->get_size(), this->stride_, + this->values_.get_const_data()}; +} + + +template +ValueType& Dense::at(size_type row, size_type col) +{ + return values_.get_data()[linearize_index(row, col)]; +} + + +template +ValueType Dense::at(size_type row, size_type col) const +{ + return values_.get_const_data()[linearize_index(row, col)]; +} + + +template +size_type Dense::get_stride() const noexcept +{ + return stride_; +} + + +template +size_type Dense::get_num_stored_elements() const noexcept +{ + return this->values_.get_size(); +} + + +template +Dense::Dense(const Dense& other) : LinOp(other.get_executor()) +{ + *this = other; +} + + +template +Dense::Dense(Dense&& other) : LinOp(other.get_executor()) +{ + *this = std::move(other); +} + + +template +Dense& Dense::operator=(const Dense& other) +{ + if (&other != this) { + auto old_size = this->get_size(); + LinOp::operator=(other); + // NOTE: keep this consistent with resize(...) + if (old_size != other.get_size()) { + this->stride_ = this->get_size()[1]; + this->values_.resize_and_reset(this->get_size()[0] * this->stride_); + } + // we need to create a executor-local clone of the target data, that + // will be copied back later. Need temporary_clone, not + // temporary_output_clone to avoid overwriting padding + auto exec = other.get_executor(); + auto exec_values_array = + make_temporary_output_clone(exec, &this->values_); + exec->run( + multivector::make_copy(other.get_const_device_view(), + device_view{this->get_size(), this->stride_, + exec_values_array->get_data()})); + } + return *this; +} + + +template +Dense& Dense::operator=(Dense&& other) +{ + if (&other != this) { + LinOp::operator=(std::move(other)); + stride_ = std::exchange(other.stride_, 0); + values_ = std::move(other.values_); + } + return *this; +} + + +template +Dense::Dense(std::shared_ptr exec, + const dim<2>& size, size_type stride) + : LinOp(exec, size), + stride_(stride == 0 ? size[1] : stride), + values_(exec, size[0] * stride_) +{} + + +template +Dense::Dense(std::shared_ptr exec, + const dim<2>& size, array values, + size_type stride) + : LinOp(exec, size), + stride_(stride == 0 ? size[1] : stride), + values_(exec, std::move(values)) +{ + if (size[0] > 0 && size[1] > 0) { + GKO_ENSURE_IN_BOUNDS((size[0] - 1) * stride_ + size[1] - 1, + values_.get_size()); + } +} + + +template +void Dense::apply_impl(const LinOp* b, LinOp* x) const +{ + precision_dispatch_real_complex( + [this](auto dense_b, auto dense_x) { + this->get_executor()->run(dense::make_simple_apply( + this->get_const_device_view(), dense_b->get_const_device_view(), + dense_x->get_device_view())); + }, + b, x); +} + + +template +void Dense::apply_impl(const LinOp* alpha, const LinOp* b, + const LinOp* beta, LinOp* x) const +{ + precision_dispatch_real_complex( + [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { + this->get_executor()->run(dense::make_advanced_apply( + dense_alpha->get_const_device_view(), + this->get_const_device_view(), dense_b->get_const_device_view(), + dense_beta->get_const_device_view(), + dense_x->get_device_view())); + }, + alpha, b, beta, x); +} + + +template +size_type Dense::linearize_index(size_type row, + size_type col) const noexcept +{ + return row * stride_ + col; +} + + +template +void Dense::resize(dim<2> new_size) +{ + if (this->get_size() != new_size) { + this->set_size(new_size); + this->stride_ = new_size[1]; + this->values_.resize_and_reset(new_size[0] * this->get_stride()); + } +} + + +#define GKO_DECLARE_DENSE(ValueType) class Dense +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE); + + +} // namespace matrix +} // namespace gko diff --git a/core/matrix/dense_kernels.hpp b/core/matrix/dense_kernels.hpp new file mode 100644 index 00000000000..4df5cde9c7d --- /dev/null +++ b/core/matrix/dense_kernels.hpp @@ -0,0 +1,172 @@ +// SPDX-FileCopyrightText: 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "core/base/kernel_declaration.hpp" + + +namespace gko { +namespace kernels { + + +#define GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL(ValueType) \ + void simple_apply(std::shared_ptr exec, \ + matrix::view::dense a, \ + matrix::view::dense b, \ + matrix::view::dense c) + +#define GKO_DECLARE_DENSE_APPLY_KERNEL(ValueType) \ + void apply(std::shared_ptr exec, \ + matrix::view::dense alpha, \ + matrix::view::dense a, \ + matrix::view::dense b, \ + matrix::view::dense beta, \ + matrix::view::dense c) + +#define GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL(ValueType, IndexType) \ + void convert_to_coo(std::shared_ptr exec, \ + matrix::view::dense source, \ + const int64* row_ptrs, \ + matrix::view::coo other) + +#define GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL(ValueType, IndexType) \ + void convert_to_csr(std::shared_ptr exec, \ + matrix::view::dense source, \ + matrix::Csr* other) + +#define GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL(ValueType, IndexType) \ + void convert_to_ell(std::shared_ptr exec, \ + matrix::view::dense source, \ + matrix::view::ell other) + +#define GKO_DECLARE_DENSE_CONVERT_TO_FBCSR_KERNEL(ValueType, IndexType) \ + void convert_to_fbcsr(std::shared_ptr exec, \ + matrix::view::dense source, \ + matrix::Fbcsr* other) + +#define GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType) \ + void convert_to_hybrid(std::shared_ptr exec, \ + matrix::view::dense source, \ + const int64* coo_row_ptrs, \ + matrix::view::hybrid other) + +#define GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType) \ + void convert_to_sellp(std::shared_ptr exec, \ + matrix::view::dense source, \ + matrix::view::sellp other) + +#define GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL(ValueType, IndexType) \ + void convert_to_sparsity_csr( \ + std::shared_ptr exec, \ + matrix::view::dense source, \ + matrix::SparsityCsr* other) + +#define GKO_DECLARE_DENSE_COMPUTE_MAX_NNZ_PER_ROW_KERNEL(ValueType) \ + void compute_max_nnz_per_row(std::shared_ptr exec, \ + matrix::view::dense source, \ + size_type& result) + +#define GKO_DECLARE_DENSE_COMPUTE_SLICE_SETS_KERNEL(ValueType) \ + void compute_slice_sets(std::shared_ptr exec, \ + matrix::view::dense source, \ + size_type slice_size, size_type stride_factor, \ + size_type* slice_sets, size_type* slice_lengths) + +#define GKO_DECLARE_DENSE_COUNT_NONZEROS_PER_ROW_KERNEL(ValueType, IndexType) \ + void count_nonzeros_per_row(std::shared_ptr exec, \ + matrix::view::dense source, \ + IndexType* result) + +#define GKO_DECLARE_DENSE_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL(ValueType, \ + IndexType) \ + void count_nonzero_blocks_per_row( \ + std::shared_ptr exec, \ + matrix::view::dense source, int block_size, \ + IndexType* result) + +#define GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL(ValueType) \ + void extract_diagonal(std::shared_ptr exec, \ + matrix::view::dense orig, \ + matrix::Diagonal* diag) + +#define GKO_DECLARE_DENSE_COUNT_NONZEROS_PER_ROW_KERNEL_SIZE_T(ValueType) \ + GKO_DECLARE_DENSE_COUNT_NONZEROS_PER_ROW_KERNEL(ValueType, ::gko::size_type) + +#define GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL(ValueType) \ + void add_scaled_diag(std::shared_ptr exec, \ + matrix::view::dense alpha, \ + const matrix::Diagonal* x, \ + matrix::view::dense y) + +#define GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL(ValueType) \ + void sub_scaled_diag(std::shared_ptr exec, \ + matrix::view::dense alpha, \ + const matrix::Diagonal* x, \ + matrix::view::dense y) + +#define GKO_DECLARE_DENSE_ADD_SCALED_IDENTITY_KERNEL(ValueType, ScalarType) \ + void add_scaled_identity(std::shared_ptr exec, \ + matrix::view::dense alpha, \ + matrix::view::dense beta, \ + matrix::view::dense mtx) + + +#define GKO_DECLARE_ALL_AS_TEMPLATES \ + template \ + GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL(ValueType); \ + template \ + GKO_DECLARE_DENSE_APPLY_KERNEL(ValueType); \ + template \ + GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL(ValueType, IndexType); \ + template \ + GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL(ValueType, IndexType); \ + template \ + GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL(ValueType, IndexType); \ + template \ + GKO_DECLARE_DENSE_CONVERT_TO_FBCSR_KERNEL(ValueType, IndexType); \ + template \ + GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType); \ + template \ + GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType); \ + template \ + GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL(ValueType, IndexType); \ + template \ + GKO_DECLARE_DENSE_COMPUTE_MAX_NNZ_PER_ROW_KERNEL(ValueType); \ + template \ + GKO_DECLARE_DENSE_COMPUTE_SLICE_SETS_KERNEL(ValueType); \ + template \ + GKO_DECLARE_DENSE_COUNT_NONZEROS_PER_ROW_KERNEL(ValueType, IndexType); \ + template \ + GKO_DECLARE_DENSE_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL(ValueType, \ + IndexType); \ + template \ + GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL(ValueType); \ + template \ + GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL(ValueType); \ + template \ + GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL(ValueType); \ + template \ + GKO_DECLARE_DENSE_ADD_SCALED_IDENTITY_KERNEL(ValueType, ScalarType) + + +GKO_DECLARE_FOR_ALL_EXECUTOR_NAMESPACES(dense, GKO_DECLARE_ALL_AS_TEMPLATES); + + +#undef GKO_DECLARE_ALL_AS_TEMPLATES + + +} // namespace kernels +} // namespace gko diff --git a/core/matrix/ell.cpp b/core/matrix/ell.cpp index d65a605225f..5314bde47a2 100644 --- a/core/matrix/ell.cpp +++ b/core/matrix/ell.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -224,7 +225,7 @@ void Ell::move_to( template -void Ell::convert_to(MultiVector* result) const +void Ell::convert_to(Dense* result) const { auto exec = this->get_executor(); auto tmp_result = make_temporary_output_clone(exec, result); @@ -236,7 +237,7 @@ void Ell::convert_to(MultiVector* result) const template -void Ell::move_to(MultiVector* result) +void Ell::move_to(Dense* result) { this->convert_to(result); } diff --git a/core/matrix/fbcsr.cpp b/core/matrix/fbcsr.cpp index b5bfa63bcab..b422ceec4d1 100644 --- a/core/matrix/fbcsr.cpp +++ b/core/matrix/fbcsr.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -215,8 +216,7 @@ void Fbcsr::move_to( template -void Fbcsr::convert_to( - MultiVector* result) const +void Fbcsr::convert_to(Dense* result) const { auto exec = this->get_executor(); auto tmp_result = make_temporary_output_clone(exec, result); @@ -227,7 +227,7 @@ void Fbcsr::convert_to( template -void Fbcsr::move_to(MultiVector* result) +void Fbcsr::move_to(Dense* result) { this->convert_to(result); } diff --git a/core/matrix/hybrid.cpp b/core/matrix/hybrid.cpp index 5c4e94b610b..98e03b755a2 100644 --- a/core/matrix/hybrid.cpp +++ b/core/matrix/hybrid.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "core/base/array_access.hpp" @@ -286,8 +287,7 @@ void Hybrid::move_to( template -void Hybrid::convert_to( - MultiVector* result) const +void Hybrid::convert_to(Dense* result) const { auto exec = this->get_executor(); result->resize(this->get_size()); @@ -303,7 +303,7 @@ void Hybrid::convert_to( template -void Hybrid::move_to(MultiVector* result) +void Hybrid::move_to(Dense* result) { this->convert_to(result); } diff --git a/core/matrix/multivector.cpp b/core/matrix/multivector.cpp index 8c87992eb98..704f54d1522 100644 --- a/core/matrix/multivector.cpp +++ b/core/matrix/multivector.cpp @@ -15,21 +15,12 @@ #include #include #include -#include -#include +#include #include -#include -#include -#include #include #include -#include -#include -#include "core/base/array_access.hpp" #include "core/base/dispatch_helper.hpp" -#include "core/components/prefix_sum_kernels.hpp" -#include "core/matrix/hybrid_kernels.hpp" #include "core/matrix/multivector_kernels.hpp" #include "core/matrix/permutation.hpp" @@ -40,16 +31,12 @@ namespace multivector { namespace { -GKO_REGISTER_OPERATION(simple_apply, multivector::simple_apply); -GKO_REGISTER_OPERATION(apply, multivector::apply); GKO_REGISTER_OPERATION(copy, multivector::copy); GKO_REGISTER_OPERATION(fill, multivector::fill); GKO_REGISTER_OPERATION(scale, multivector::scale); GKO_REGISTER_OPERATION(inv_scale, multivector::inv_scale); GKO_REGISTER_OPERATION(add_scaled, multivector::add_scaled); GKO_REGISTER_OPERATION(sub_scaled, multivector::sub_scaled); -GKO_REGISTER_OPERATION(add_scaled_diag, multivector::add_scaled_diag); -GKO_REGISTER_OPERATION(sub_scaled_diag, multivector::sub_scaled_diag); GKO_REGISTER_OPERATION(compute_dot, multivector::compute_dot_dispatch); GKO_REGISTER_OPERATION(compute_conj_dot, multivector::compute_conj_dot_dispatch); @@ -59,17 +46,6 @@ GKO_REGISTER_OPERATION(compute_mean, multivector::compute_mean); GKO_REGISTER_OPERATION(compute_squared_norm2, multivector::compute_squared_norm2); GKO_REGISTER_OPERATION(compute_sqrt, multivector::compute_sqrt); -GKO_REGISTER_OPERATION(compute_max_nnz_per_row, - multivector::compute_max_nnz_per_row); -GKO_REGISTER_OPERATION(compute_hybrid_coo_row_ptrs, - hybrid::compute_coo_row_ptrs); -GKO_REGISTER_OPERATION(count_nonzeros_per_row, - multivector::count_nonzeros_per_row); -GKO_REGISTER_OPERATION(count_nonzero_blocks_per_row, - multivector::count_nonzero_blocks_per_row); -GKO_REGISTER_OPERATION(prefix_sum_nonnegative, - components::prefix_sum_nonnegative); -GKO_REGISTER_OPERATION(compute_slice_sets, multivector::compute_slice_sets); GKO_REGISTER_OPERATION(transpose, multivector::transpose); GKO_REGISTER_OPERATION(conj_transpose, multivector::conj_transpose); GKO_REGISTER_OPERATION(symm_permute, multivector::symm_permute); @@ -95,15 +71,6 @@ GKO_REGISTER_OPERATION(inv_row_scale_permute, GKO_REGISTER_OPERATION(inv_col_scale_permute, multivector::inv_col_scale_permute); GKO_REGISTER_OPERATION(fill_in_matrix_data, multivector::fill_in_matrix_data); -GKO_REGISTER_OPERATION(convert_to_coo, multivector::convert_to_coo); -GKO_REGISTER_OPERATION(convert_to_csr, multivector::convert_to_csr); -GKO_REGISTER_OPERATION(convert_to_ell, multivector::convert_to_ell); -GKO_REGISTER_OPERATION(convert_to_fbcsr, multivector::convert_to_fbcsr); -GKO_REGISTER_OPERATION(convert_to_hybrid, multivector::convert_to_hybrid); -GKO_REGISTER_OPERATION(convert_to_sellp, multivector::convert_to_sellp); -GKO_REGISTER_OPERATION(convert_to_sparsity_csr, - multivector::convert_to_sparsity_csr); -GKO_REGISTER_OPERATION(extract_diagonal, multivector::extract_diagonal); GKO_REGISTER_OPERATION(inplace_absolute_dense, multivector::inplace_absolute_dense); GKO_REGISTER_OPERATION(outplace_absolute_dense, @@ -111,7 +78,6 @@ GKO_REGISTER_OPERATION(outplace_absolute_dense, GKO_REGISTER_OPERATION(make_complex, multivector::make_complex); GKO_REGISTER_OPERATION(get_real, multivector::get_real); GKO_REGISTER_OPERATION(get_imag, multivector::get_imag); -GKO_REGISTER_OPERATION(add_scaled_identity, multivector::add_scaled_identity); } // anonymous namespace @@ -119,33 +85,14 @@ GKO_REGISTER_OPERATION(add_scaled_identity, multivector::add_scaled_identity); template -void MultiVector::apply_impl(const LinOp* b, LinOp* x) const -{ - precision_dispatch_real_complex( - [this](auto dense_b, auto dense_x) { - this->get_executor()->run(multivector::make_simple_apply( - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_x->get_device_view())); - }, - b, x); -} +void MultiVector::apply_impl(const LinOp* b, + LinOp* x) const GKO_NOT_IMPLEMENTED; template void MultiVector::apply_impl(const LinOp* alpha, const LinOp* b, - const LinOp* beta, LinOp* x) const -{ - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_b, auto dense_beta, auto dense_x) { - this->get_executor()->run(multivector::make_apply( - dense_alpha->get_const_device_view(), - this->get_const_device_view(), dense_b->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); - }, - alpha, b, beta, x); -} - + const LinOp* beta, + LinOp* x) const GKO_NOT_IMPLEMENTED; template void MultiVector::fill(const ValueType value) @@ -317,20 +264,11 @@ void MultiVector::add_scaled_impl(const LinOp* alpha, const LinOp* b) ->get_const_device_view(), dynamic_cast(this)->get_device_view())); } else { - if (dynamic_cast*>(b)) { - exec->run(multivector::make_add_scaled_diag( - make_temporary_conversion(alpha) - ->get_const_device_view(), - dynamic_cast*>(b), - this->get_device_view())); - } else { - exec->run(multivector::make_add_scaled( - make_temporary_conversion(alpha) - ->get_const_device_view(), - make_temporary_conversion(b) - ->get_const_device_view(), - this->get_device_view())); - } + exec->run(multivector::make_add_scaled( + make_temporary_conversion(alpha) + ->get_const_device_view(), + make_temporary_conversion(b)->get_const_device_view(), + this->get_device_view())); } } @@ -355,20 +293,11 @@ void MultiVector::sub_scaled_impl(const LinOp* alpha, const LinOp* b) ->get_const_device_view(), dynamic_cast(this)->get_device_view())); } else { - if (dynamic_cast*>(b)) { - exec->run(multivector::make_sub_scaled_diag( - make_temporary_conversion(alpha) - ->get_const_device_view(), - dynamic_cast*>(b), - this->get_device_view())); - } else { - exec->run(multivector::make_sub_scaled( - make_temporary_conversion(alpha) - ->get_const_device_view(), - make_temporary_conversion(b) - ->get_const_device_view(), - this->get_device_view())); - } + exec->run(multivector::make_sub_scaled( + make_temporary_conversion(alpha) + ->get_const_device_view(), + make_temporary_conversion(b)->get_const_device_view(), + this->get_device_view())); } } @@ -712,369 +641,28 @@ void MultiVector::move_to( template -template -void MultiVector::convert_impl( - Coo* result) const -{ - auto exec = this->get_executor(); - const auto num_rows = this->get_size()[0]; - - array row_ptrs{exec, num_rows + 1}; - exec->run(multivector::make_count_nonzeros_per_row( - this->get_const_device_view(), row_ptrs.get_data())); - exec->run(multivector::make_prefix_sum_nonnegative(row_ptrs.get_data(), - num_rows + 1)); - const auto nnz = get_element(row_ptrs, num_rows); - result->resize(this->get_size(), nnz); - exec->run(multivector::make_convert_to_coo( - this->get_const_device_view(), row_ptrs.get_const_data(), - make_temporary_clone(exec, result)->get_device_view())); -} - - -template -void MultiVector::convert_to(Coo* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Coo* result) +void MultiVector::convert_to(Dense* result) const { - this->convert_to(result); -} - - -template -void MultiVector::convert_to(Coo* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Coo* result) -{ - this->convert_to(result); -} - - -template -template -void MultiVector::convert_impl( - Csr* result) const -{ - { - auto exec = this->get_executor(); - const auto num_rows = this->get_size()[0]; - auto tmp = make_temporary_clone(exec, result); - tmp->row_ptrs_.resize_and_reset(num_rows + 1); - exec->run(multivector::make_count_nonzeros_per_row( - this->get_const_device_view(), tmp->get_row_ptrs())); - exec->run(multivector::make_prefix_sum_nonnegative(tmp->get_row_ptrs(), - num_rows + 1)); - const auto nnz = - exec->copy_val_to_host(tmp->get_const_row_ptrs() + num_rows); - tmp->col_idxs_.resize_and_reset(nnz); - tmp->values_.resize_and_reset(nnz); - tmp->set_size(this->get_size()); - exec->run(multivector::make_convert_to_csr( - this->get_const_device_view(), tmp.get())); + if (result->get_size() != this->get_size()) { + result->set_size(this->get_size()); + result->stride_ = stride_; + result->values_.resize_and_reset(result->get_size()[0] * + result->stride_); } - result->make_srow(); -} - - -template -void MultiVector::convert_to(Csr* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Csr* result) -{ - this->convert_to(result); -} - - -template -void MultiVector::convert_to(Csr* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Csr* result) -{ - this->convert_to(result); -} - - -template -template -void MultiVector::convert_impl( - Fbcsr* result) const -{ - auto exec = this->get_executor(); - const auto bs = result->get_block_size(); - const auto row_blocks = detail::get_num_blocks(bs, this->get_size()[0]); - const auto col_blocks = detail::get_num_blocks(bs, this->get_size()[1]); - auto tmp = make_temporary_clone(exec, result); - tmp->row_ptrs_.resize_and_reset(row_blocks + 1); - exec->run(multivector::make_count_nonzero_blocks_per_row( - this->get_const_device_view(), bs, tmp->get_row_ptrs())); - exec->run(multivector::make_prefix_sum_nonnegative(tmp->get_row_ptrs(), - row_blocks + 1)); - const auto nnz_blocks = - exec->copy_val_to_host(tmp->get_const_row_ptrs() + row_blocks); - tmp->col_idxs_.resize_and_reset(nnz_blocks); - tmp->values_.resize_and_reset(nnz_blocks * bs * bs); - tmp->values_.fill(zero()); - tmp->set_size(this->get_size()); - exec->run(multivector::make_convert_to_fbcsr(this->get_const_device_view(), - tmp.get())); -} - - -template -void MultiVector::convert_to(Fbcsr* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Fbcsr* result) -{ - this->convert_to(result); -} - - -template -void MultiVector::convert_to(Fbcsr* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Fbcsr* result) -{ - this->convert_to(result); -} - - -template -template -void MultiVector::convert_impl( - Ell* result) const -{ auto exec = this->get_executor(); - size_type num_stored_elements_per_row{}; - exec->run(multivector::make_compute_max_nnz_per_row( - this->get_const_device_view(), num_stored_elements_per_row)); - result->resize(this->get_size(), num_stored_elements_per_row); - exec->run(multivector::make_convert_to_ell( + exec->run(multivector::make_copy( this->get_const_device_view(), - make_temporary_clone(exec, result)->get_device_view())); -} - - -template -void MultiVector::convert_to(Ell* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Ell* result) -{ - this->convert_to(result); -} - - -template -void MultiVector::convert_to(Ell* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Ell* result) -{ - this->convert_to(result); -} - - -template -template -void MultiVector::convert_impl( - Hybrid* result) const -{ - auto exec = this->get_executor(); - const auto num_rows = this->get_size()[0]; - const auto num_cols = this->get_size()[1]; - array row_nnz{exec, num_rows}; - array coo_row_ptrs{exec, num_rows + 1}; - exec->run(multivector::make_count_nonzeros_per_row( - this->get_const_device_view(), row_nnz.get_data())); - size_type ell_lim{}; - size_type coo_nnz{}; - result->get_strategy()->compute_hybrid_config(row_nnz, &ell_lim, &coo_nnz); - if (ell_lim > num_cols) { - // TODO remove temporary fix after ELL gains true structural zeros - ell_lim = num_cols; - } - exec->run(multivector::make_compute_hybrid_coo_row_ptrs( - row_nnz, ell_lim, coo_row_ptrs.get_data())); - coo_nnz = get_element(coo_row_ptrs, num_rows); - auto tmp = make_temporary_clone(exec, result); - tmp->resize(this->get_size(), ell_lim, coo_nnz); - exec->run(multivector::make_convert_to_hybrid(this->get_const_device_view(), - coo_row_ptrs.get_const_data(), - tmp->get_device_view())); -} - - -template -void MultiVector::convert_to(Hybrid* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Hybrid* result) -{ - this->convert_to(result); -} - - -template -void MultiVector::convert_to(Hybrid* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Hybrid* result) -{ - this->convert_to(result); -} - - -template -template -void MultiVector::convert_impl( - Sellp* result) const -{ - auto exec = this->get_executor(); - const auto num_rows = this->get_size()[0]; - const auto stride_factor = result->get_stride_factor(); - const auto slice_size = result->get_slice_size(); - const auto num_slices = ceildiv(num_rows, slice_size); - auto tmp = make_temporary_clone(exec, result); - tmp->stride_factor_ = stride_factor; - tmp->slice_size_ = slice_size; - tmp->slice_sets_.resize_and_reset(num_slices + 1); - tmp->slice_lengths_.resize_and_reset(num_slices); - exec->run(multivector::make_compute_slice_sets( - this->get_const_device_view(), slice_size, stride_factor, - tmp->get_slice_sets(), tmp->get_slice_lengths())); - auto total_cols = - exec->copy_val_to_host(tmp->get_slice_sets() + num_slices); - tmp->col_idxs_.resize_and_reset(total_cols * slice_size); - tmp->values_.resize_and_reset(total_cols * slice_size); - tmp->set_size(this->get_size()); - exec->run(multivector::make_convert_to_sellp(this->get_const_device_view(), - tmp->get_device_view())); -} - - -template -void MultiVector::convert_to(Sellp* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Sellp* result) -{ - this->convert_to(result); -} - - -template -void MultiVector::convert_to(Sellp* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(Sellp* result) -{ - this->convert_to(result); -} - - -template -template -void MultiVector::convert_impl( - SparsityCsr* result) const -{ - auto exec = this->get_executor(); - const auto num_rows = this->get_size()[0]; - auto tmp = make_temporary_clone(exec, result); - tmp->row_ptrs_.resize_and_reset(num_rows + 1); - exec->run(multivector::make_count_nonzeros_per_row( - this->get_const_device_view(), tmp->row_ptrs_.get_data())); - exec->run(multivector::make_prefix_sum_nonnegative( - tmp->row_ptrs_.get_data(), num_rows + 1)); - const auto nnz = get_element(tmp->row_ptrs_, num_rows); - tmp->col_idxs_.resize_and_reset(nnz); - tmp->value_.fill(one()); - tmp->set_size(this->get_size()); - exec->run(multivector::make_convert_to_sparsity_csr( - this->get_const_device_view(), tmp.get())); -} - - -template -void MultiVector::convert_to( - SparsityCsr* result) const -{ - this->convert_impl(result); -} - - -template -void MultiVector::move_to(SparsityCsr* result) -{ - this->convert_to(result); -} - - -template -void MultiVector::convert_to( - SparsityCsr* result) const -{ - this->convert_impl(result); + make_temporary_output_clone(exec, result)->get_device_view())); } template -void MultiVector::move_to(SparsityCsr* result) +void MultiVector::move_to(Dense* result) { - this->convert_to(result); + result->set_size(this->get_size()); + this->set_size(dim<2>{0, 0}); + result->stride_ = std::exchange(stride_, 0); + result->values_ = std::move(values_); } @@ -1947,31 +1535,6 @@ void MultiVector::scale_permute( } -template -void MultiVector::extract_diagonal( - ptr_param> output) const -{ - auto exec = this->get_executor(); - const auto diag_size = std::min(this->get_size()[0], this->get_size()[1]); - GKO_ASSERT_EQ(output->get_size()[0], diag_size); - - exec->run(multivector::make_extract_diagonal( - this->get_const_device_view(), - make_temporary_output_clone(exec, output).get())); -} - - -template -std::unique_ptr> MultiVector::extract_diagonal() - const -{ - const auto diag_size = std::min(this->get_size()[0], this->get_size()[1]); - auto diag = Diagonal::create(this->get_executor(), diag_size); - this->extract_diagonal(diag); - return diag; -} - - template void MultiVector::compute_absolute_inplace() { @@ -2086,21 +1649,6 @@ auto MultiVector::get_const_device_view() const -> const_device_view }; -template -void MultiVector::add_scaled_identity_impl(const LinOp* a, - const LinOp* b) -{ - precision_dispatch_real_complex( - [this](auto dense_alpha, auto dense_beta, auto dense_x) { - this->get_executor()->run(multivector::make_add_scaled_identity( - dense_alpha->get_const_device_view(), - dense_beta->get_const_device_view(), - dense_x->get_device_view())); - }, - a, b, this); -} - - template std::unique_ptr::real_type> MultiVector::create_real_view() @@ -2191,6 +1739,24 @@ MultiVector::create_const( } +template +std::unique_ptr> +MultiVector::as_const_dense_view() const +{ + return Dense::create_const(this->get_executor(), + this->get_size(), + values_.as_const_view(), stride_); +} + + +template +std::unique_ptr> MultiVector::as_dense_view() +{ + return Dense::create(this->get_executor(), this->get_size(), + values_.as_view(), stride_); +} + + template MultiVector::MultiVector(std::shared_ptr exec, const dim<2>& size, size_type stride) diff --git a/core/matrix/multivector_kernels.hpp b/core/matrix/multivector_kernels.hpp index e85e6ef0fae..dcfb85f1a8a 100644 --- a/core/matrix/multivector_kernels.hpp +++ b/core/matrix/multivector_kernels.hpp @@ -21,20 +21,6 @@ namespace gko { namespace kernels { -#define GKO_DECLARE_MULTIVECTOR_SIMPLE_APPLY_KERNEL(ValueType) \ - void simple_apply(std::shared_ptr exec, \ - matrix::view::dense a, \ - matrix::view::dense b, \ - matrix::view::dense c) - -#define GKO_DECLARE_MULTIVECTOR_APPLY_KERNEL(ValueType) \ - void apply(std::shared_ptr exec, \ - matrix::view::dense alpha, \ - matrix::view::dense a, \ - matrix::view::dense b, \ - matrix::view::dense beta, \ - matrix::view::dense c) - #define GKO_DECLARE_MULTIVECTOR_COPY_KERNEL(InValueType, OutValueType) \ void copy(std::shared_ptr exec, \ matrix::view::dense input, \ @@ -66,18 +52,6 @@ namespace kernels { matrix::view::dense x, \ matrix::view::dense y) -#define GKO_DECLARE_MULTIVECTOR_ADD_SCALED_DIAG_KERNEL(ValueType) \ - void add_scaled_diag(std::shared_ptr exec, \ - matrix::view::dense alpha, \ - const matrix::Diagonal* x, \ - matrix::view::dense y) - -#define GKO_DECLARE_MULTIVECTOR_SUB_SCALED_DIAG_KERNEL(ValueType) \ - void sub_scaled_diag(std::shared_ptr exec, \ - matrix::view::dense alpha, \ - const matrix::Diagonal* x, \ - matrix::view::dense y) - #define GKO_DECLARE_MULTIVECTOR_COMPUTE_DOT_DISPATCH_KERNEL(ValueType) \ void compute_dot_dispatch(std::shared_ptr exec, \ matrix::view::dense x, \ @@ -145,74 +119,6 @@ namespace kernels { void compute_sqrt(std::shared_ptr exec, \ matrix::view::dense data) -#define GKO_DECLARE_MULTIVECTOR_CONVERT_TO_COO_KERNEL(ValueType, IndexType) \ - void convert_to_coo(std::shared_ptr exec, \ - matrix::view::dense source, \ - const int64* row_ptrs, \ - matrix::view::coo other) - -#define GKO_DECLARE_MULTIVECTOR_CONVERT_TO_CSR_KERNEL(ValueType, IndexType) \ - void convert_to_csr(std::shared_ptr exec, \ - matrix::view::dense source, \ - matrix::Csr* other) - -#define GKO_DECLARE_MULTIVECTOR_CONVERT_TO_ELL_KERNEL(ValueType, IndexType) \ - void convert_to_ell(std::shared_ptr exec, \ - matrix::view::dense source, \ - matrix::view::ell other) - -#define GKO_DECLARE_MULTIVECTOR_CONVERT_TO_FBCSR_KERNEL(ValueType, IndexType) \ - void convert_to_fbcsr(std::shared_ptr exec, \ - matrix::view::dense source, \ - matrix::Fbcsr* other) - -#define GKO_DECLARE_MULTIVECTOR_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType) \ - void convert_to_hybrid(std::shared_ptr exec, \ - matrix::view::dense source, \ - const int64* coo_row_ptrs, \ - matrix::view::hybrid other) - -#define GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType) \ - void convert_to_sellp(std::shared_ptr exec, \ - matrix::view::dense source, \ - matrix::view::sellp other) - -#define GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SPARSITY_CSR_KERNEL(ValueType, \ - IndexType) \ - void convert_to_sparsity_csr( \ - std::shared_ptr exec, \ - matrix::view::dense source, \ - matrix::SparsityCsr* other) - -#define GKO_DECLARE_MULTIVECTOR_COMPUTE_MAX_NNZ_PER_ROW_KERNEL(ValueType) \ - void compute_max_nnz_per_row(std::shared_ptr exec, \ - matrix::view::dense source, \ - size_type& result) - -#define GKO_DECLARE_MULTIVECTOR_COMPUTE_SLICE_SETS_KERNEL(ValueType) \ - void compute_slice_sets(std::shared_ptr exec, \ - matrix::view::dense source, \ - size_type slice_size, size_type stride_factor, \ - size_type* slice_sets, size_type* slice_lengths) - -#define GKO_DECLARE_MULTIVECTOR_COUNT_NONZEROS_PER_ROW_KERNEL(ValueType, \ - IndexType) \ - void count_nonzeros_per_row(std::shared_ptr exec, \ - matrix::view::dense source, \ - IndexType* result) - -#define GKO_DECLARE_MULTIVECTOR_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL(ValueType, \ - IndexType) \ - void count_nonzero_blocks_per_row( \ - std::shared_ptr exec, \ - matrix::view::dense source, int block_size, \ - IndexType* result) - -#define GKO_DECLARE_MULTIVECTOR_COUNT_NONZEROS_PER_ROW_KERNEL_SIZE_T( \ - ValueType) \ - GKO_DECLARE_MULTIVECTOR_COUNT_NONZEROS_PER_ROW_KERNEL(ValueType, \ - ::gko::size_type) - #define GKO_DECLARE_MULTIVECTOR_TRANSPOSE_KERNEL(ValueType) \ void transpose(std::shared_ptr exec, \ matrix::view::dense orig, \ @@ -350,11 +256,6 @@ namespace kernels { matrix::view::dense orig, \ matrix::view::dense col_permuted) -#define GKO_DECLARE_MULTIVECTOR_EXTRACT_DIAGONAL_KERNEL(ValueType) \ - void extract_diagonal(std::shared_ptr exec, \ - matrix::view::dense orig, \ - matrix::Diagonal* diag) - #define GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL(ValueType) \ void inplace_absolute_dense(std::shared_ptr exec, \ matrix::view::dense source) @@ -380,19 +281,8 @@ namespace kernels { matrix::view::dense source, \ matrix::view::dense> result) -#define GKO_DECLARE_MULTIVECTOR_ADD_SCALED_IDENTITY_KERNEL(ValueType, \ - ScalarType) \ - void add_scaled_identity(std::shared_ptr exec, \ - matrix::view::dense alpha, \ - matrix::view::dense beta, \ - matrix::view::dense mtx) - #define GKO_DECLARE_ALL_AS_TEMPLATES \ - template \ - GKO_DECLARE_MULTIVECTOR_SIMPLE_APPLY_KERNEL(ValueType); \ - template \ - GKO_DECLARE_MULTIVECTOR_APPLY_KERNEL(ValueType); \ template \ GKO_DECLARE_MULTIVECTOR_COPY_KERNEL(InValueType, OutValueType); \ template \ @@ -406,10 +296,6 @@ namespace kernels { template \ GKO_DECLARE_MULTIVECTOR_SUB_SCALED_KERNEL(ValueType, ScalarType); \ template \ - GKO_DECLARE_MULTIVECTOR_ADD_SCALED_DIAG_KERNEL(ValueType); \ - template \ - GKO_DECLARE_MULTIVECTOR_SUB_SCALED_DIAG_KERNEL(ValueType); \ - template \ GKO_DECLARE_MULTIVECTOR_COMPUTE_DOT_KERNEL(ValueType); \ template \ GKO_DECLARE_MULTIVECTOR_COMPUTE_DOT_DISPATCH_KERNEL(ValueType); \ @@ -431,31 +317,6 @@ namespace kernels { GKO_DECLARE_MULTIVECTOR_COMPUTE_SQUARED_NORM2_KERNEL(ValueType); \ template \ GKO_DECLARE_MULTIVECTOR_COMPUTE_SQRT_KERNEL(ValueType); \ - template \ - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_COO_KERNEL(ValueType, IndexType); \ - template \ - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_CSR_KERNEL(ValueType, IndexType); \ - template \ - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_ELL_KERNEL(ValueType, IndexType); \ - template \ - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_FBCSR_KERNEL(ValueType, IndexType); \ - template \ - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_HYBRID_KERNEL(ValueType, IndexType); \ - template \ - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SELLP_KERNEL(ValueType, IndexType); \ - template \ - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SPARSITY_CSR_KERNEL(ValueType, \ - IndexType); \ - template \ - GKO_DECLARE_MULTIVECTOR_COMPUTE_MAX_NNZ_PER_ROW_KERNEL(ValueType); \ - template \ - GKO_DECLARE_MULTIVECTOR_COMPUTE_SLICE_SETS_KERNEL(ValueType); \ - template \ - GKO_DECLARE_MULTIVECTOR_COUNT_NONZEROS_PER_ROW_KERNEL(ValueType, \ - IndexType); \ - template \ - GKO_DECLARE_MULTIVECTOR_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL(ValueType, \ - IndexType); \ template \ GKO_DECLARE_MULTIVECTOR_TRANSPOSE_KERNEL(ValueType); \ template \ @@ -502,8 +363,6 @@ namespace kernels { GKO_DECLARE_MULTIVECTOR_INV_NONSYMM_SCALE_PERMUTE_KERNEL(ValueType, \ IndexType); \ template \ - GKO_DECLARE_MULTIVECTOR_EXTRACT_DIAGONAL_KERNEL(ValueType); \ - template \ GKO_DECLARE_INPLACE_ABSOLUTE_DENSE_KERNEL(ValueType); \ template \ GKO_DECLARE_OUTPLACE_ABSOLUTE_DENSE_KERNEL(ValueType); \ @@ -512,9 +371,7 @@ namespace kernels { template \ GKO_DECLARE_GET_REAL_KERNEL(ValueType); \ template \ - GKO_DECLARE_GET_IMAG_KERNEL(ValueType); \ - template \ - GKO_DECLARE_MULTIVECTOR_ADD_SCALED_IDENTITY_KERNEL(ValueType, ScalarType) + GKO_DECLARE_GET_IMAG_KERNEL(ValueType) GKO_DECLARE_FOR_ALL_EXECUTOR_NAMESPACES(multivector, diff --git a/core/matrix/sellp.cpp b/core/matrix/sellp.cpp index 7f50fcd9401..8377a0f09c2 100644 --- a/core/matrix/sellp.cpp +++ b/core/matrix/sellp.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "core/base/allocator.hpp" @@ -277,8 +278,7 @@ void Sellp::move_to( template -void Sellp::convert_to( - MultiVector* result) const +void Sellp::convert_to(Dense* result) const { auto exec = this->get_executor(); auto tmp_result = make_temporary_output_clone(exec, result); @@ -290,7 +290,7 @@ void Sellp::convert_to( template -void Sellp::move_to(MultiVector* result) +void Sellp::move_to(Dense* result) { this->convert_to(result); } diff --git a/core/matrix/sparsity_csr.cpp b/core/matrix/sparsity_csr.cpp index f8484c150b2..403848c3f4e 100644 --- a/core/matrix/sparsity_csr.cpp +++ b/core/matrix/sparsity_csr.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include "core/base/array_access.hpp" @@ -220,7 +221,7 @@ void SparsityCsr::move_to( template void SparsityCsr::convert_to( - MultiVector* result) const + Dense* result) const { auto exec = this->get_executor(); auto tmp_result = make_temporary_output_clone(exec, result); @@ -232,7 +233,7 @@ void SparsityCsr::convert_to( template -void SparsityCsr::move_to(MultiVector* result) +void SparsityCsr::move_to(Dense* result) { this->convert_to(result); } diff --git a/core/multigrid/pgm.cpp b/core/multigrid/pgm.cpp index 77327e81a52..8c3e1f11d2f 100644 --- a/core/multigrid/pgm.cpp +++ b/core/multigrid/pgm.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/core/preconditioner/jacobi.cpp b/core/preconditioner/jacobi.cpp index 4d23ddabfdf..68a4d2d40d8 100644 --- a/core/preconditioner/jacobi.cpp +++ b/core/preconditioner/jacobi.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "core/base/extended_float.hpp" @@ -198,10 +199,10 @@ void Jacobi::apply_impl(const LinOp* alpha, template void Jacobi::convert_to( - matrix::MultiVector* result) const + matrix::Dense* result) const { auto exec = this->get_executor(); - auto tmp = matrix::MultiVector::create(exec, this->get_size()); + auto tmp = matrix::Dense::create(exec, this->get_size()); if (parameters_.max_block_size == 1) { exec->run(jacobi::make_scalar_convert_to_dense(blocks_, tmp->get_device_view())); @@ -216,8 +217,7 @@ void Jacobi::convert_to( template -void Jacobi::move_to( - matrix::MultiVector* result) +void Jacobi::move_to(matrix::Dense* result) { this->convert_to(result); // no special optimization possible here } diff --git a/core/reorder/amd.cpp b/core/reorder/amd.cpp index c21f6e79187..6f699832920 100644 --- a/core/reorder/amd.cpp +++ b/core/reorder/amd.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include diff --git a/core/reorder/rcm.cpp b/core/reorder/rcm.cpp index c434e620b91..5c9bf5a11ee 100644 --- a/core/reorder/rcm.cpp +++ b/core/reorder/rcm.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/core/solver/cb_gmres.cpp b/core/solver/cb_gmres.cpp index 28f754a6a4a..2f2cd0bf823 100644 --- a/core/solver/cb_gmres.cpp +++ b/core/solver/cb_gmres.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "core/base/extended_float.hpp" diff --git a/core/solver/ir.cpp b/core/solver/ir.cpp index 31c563603a6..7bfb42aaeaf 100644 --- a/core/solver/ir.cpp +++ b/core/solver/ir.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include diff --git a/core/solver/multigrid.cpp b/core/solver/multigrid.cpp index 48af1f24336..f1c5dbe114a 100644 --- a/core/solver/multigrid.cpp +++ b/core/solver/multigrid.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/core/stop/residual_norm.cpp b/core/stop/residual_norm.cpp index a1e811d229e..a46b1ba40dd 100644 --- a/core/stop/residual_norm.cpp +++ b/core/stop/residual_norm.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include "core/base/dispatch_helper.hpp" diff --git a/core/test/base/batch_multi_vector.cpp b/core/test/base/batch_multi_vector.cpp index e28f53d8f57..d105135ac19 100644 --- a/core/test/base/batch_multi_vector.cpp +++ b/core/test/base/batch_multi_vector.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "core/base/batch_utilities.hpp" diff --git a/core/test/base/block_operator.cpp b/core/test/base/block_operator.cpp index 0dbabf15134..41613bf0af7 100644 --- a/core/test/base/block_operator.cpp +++ b/core/test/base/block_operator.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include "core/test/utils.hpp" @@ -185,7 +186,7 @@ TEST_F(BlockOperator, ThrowsOnOutOfBoundsBlockAccess) TEST_F(BlockOperator, CanBeCopied) { - using Mtx = gko::matrix::MultiVector<>; + using Mtx = gko::matrix::Dense<>; auto bop = gko::BlockOperator::create( exec, {{gko::initialize({{1, 2}, {2, 1}}, exec), nullptr}, {nullptr, gko::initialize({{3, 4}, {4, 3}}, exec)}}); @@ -211,7 +212,7 @@ TEST_F(BlockOperator, CanBeCopied) TEST_F(BlockOperator, CanBeMoved) { - using Mtx = gko::matrix::MultiVector<>; + using Mtx = gko::matrix::Dense<>; auto bop = gko::BlockOperator::create( exec, {{gko::initialize({{1, 2}, {2, 1}}, exec), nullptr}, {nullptr, gko::initialize({{3, 4}, {4, 3}}, exec)}}); @@ -240,13 +241,14 @@ TEST_F(BlockOperator, CanBeMoved) TEST_F(BlockOperator, CanApply) { using vtype = double; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; + using Vec = gko::matrix::MultiVector; auto bop = gko::BlockOperator::create( exec, {{gko::initialize({{1, 2}, {2, 1}}, exec), gko::initialize({{5, 6}, {6, 5}}, exec)}, {nullptr, gko::initialize({{3, 4}, {4, 3}}, exec)}}); - auto x = gko::initialize({{1, 10}, {2, 20}, {3, 30}, {4, 40}}, exec); - auto y = Mtx::create_with_config_of(x); + auto x = gko::initialize({{1, 10}, {2, 20}, {3, 30}, {4, 40}}, exec); + auto y = Vec::create_with_config_of(x); bop->apply(x, y); @@ -259,15 +261,16 @@ TEST_F(BlockOperator, CanApply) TEST_F(BlockOperator, CanAdvancedApply) { using vtype = double; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; + using Vec = gko::matrix::MultiVector; auto bop = gko::BlockOperator::create( exec, {{gko::initialize({{1, 2}, {2, 1}}, exec), gko::initialize({{5, 6}, {6, 5}}, exec)}, {nullptr, gko::initialize({{3, 4}, {4, 3}}, exec)}}); - auto x = gko::initialize({1, 2, 3, 4}, exec); - auto y = gko::initialize({-4, -3, -2, -1}, exec); - auto alpha = gko::initialize({0.5}, exec); - auto beta = gko::initialize({-1}, exec); + auto x = gko::initialize({1, 2, 3, 4}, exec); + auto y = gko::initialize({-4, -3, -2, -1}, exec); + auto alpha = gko::initialize({0.5}, exec); + auto beta = gko::initialize({-1}, exec); bop->apply(alpha, x, beta, y); @@ -287,7 +290,7 @@ TEST_F(BlockOperator, CanApplyAndAdvancedApplyLarge) block_num_rows * local_num_rows, block_num_cols * local_num_cols, std::uniform_real_distribution(-1, 1), engine, exec); auto get_submatrix = [&](auto i, auto j) { - return dense->create_submatrix( + return dense->create_subview( {i * local_num_rows, (i + 1) * local_num_rows}, {j * local_num_cols, (j + 1) * local_num_cols}); }; @@ -300,7 +303,7 @@ TEST_F(BlockOperator, CanApplyAndAdvancedApplyLarge) } } auto bop = gko::BlockOperator::create(exec, blocks); - auto x = gko::test::generate_random_dense_matrix( + auto x = gko::test::generate_random_multi_vector( block_num_cols * local_num_cols, 3, std::uniform_real_distribution(-1, 1), engine, exec); auto y = Mtx::create(exec, gko::dim<2>{block_num_rows * local_num_rows, 3}); diff --git a/core/test/base/mtx_io.cpp b/core/test/base/mtx_io.cpp index ccdbe63a5a7..dc8e4919d74 100644 --- a/core/test/base/mtx_io.cpp +++ b/core/test/base/mtx_io.cpp @@ -1295,7 +1295,7 @@ TYPED_TEST(MultiVectorTest, WritesToStreamFromLinOpPtrOnMultiVector) "2.0\n" "0.0\n"); std::unique_ptr lin_op = - gko::read>( + gko::read>( iss, gko::ReferenceExecutor::create()); std::ostringstream oss{}; diff --git a/core/test/base/perturbation.cpp b/core/test/base/perturbation.cpp index 3c2ec89871f..30f174e806d 100644 --- a/core/test/base/perturbation.cpp +++ b/core/test/base/perturbation.cpp @@ -7,6 +7,7 @@ #include #include +#include namespace { @@ -62,14 +63,14 @@ class Perturbation : public ::testing::Test { projector{std::make_shared(exec, gko::dim<2>{1, 2})}, trans_basis{std::make_shared( exec, gko::dim<2>{3, 1})}, - scalar{std::make_shared(exec, gko::dim<2>{1, 1})} + scalar{gko::matrix::MultiVector<>::create(exec, gko::dim<2>{1, 1})} {} std::shared_ptr exec; std::shared_ptr basis; std::shared_ptr projector; std::shared_ptr trans_basis; - std::shared_ptr scalar; + std::shared_ptr> scalar; }; diff --git a/core/test/config/config.cpp b/core/test/config/config.cpp index be5b281b8ce..f88f681dc62 100644 --- a/core/test/config/config.cpp +++ b/core/test/config/config.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -28,7 +29,7 @@ using namespace gko::config; class Config : public ::testing::Test { protected: using value_type = double; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; Config() : exec(gko::ReferenceExecutor::create()), mtx(gko::initialize( diff --git a/core/test/config/preconditioner.cpp b/core/test/config/preconditioner.cpp index 6181a76d6e0..4fd28bf1486 100644 --- a/core/test/config/preconditioner.cpp +++ b/core/test/config/preconditioner.cpp @@ -501,7 +501,7 @@ class Preconditioner : public ::testing::Test { l_solver(DummyIr::build().on(exec)), u_solver(DummyIr::build().on(exec)), factorization(DummyIr::build().on(exec)), - linop(gko::matrix::MultiVector<>::create(exec)), + linop(gko::matrix::Dense<>::create(exec)), coarse_level(DummyMgLevel::build().on(exec)), reg() { diff --git a/core/test/config/registry.cpp b/core/test/config/registry.cpp index 743aaaa9b07..4494c296eb4 100644 --- a/core/test/config/registry.cpp +++ b/core/test/config/registry.cpp @@ -23,7 +23,7 @@ using namespace gko::config; class Registry : public ::testing::Test { protected: - using Matrix = gko::matrix::MultiVector; + using Matrix = gko::matrix::Dense; using Solver = gko::solver::Cg; using Stop = gko::stop::Iteration; @@ -173,7 +173,7 @@ TEST_F(Registry, ThrowWithWrongType) reg.emplace("stop_factory", stop_factory); ASSERT_THROW( - detail::registry_accessor::get_data>( + detail::registry_accessor::get_data>( reg, "matrix"), gko::NotSupported); ASSERT_THROW( diff --git a/core/test/config/solver.cpp b/core/test/config/solver.cpp index 58da9e6d307..ab161edbb21 100644 --- a/core/test/config/solver.cpp +++ b/core/test/config/solver.cpp @@ -500,7 +500,7 @@ template class Solver : public ::testing::Test { protected: using Config = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; Solver() : exec(gko::ReferenceExecutor::create()), mtx(Mtx::create(exec)), diff --git a/core/test/factorization/elimination_forest.cpp b/core/test/factorization/elimination_forest.cpp index 9bbc9536c93..864eeed1cde 100644 --- a/core/test/factorization/elimination_forest.cpp +++ b/core/test/factorization/elimination_forest.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -10,6 +10,7 @@ #include #include +#include #include "core/test/utils.hpp" #include "matrices/config.hpp" diff --git a/core/test/log/convergence.cpp b/core/test/log/convergence.cpp index 599706bac8a..4f0f4cd79c0 100644 --- a/core/test/log/convergence.cpp +++ b/core/test/log/convergence.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -22,6 +23,7 @@ class Convergence : public ::testing::Test { using MultiVector = gko::matrix::MultiVector; using AbsoluteMultiVector = gko::matrix::MultiVector>; + using Dense = gko::matrix::Dense; Convergence() { @@ -31,8 +33,8 @@ class Convergence : public ::testing::Test { gko::solver::Ir::build() .with_criteria(gko::stop::Iteration::build().with_max_iters(1u)) .on(exec) - ->generate(gko::initialize(I>{{1, 2}, {0, 3}}, - exec)); + ->generate( + gko::initialize(I>{{1, 2}, {0, 3}}, exec)); } std::shared_ptr exec = diff --git a/core/test/log/logger.cpp b/core/test/log/logger.cpp index b065db66768..bf408507200 100644 --- a/core/test/log/logger.cpp +++ b/core/test/log/logger.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause diff --git a/core/test/log/profiler_hook.cpp b/core/test/log/profiler_hook.cpp index d35d46424d5..7eb4a27f7e3 100644 --- a/core/test/log/profiler_hook.cpp +++ b/core/test/log/profiler_hook.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include diff --git a/core/test/log/record.cpp b/core/test/log/record.cpp index 61316b0aa2d..cb2b9180db9 100644 --- a/core/test/log/record.cpp +++ b/core/test/log/record.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -312,10 +313,11 @@ TEST(Record, CatchesPolymorphicObjectDeleted) TEST(Record, CatchesLinOpApplyStarted) { using MultiVector = gko::matrix::MultiVector<>; + using Dense = gko::matrix::Dense<>; auto exec = gko::ReferenceExecutor::create(); auto logger = gko::log::Record::create(gko::log::Logger::linop_apply_started_mask); - auto A = gko::initialize({1.1}, exec); + auto A = gko::initialize({1.1}, exec); auto b = gko::initialize({-2.2}, exec); auto x = gko::initialize({3.3}, exec); @@ -323,7 +325,7 @@ TEST(Record, CatchesLinOpApplyStarted) x.get()); auto& data = logger->get().linop_apply_started.back(); - GKO_ASSERT_MTX_NEAR(gko::as(data->A.get()), A, 0); + GKO_ASSERT_MTX_NEAR(gko::as(data->A.get()), A, 0); ASSERT_EQ(data->alpha, nullptr); GKO_ASSERT_MTX_NEAR(gko::as(data->b.get()), b, 0); ASSERT_EQ(data->beta, nullptr); @@ -334,10 +336,11 @@ TEST(Record, CatchesLinOpApplyStarted) TEST(Record, CatchesLinOpApplyCompleted) { using MultiVector = gko::matrix::MultiVector<>; + using Dense = gko::matrix::Dense<>; auto exec = gko::ReferenceExecutor::create(); auto logger = gko::log::Record::create(gko::log::Logger::linop_apply_completed_mask); - auto A = gko::initialize({1.1}, exec); + auto A = gko::initialize({1.1}, exec); auto b = gko::initialize({-2.2}, exec); auto x = gko::initialize({3.3}, exec); @@ -345,7 +348,7 @@ TEST(Record, CatchesLinOpApplyCompleted) x.get()); auto& data = logger->get().linop_apply_completed.back(); - GKO_ASSERT_MTX_NEAR(gko::as(data->A.get()), A, 0); + GKO_ASSERT_MTX_NEAR(gko::as(data->A.get()), A, 0); ASSERT_EQ(data->alpha, nullptr); GKO_ASSERT_MTX_NEAR(gko::as(data->b.get()), b, 0); ASSERT_EQ(data->beta, nullptr); @@ -356,10 +359,11 @@ TEST(Record, CatchesLinOpApplyCompleted) TEST(Record, CatchesLinOpAdvancedApplyStarted) { using MultiVector = gko::matrix::MultiVector<>; + using Dense = gko::matrix::Dense<>; auto exec = gko::ReferenceExecutor::create(); auto logger = gko::log::Record::create( gko::log::Logger::linop_advanced_apply_started_mask); - auto A = gko::initialize({1.1}, exec); + auto A = gko::initialize({1.1}, exec); auto alpha = gko::initialize({-4.4}, exec); auto b = gko::initialize({-2.2}, exec); auto beta = gko::initialize({-5.5}, exec); @@ -369,7 +373,7 @@ TEST(Record, CatchesLinOpAdvancedApplyStarted) A.get(), alpha.get(), b.get(), beta.get(), x.get()); auto& data = logger->get().linop_advanced_apply_started.back(); - GKO_ASSERT_MTX_NEAR(gko::as(data->A.get()), A, 0); + GKO_ASSERT_MTX_NEAR(gko::as(data->A.get()), A, 0); GKO_ASSERT_MTX_NEAR(gko::as(data->alpha.get()), alpha, 0); GKO_ASSERT_MTX_NEAR(gko::as(data->b.get()), b, 0); GKO_ASSERT_MTX_NEAR(gko::as(data->beta.get()), beta, 0); @@ -380,10 +384,11 @@ TEST(Record, CatchesLinOpAdvancedApplyStarted) TEST(Record, CatchesLinOpAdvancedApplyCompleted) { using MultiVector = gko::matrix::MultiVector<>; + using Dense = gko::matrix::Dense<>; auto exec = gko::ReferenceExecutor::create(); auto logger = gko::log::Record::create( gko::log::Logger::linop_advanced_apply_completed_mask); - auto A = gko::initialize({1.1}, exec); + auto A = gko::initialize({1.1}, exec); auto alpha = gko::initialize({-4.4}, exec); auto b = gko::initialize({-2.2}, exec); auto beta = gko::initialize({-5.5}, exec); @@ -393,7 +398,7 @@ TEST(Record, CatchesLinOpAdvancedApplyCompleted) A.get(), alpha.get(), b.get(), beta.get(), x.get()); auto& data = logger->get().linop_advanced_apply_completed.back(); - GKO_ASSERT_MTX_NEAR(gko::as(data->A.get()), A, 0); + GKO_ASSERT_MTX_NEAR(gko::as(data->A.get()), A, 0); GKO_ASSERT_MTX_NEAR(gko::as(data->alpha.get()), alpha, 0); GKO_ASSERT_MTX_NEAR(gko::as(data->b.get()), b, 0); GKO_ASSERT_MTX_NEAR(gko::as(data->beta.get()), beta, 0); diff --git a/core/test/log/solver_progress.cpp b/core/test/log/solver_progress.cpp index 6754cf9ee84..d3984bcc898 100644 --- a/core/test/log/solver_progress.cpp +++ b/core/test/log/solver_progress.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -20,11 +21,12 @@ template class SolverProgress : public ::testing::Test { public: using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Cg = gko::solver::Cg; SolverProgress() : ref{gko::ReferenceExecutor::create()} { - mtx = gko::initialize({T{1.0}}, ref); + mtx = gko::initialize({T{1.0}}, ref); in = gko::initialize({T{2.0}}, ref); out = gko::initialize({T{4.0}}, ref); zero = gko::initialize({T{0.0}}, ref); @@ -53,15 +55,15 @@ class SolverProgress : public ::testing::Test { return; } // check that the files have the correct contents - auto mtx = gko::read(stream_mtx, ref); - auto mtx_bin = gko::read_binary(stream_bin, ref); + auto mtx = gko::read(stream_mtx, ref); + auto mtx_bin = gko::read_binary(stream_bin, ref); cleanup(); GKO_ASSERT_MTX_NEAR(mtx, ref_mtx, 0.0); GKO_ASSERT_MTX_NEAR(mtx_bin, ref_mtx, 0.0); } std::shared_ptr ref; - std::shared_ptr mtx; + std::shared_ptr mtx; std::shared_ptr in; std::unique_ptr out; std::unique_ptr zero; @@ -139,6 +141,7 @@ TYPED_TEST(SolverProgress, StorageWorks) { using T = TypeParam; using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; auto orig_out = this->out->clone(); auto init_residual = gko::initialize({T{-2.0}}, this->ref); std::vector> files{ @@ -167,8 +170,9 @@ TYPED_TEST(SolverProgress, StorageWorks) {"solver_progress_test_1_solution", this->in.get()}, {"solver_progress_test_1_z", nullptr}, {"solver_progress_test_initial_guess", orig_out.get()}, - {"solver_progress_test_rhs", this->in.get()}, - {"solver_progress_test_system_matrix", this->mtx.get()}}; + {"solver_progress_test_rhs", this->in.get()}}; + std::pair mtx_file{ + "solver_progress_test_system_matrix", this->mtx.get()}; // run the solve once so the internal vectors are initialized before // attaching the logger this->solver->apply(this->in, this->out->clone()); @@ -182,4 +186,5 @@ TYPED_TEST(SolverProgress, StorageWorks) for (auto pair : files) { this->assert_file_equals(pair.first, pair.second); } + this->assert_file_equals(mtx_file.first, mtx_file.second); } diff --git a/core/test/log/stream.cpp b/core/test/log/stream.cpp index 5e561150e53..33e2907fd3a 100644 --- a/core/test/log/stream.cpp +++ b/core/test/log/stream.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -348,11 +349,12 @@ TYPED_TEST(Stream, CatchesPolymorphicObjectDeleted) TYPED_TEST(Stream, CatchesLinOpApplyStarted) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( gko::log::Logger::linop_apply_started_mask, out); - auto A = MultiVector::create(exec); + auto A = Dense::create(exec); auto b = MultiVector::create(exec); auto x = MultiVector::create(exec); std::stringstream ptrstream_A; @@ -376,11 +378,12 @@ TYPED_TEST(Stream, CatchesLinOpApplyStarted) TYPED_TEST(Stream, CatchesLinOpApplyStartedWithVerbose) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( gko::log::Logger::linop_apply_started_mask, out, true); - auto A = gko::initialize({1.5}, exec); + auto A = gko::initialize({1.5}, exec); auto b = gko::initialize({-2.25}, exec); auto x = gko::initialize({3.125}, exec); @@ -397,11 +400,12 @@ TYPED_TEST(Stream, CatchesLinOpApplyStartedWithVerbose) TYPED_TEST(Stream, CatchesLinOpApplyCompleted) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( gko::log::Logger::linop_apply_completed_mask, out); - auto A = MultiVector::create(exec); + auto A = Dense::create(exec); auto b = MultiVector::create(exec); auto x = MultiVector::create(exec); std::stringstream ptrstream_A; @@ -425,11 +429,12 @@ TYPED_TEST(Stream, CatchesLinOpApplyCompleted) TYPED_TEST(Stream, CatchesLinOpApplyCompletedWithVerbose) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( gko::log::Logger::linop_apply_completed_mask, out, true); - auto A = gko::initialize({1.5}, exec); + auto A = gko::initialize({1.5}, exec); auto b = gko::initialize({-2.25}, exec); auto x = gko::initialize({3.125}, exec); @@ -446,11 +451,12 @@ TYPED_TEST(Stream, CatchesLinOpApplyCompletedWithVerbose) TYPED_TEST(Stream, CatchesLinOpAdvancedApplyStarted) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( gko::log::Logger::linop_advanced_apply_started_mask, out); - auto A = MultiVector::create(exec); + auto A = Dense::create(exec); auto alpha = MultiVector::create(exec); auto b = MultiVector::create(exec); auto beta = MultiVector::create(exec); @@ -482,11 +488,12 @@ TYPED_TEST(Stream, CatchesLinOpAdvancedApplyStarted) TYPED_TEST(Stream, CatchesLinOpAdvancedApplyStartedWithVerbose) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( gko::log::Logger::linop_advanced_apply_started_mask, out, true); - auto A = gko::initialize({1.5}, exec); + auto A = gko::initialize({1.5}, exec); auto alpha = gko::initialize({-4.75}, exec); auto b = gko::initialize({-2.25}, exec); auto beta = gko::initialize({-5.5}, exec); @@ -507,11 +514,12 @@ TYPED_TEST(Stream, CatchesLinOpAdvancedApplyStartedWithVerbose) TYPED_TEST(Stream, CatchesLinOpAdvancedApplyCompleted) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( gko::log::Logger::linop_advanced_apply_completed_mask, out); - auto A = MultiVector::create(exec); + auto A = Dense::create(exec); auto alpha = MultiVector::create(exec); auto b = MultiVector::create(exec); auto beta = MultiVector::create(exec); @@ -543,11 +551,12 @@ TYPED_TEST(Stream, CatchesLinOpAdvancedApplyCompleted) TYPED_TEST(Stream, CatchesLinOpAdvancedApplyCompletedWithVerbose) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( gko::log::Logger::linop_advanced_apply_completed_mask, out, true); - auto A = gko::initialize({1.5}, exec); + auto A = gko::initialize({1.5}, exec); auto alpha = gko::initialize({-4.75}, exec); auto b = gko::initialize({-2.25}, exec); auto beta = gko::initialize({-5.5}, exec); @@ -575,8 +584,7 @@ TYPED_TEST(Stream, CatchesLinopFactoryGenerateStarted) gko::solver::Bicgstab::build() .with_criteria(gko::stop::Iteration::build().with_max_iters(3u)) .on(exec); - auto input = - factory->generate(gko::matrix::MultiVector::create(exec)); + auto input = factory->generate(gko::matrix::Dense::create(exec)); std::stringstream ptrstream_factory; ptrstream_factory << factory.get(); std::stringstream ptrstream_input; @@ -602,10 +610,9 @@ TYPED_TEST(Stream, CatchesLinopFactoryGenerateCompleted) gko::solver::Bicgstab::build() .with_criteria(gko::stop::Iteration::build().with_max_iters(3u)) .on(exec); - auto input = - factory->generate(gko::matrix::MultiVector::create(exec)); + auto input = factory->generate(gko::matrix::Dense::create(exec)); auto output = - factory->generate(gko::matrix::MultiVector::create(exec)); + factory->generate(gko::matrix::Dense::create(exec)); std::stringstream ptrstream_factory; ptrstream_factory << factory.get(); std::stringstream ptrstream_input; @@ -713,11 +720,12 @@ TYPED_TEST(Stream, CatchesCriterionCheckCompletedWithVerbose) TYPED_TEST(Stream, CatchesIterationsWithoutStoppingStatus) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( gko::log::Logger::iteration_complete_mask, out); - auto solver = MultiVector::create(exec); + auto solver = Dense::create(exec); auto right_hand_side = MultiVector::create(exec); auto residual = MultiVector::create(exec); auto solution = MultiVector::create(exec); @@ -743,11 +751,12 @@ TYPED_TEST(Stream, CatchesIterationsWithoutStoppingStatus) TYPED_TEST(Stream, CatchesIterationsWithStoppingStatus) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( gko::log::Logger::iteration_complete_mask, out); - auto solver = MultiVector::create(exec); + auto solver = Dense::create(exec); auto right_hand_side = MultiVector::create(exec); auto residual = MultiVector::create(exec); auto solution = MultiVector::create(exec); @@ -775,6 +784,7 @@ TYPED_TEST(Stream, CatchesIterationsWithStoppingStatus) TYPED_TEST(Stream, CatchesIterationsWithVerbose) { using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto exec = gko::ReferenceExecutor::create(); std::stringstream out; auto logger = gko::log::Stream::create( @@ -784,7 +794,7 @@ TYPED_TEST(Stream, CatchesIterationsWithVerbose) gko::solver::Bicgstab::build() .with_criteria(gko::stop::Iteration::build().with_max_iters(3u)) .on(exec); - auto solver = factory->generate(gko::initialize({1.25}, exec)); + auto solver = factory->generate(gko::initialize({1.25}, exec)); auto right_hand_side = gko::initialize({-5.5}, exec); auto residual = gko::initialize({-4.5}, exec); auto solution = gko::initialize({-2.25}, exec); diff --git a/core/test/matrix/CMakeLists.txt b/core/test/matrix/CMakeLists.txt index 58c6d0605bb..020b263f2e2 100644 --- a/core/test/matrix/CMakeLists.txt +++ b/core/test/matrix/CMakeLists.txt @@ -6,6 +6,7 @@ ginkgo_create_test(coo) ginkgo_create_test(coo_builder) ginkgo_create_test(csr) ginkgo_create_test(csr_builder) +ginkgo_create_test(dense) ginkgo_create_test(device_views) ginkgo_create_test(diagonal) ginkgo_create_test(ell) diff --git a/core/test/matrix/batch_csr.cpp b/core/test/matrix/batch_csr.cpp index 6f296a0931c..e380593f77b 100644 --- a/core/test/matrix/batch_csr.cpp +++ b/core/test/matrix/batch_csr.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include "core/base/batch_utilities.hpp" #include "core/test/utils.hpp" diff --git a/core/test/matrix/batch_dense.cpp b/core/test/matrix/batch_dense.cpp index efd6554ddcc..8c8c082f3a3 100644 --- a/core/test/matrix/batch_dense.cpp +++ b/core/test/matrix/batch_dense.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "core/base/batch_utilities.hpp" diff --git a/core/test/matrix/batch_ell.cpp b/core/test/matrix/batch_ell.cpp index 0decb001db4..7d9c9ec9ca0 100644 --- a/core/test/matrix/batch_ell.cpp +++ b/core/test/matrix/batch_ell.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "core/base/batch_utilities.hpp" diff --git a/core/test/matrix/dense.cpp b/core/test/matrix/dense.cpp new file mode 100644 index 00000000000..8e04d43e0a4 --- /dev/null +++ b/core/test/matrix/dense.cpp @@ -0,0 +1,313 @@ +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + +#include +#include +#include + +#include "core/test/utils.hpp" + + +namespace { + + +template +class Dense : public ::testing::Test { +protected: + using value_type = T; + Dense() + : exec(gko::ReferenceExecutor::create()), + mtx(gko::initialize>( + 4, {{1.0, 2.0, 3.0}, {1.5, 2.5, 3.5}}, exec)) + {} + + + static void assert_equal_to_original_mtx( + gko::ptr_param> m) + { + ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(m->get_num_stored_elements(), 2 * m->get_stride()); + EXPECT_EQ(m->at(0, 0), value_type{1.0}); + EXPECT_EQ(m->at(0, 1), value_type{2.0}); + EXPECT_EQ(m->at(0, 2), value_type{3.0}); + EXPECT_EQ(m->at(1, 0), value_type{1.5}); + EXPECT_EQ(m->at(1, 1), value_type{2.5}); + ASSERT_EQ(m->at(1, 2), value_type{3.5}); + } + + static void assert_empty(gko::ptr_param> m) + { + ASSERT_EQ(m->get_precision(), gko::type_to_precision); + ASSERT_EQ(m->get_size(), gko::dim<2>(0, 0)); + ASSERT_EQ(m->get_num_stored_elements(), 0); + } + + std::shared_ptr exec; + std::unique_ptr> mtx; +}; + +TYPED_TEST_SUITE(Dense, gko::test::ValueTypes, TypenameNameGenerator); + + +TYPED_TEST(Dense, CanBeEmpty) +{ + auto empty = gko::matrix::Dense::create(this->exec); + this->assert_empty(empty.get()); +} + + +TYPED_TEST(Dense, ReturnsNullValuesArrayWhenEmpty) +{ + auto empty = gko::matrix::Dense::create(this->exec); + ASSERT_EQ(empty->get_const_values(), nullptr); +} + + +TYPED_TEST(Dense, CanBeConstructedWithSize) +{ + auto m = + gko::matrix::Dense::create(this->exec, gko::dim<2>{2, 3}); + + ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3)); + EXPECT_EQ(m->get_stride(), 3); + ASSERT_EQ(m->get_num_stored_elements(), 6); +} + + +TYPED_TEST(Dense, CanBeConstructedWithSizeAndStride) +{ + auto m = + gko::matrix::Dense::create(this->exec, gko::dim<2>{2, 3}, 4); + + ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3)); + EXPECT_EQ(m->get_stride(), 4); + ASSERT_EQ(m->get_num_stored_elements(), 8); +} + + +TYPED_TEST(Dense, CanBeConstructedFromExistingData) +{ + using value_type = typename TestFixture::value_type; + // clang-format off + value_type data[] = { + 1.0, 2.0, -1.0, + 3.0, 4.0, -1.0, + 5.0, 6.0, -1.0}; + // clang-format on + + auto m = gko::matrix::Dense::create( + this->exec, gko::dim<2>{3, 2}, + gko::make_array_view(this->exec, 9, data), 3); + + ASSERT_EQ(m->get_const_values(), data); + ASSERT_EQ(m->at(2, 1), value_type{6.0}); +} + + +TYPED_TEST(Dense, CanBeConstructedFromExistingConstData) +{ + using value_type = typename TestFixture::value_type; + // clang-format off + const value_type data[] = { + 1.0, 2.0, -1.0, + 3.0, 4.0, -1.0, + 5.0, 6.0, -1.0}; + // clang-format on + + auto m = gko::matrix::Dense::create_const( + this->exec, gko::dim<2>{3, 2}, + gko::array::const_view(this->exec, 9, data), 3); + + ASSERT_EQ(m->get_const_values(), data); + ASSERT_EQ(m->at(2, 1), value_type{6.0}); +} + + +TYPED_TEST(Dense, KnowsItsSizeAndValues) +{ + this->assert_equal_to_original_mtx(this->mtx); + ASSERT_EQ(this->mtx->get_stride(), 4); +} + + +TYPED_TEST(Dense, CanBeListConstructed) +{ + using value_type = typename TestFixture::value_type; + auto m = + gko::initialize>({1.0, 2.0}, this->exec); + + ASSERT_EQ(m->get_size(), gko::dim<2>(2, 1)); + ASSERT_EQ(m->get_num_stored_elements(), 2); + EXPECT_EQ(m->at(0, 0), value_type{1}); + EXPECT_EQ(m->at(1, 0), value_type{2}); +} + + +TYPED_TEST(Dense, CanBeListConstructedWithstride) +{ + using value_type = typename TestFixture::value_type; + auto m = gko::initialize>(2, {1.0, 2.0}, + this->exec); + ASSERT_EQ(m->get_size(), gko::dim<2>(2, 1)); + ASSERT_EQ(m->get_num_stored_elements(), 4); + EXPECT_EQ(m->at(0, 0), value_type{1.0}); + EXPECT_EQ(m->at(1, 0), value_type{2.0}); +} + + +TYPED_TEST(Dense, CanBeDoubleListConstructed) +{ + using value_type = typename TestFixture::value_type; + using T = value_type; + auto m = gko::initialize>( + {I{1.0, 2.0}, I{3.0, 4.0}, I{5.0, 6.0}}, this->exec); + + ASSERT_EQ(m->get_size(), gko::dim<2>(3, 2)); + ASSERT_EQ(m->get_num_stored_elements(), 6); + EXPECT_EQ(m->at(0, 0), value_type{1.0}); + EXPECT_EQ(m->at(0, 1), value_type{2.0}); + EXPECT_EQ(m->at(1, 0), value_type{3.0}); + ASSERT_EQ(m->at(1, 1), value_type{4.0}); + EXPECT_EQ(m->at(2, 0), value_type{5.0}); +} + + +TYPED_TEST(Dense, CanBeDoubleListConstructedWithstride) +{ + using value_type = typename TestFixture::value_type; + using T = value_type; + auto m = gko::initialize>( + 4, {I{1.0, 2.0}, I{3.0, 4.0}, I{5.0, 6.0}}, this->exec); + + ASSERT_EQ(m->get_size(), gko::dim<2>(3, 2)); + ASSERT_EQ(m->get_num_stored_elements(), 12); + EXPECT_EQ(m->at(0, 0), value_type{1.0}); + EXPECT_EQ(m->at(0, 1), value_type{2.0}); + EXPECT_EQ(m->at(1, 0), value_type{3.0}); + ASSERT_EQ(m->at(1, 1), value_type{4.0}); + EXPECT_EQ(m->at(2, 0), value_type{5.0}); +} + + +TYPED_TEST(Dense, CanBeCopied) +{ + auto mtx_copy = gko::matrix::Dense::create(this->exec); + mtx_copy->copy_from(this->mtx); + this->assert_equal_to_original_mtx(this->mtx); + this->mtx->at(0, 0) = 7; + this->assert_equal_to_original_mtx(mtx_copy); + ASSERT_EQ(this->mtx->get_stride(), 4); + ASSERT_EQ(mtx_copy->get_stride(), 3); + ASSERT_EQ(mtx_copy->get_precision(), this->mtx->get_precision()); +} + + +TYPED_TEST(Dense, CanBeMoved) +{ + auto mtx_copy = gko::matrix::Dense::create(this->exec); + mtx_copy->move_from(this->mtx); + this->assert_equal_to_original_mtx(mtx_copy); + ASSERT_EQ(mtx_copy->get_stride(), 4); + ASSERT_EQ(mtx_copy->get_precision(), this->mtx->get_precision()); +} + + +TYPED_TEST(Dense, CanBeCloned) +{ + auto mtx_clone = this->mtx->clone(); + this->assert_equal_to_original_mtx(mtx_clone); + ASSERT_EQ(mtx_clone->get_stride(), 3); + ASSERT_EQ(mtx_clone->get_precision(), this->mtx->get_precision()); +} + + +TYPED_TEST(Dense, CanBeReadFromMatrixData) +{ + using value_type = typename TestFixture::value_type; + auto m = gko::matrix::Dense::create(this->exec); + m->read(gko::matrix_data{{2, 3}, + {{0, 0, 1.0}, + {0, 1, 3.0}, + {0, 2, 2.0}, + {1, 0, 0.0}, + {1, 1, 5.0}, + {1, 2, 0.0}}}); + + ASSERT_EQ(m->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(m->get_num_stored_elements(), 6); + EXPECT_EQ(m->at(0, 0), value_type{1.0}); + EXPECT_EQ(m->at(1, 0), value_type{0.0}); + EXPECT_EQ(m->at(0, 1), value_type{3.0}); + EXPECT_EQ(m->at(1, 1), value_type{5.0}); + EXPECT_EQ(m->at(0, 2), value_type{2.0}); + ASSERT_EQ(m->at(1, 2), value_type{0.0}); +} + + +TYPED_TEST(Dense, GeneratesCorrectMatrixData) +{ + using value_type = typename TestFixture::value_type; + using tpl = typename gko::matrix_data::nonzero_type; + gko::matrix_data data; + + this->mtx->write(data); + + ASSERT_EQ(data.size, gko::dim<2>(2, 3)); + ASSERT_EQ(data.nonzeros.size(), 6); + EXPECT_EQ(data.nonzeros[0], tpl(0, 0, value_type{1.0})); + EXPECT_EQ(data.nonzeros[1], tpl(0, 1, value_type{2.0})); + EXPECT_EQ(data.nonzeros[2], tpl(0, 2, value_type{3.0})); + EXPECT_EQ(data.nonzeros[3], tpl(1, 0, value_type{1.5})); + EXPECT_EQ(data.nonzeros[4], tpl(1, 1, value_type{2.5})); + EXPECT_EQ(data.nonzeros[5], tpl(1, 2, value_type{3.5})); +} + + +TYPED_TEST(Dense, CanCreateDeviceView) +{ + auto view = this->mtx->get_device_view(); + + EXPECT_EQ(view.size, this->mtx->get_size()); + EXPECT_EQ(view.stride, this->mtx->get_stride()); + EXPECT_EQ(view.values, this->mtx->get_values()); +} + + +TYPED_TEST(Dense, CanCreateConstDeviceView) +{ + auto view = this->mtx->get_const_device_view(); + + EXPECT_EQ(view.size, this->mtx->get_size()); + EXPECT_EQ(view.stride, this->mtx->get_stride()); + EXPECT_EQ(view.values, this->mtx->get_values()); +} + + +TYPED_TEST(Dense, CanCreateSubmatrix) +{ + using value_type = typename TestFixture::value_type; + auto submtx = this->mtx->create_subview(gko::span{0, 1}, gko::span{1, 3}); + + EXPECT_EQ(submtx->get_precision(), this->mtx->get_precision()); + EXPECT_EQ(submtx->get_size(), gko::dim<2>(1, 2)); + EXPECT_EQ(submtx->at(0, 0), value_type{2.0}); + EXPECT_EQ(submtx->at(0, 1), value_type{3.0}); + EXPECT_LT(std::distance(this->mtx->get_values(), submtx->get_values()), + this->mtx->get_num_stored_elements()); + EXPECT_EQ(&submtx->at(0, 0), &this->mtx->at(0, 1)); + EXPECT_EQ(&submtx->at(0, 1), &this->mtx->at(0, 2)); +} + + +TYPED_TEST(Dense, CanCreateEmptySubmatrix) +{ + auto submtx = this->mtx->create_subview(gko::span{0, 0}, gko::span{1, 1}); + + EXPECT_EQ(submtx->get_size(), gko::dim<2>{}); +} + + +} // namespace diff --git a/core/test/matrix/identity.cpp b/core/test/matrix/identity.cpp index 29dfabb7f2c..b04ef36fe13 100644 --- a/core/test/matrix/identity.cpp +++ b/core/test/matrix/identity.cpp @@ -88,8 +88,7 @@ TYPED_TEST(IdentityFactory, CanGenerateIdentityMatrix) { auto exec = gko::ReferenceExecutor::create(); auto id_factory = gko::matrix::IdentityFactory::create(exec); - auto mtx = - gko::matrix::MultiVector::create(exec, gko::dim<2>{5, 5}); + auto mtx = gko::matrix::Dense::create(exec, gko::dim<2>{5, 5}); auto id = id_factory->generate(std::move(mtx)); @@ -101,8 +100,7 @@ TYPED_TEST(IdentityFactory, FailsToGenerateRectangularIdentityMatrix) { auto exec = gko::ReferenceExecutor::create(); auto id_factory = gko::matrix::IdentityFactory::create(exec); - auto mtx = - gko::matrix::MultiVector::create(exec, gko::dim<2>{5, 4}); + auto mtx = gko::matrix::Dense::create(exec, gko::dim<2>{5, 4}); ASSERT_THROW(id_factory->generate(std::move(mtx)), gko::DimensionMismatch); } diff --git a/core/test/matrix/multivector.cpp b/core/test/matrix/multivector.cpp index 62b72d605af..663597dd08d 100644 --- a/core/test/matrix/multivector.cpp +++ b/core/test/matrix/multivector.cpp @@ -6,14 +6,12 @@ #include #include +#include #include #include "core/test/utils.hpp" -namespace { - - template class MultiVector : public ::testing::Test { protected: @@ -532,6 +530,3 @@ TEST(CustomMultiVector, CustomViewKeepsRuntimeType) EXPECT_TRUE(dynamic_cast(view.get())); ASSERT_EQ(dynamic_cast(view.get())->get_data(), 2); } - - -} // namespace diff --git a/core/test/matrix/row_gatherer.cpp b/core/test/matrix/row_gatherer.cpp index d3e657091c9..d1fc88c6fd4 100644 --- a/core/test/matrix/row_gatherer.cpp +++ b/core/test/matrix/row_gatherer.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include diff --git a/core/test/reorder/amd.cpp b/core/test/reorder/amd.cpp index 9eecf3777e1..fe92d330de0 100644 --- a/core/test/reorder/amd.cpp +++ b/core/test/reorder/amd.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -9,6 +9,7 @@ #include #include +#include #include #include "core/factorization/symbolic.hpp" diff --git a/core/test/solver/bicg.cpp b/core/test/solver/bicg.cpp index 72eac4845b1..069a18a7439 100644 --- a/core/test/solver/bicg.cpp +++ b/core/test/solver/bicg.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -23,7 +24,7 @@ template class Bicg : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::Bicg; Bicg() diff --git a/core/test/solver/bicgstab.cpp b/core/test/solver/bicgstab.cpp index 9a2218268ab..e054e7122c3 100644 --- a/core/test/solver/bicgstab.cpp +++ b/core/test/solver/bicgstab.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -22,7 +23,7 @@ template class Bicgstab : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::Bicgstab; Bicgstab() diff --git a/core/test/solver/cb_gmres.cpp b/core/test/solver/cb_gmres.cpp index 027a9d978ba..a34915c718d 100644 --- a/core/test/solver/cb_gmres.cpp +++ b/core/test/solver/cb_gmres.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -27,7 +28,7 @@ class CbGmres : public ::testing::Test { using nc_value_type = gko::remove_complex; using storage_helper_type = typename std::tuple_element<1, decltype(ValueEnumType())>::type; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::CbGmres; CbGmres() diff --git a/core/test/solver/cg.cpp b/core/test/solver/cg.cpp index 347a271aec8..f8831f203b0 100644 --- a/core/test/solver/cg.cpp +++ b/core/test/solver/cg.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -23,7 +24,7 @@ template class Cg : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::Cg; Cg() diff --git a/core/test/solver/cgs.cpp b/core/test/solver/cgs.cpp index b97f13717f5..b362dccd5d2 100644 --- a/core/test/solver/cgs.cpp +++ b/core/test/solver/cgs.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -23,7 +24,7 @@ template class Cgs : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::Cgs; Cgs() diff --git a/core/test/solver/chebyshev.cpp b/core/test/solver/chebyshev.cpp index 8a07e2a686b..b08f913bdca 100644 --- a/core/test/solver/chebyshev.cpp +++ b/core/test/solver/chebyshev.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -21,7 +22,7 @@ template class Chebyshev : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::Chebyshev; Chebyshev() diff --git a/core/test/solver/fcg.cpp b/core/test/solver/fcg.cpp index 38127da0e07..780f41f0438 100644 --- a/core/test/solver/fcg.cpp +++ b/core/test/solver/fcg.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -21,7 +22,7 @@ template class Fcg : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::Fcg; Fcg() diff --git a/core/test/solver/gcr.cpp b/core/test/solver/gcr.cpp index 933310e8cb7..b195aa956a6 100644 --- a/core/test/solver/gcr.cpp +++ b/core/test/solver/gcr.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -23,7 +24,7 @@ template class Gcr : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::Gcr; using Big_solver = gko::solver::Gcr; diff --git a/core/test/solver/gmres.cpp b/core/test/solver/gmres.cpp index fe98ab57e7c..162d7055dea 100644 --- a/core/test/solver/gmres.cpp +++ b/core/test/solver/gmres.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -23,7 +24,7 @@ template class Gmres : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::Gmres; using Big_solver = gko::solver::Gmres; diff --git a/core/test/solver/idr.cpp b/core/test/solver/idr.cpp index e9173b89988..b4aacf72a34 100644 --- a/core/test/solver/idr.cpp +++ b/core/test/solver/idr.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -22,7 +23,7 @@ template class Idr : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::Idr; Idr() diff --git a/core/test/solver/ir.cpp b/core/test/solver/ir.cpp index 064ac734876..f0bf18adcfc 100644 --- a/core/test/solver/ir.cpp +++ b/core/test/solver/ir.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -24,7 +25,8 @@ template class Ir : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; + using Vec = gko::matrix::MultiVector; using Solver = gko::solver::Ir; Ir() @@ -373,11 +375,9 @@ struct TestSummaryWriter : gko::log::ProfilerHook::SummaryWriter { TYPED_TEST(Ir, RunResidualNormCheckCorrectTimes) { - using value_type = typename TestFixture::value_type; - using Solver = typename TestFixture::Solver; - using Mtx = typename TestFixture::Mtx; - auto b = gko::initialize({2, -1.0, 1.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); + using Vec = typename TestFixture::Vec; + auto b = gko::initialize({2, -1.0, 1.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); auto logger = gko::share(gko::log::ProfilerHook::create_summary( std::make_shared(), std::make_unique())); diff --git a/core/test/solver/lower_trs.cpp b/core/test/solver/lower_trs.cpp index 1e86d631379..006a8fed40a 100644 --- a/core/test/solver/lower_trs.cpp +++ b/core/test/solver/lower_trs.cpp @@ -45,7 +45,7 @@ TYPED_TEST(LowerTrs, LowerTrsFactoryKnowsItsExecutor) TYPED_TEST(LowerTrs, ThrowsOnRectangularMatrixInFactory) { - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; std::shared_ptr rectangular_matrix = Mtx::create(this->exec, gko::dim<2>{1, 2}); diff --git a/core/test/solver/multigrid.cpp b/core/test/solver/multigrid.cpp index 9114f99ba7e..56e09d075ae 100644 --- a/core/test/solver/multigrid.cpp +++ b/core/test/solver/multigrid.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -95,7 +96,7 @@ template class Multigrid : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::Multigrid; using DummyRPFactory = DummyLinOpWithFactory; using DummyFactory = DummyLinOpWithFactory; diff --git a/core/test/solver/pipe_cg.cpp b/core/test/solver/pipe_cg.cpp index 853b3e81c2e..d36f796d2d0 100644 --- a/core/test/solver/pipe_cg.cpp +++ b/core/test/solver/pipe_cg.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -20,7 +21,7 @@ template class PipeCg : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::PipeCg; PipeCg() diff --git a/core/test/solver/upper_trs.cpp b/core/test/solver/upper_trs.cpp index fba01ea007b..0f0d07bf617 100644 --- a/core/test/solver/upper_trs.cpp +++ b/core/test/solver/upper_trs.cpp @@ -45,7 +45,7 @@ TYPED_TEST(UpperTrs, UpperTrsFactoryKnowsItsExecutor) TYPED_TEST(UpperTrs, ThrowsOnRectangularMatrixInFactory) { - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; std::shared_ptr rectangular_matrix = Mtx::create(this->exec, gko::dim<2>{1, 2}); diff --git a/core/test/utils/assertions.hpp b/core/test/utils/assertions.hpp index 04336ec6dc4..b13fb560bdc 100644 --- a/core/test/utils/assertions.hpp +++ b/core/test/utils/assertions.hpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "core/base/batch_utilities.hpp" diff --git a/core/test/utils/assertions_test.cpp b/core/test/utils/assertions_test.cpp index 77f223c8c34..90472e4a11d 100644 --- a/core/test/utils/assertions_test.cpp +++ b/core/test/utils/assertions_test.cpp @@ -10,6 +10,7 @@ #include #include +#include #include diff --git a/core/test/utils/matrix_generator.hpp b/core/test/utils/matrix_generator.hpp index 44a1dadd7eb..99ebf8a0737 100644 --- a/core/test/utils/matrix_generator.hpp +++ b/core/test/utils/matrix_generator.hpp @@ -250,9 +250,8 @@ std::unique_ptr generate_random_matrix( return result; } - /** - * Generates a random dense matrix. + * Generates a random multivector. * * @tparam ValueType value type of the generated matrix * @tparam ValueDistribution type of value distribution @@ -270,7 +269,7 @@ std::unique_ptr generate_random_matrix( template std::unique_ptr> -generate_random_dense_matrix(size_type num_rows, size_type num_cols, +generate_random_multi_vector(size_type num_rows, size_type num_cols, ValueDistribution&& value_dist, Engine&& engine, std::shared_ptr exec, MatrixArgs&&... args) @@ -286,6 +285,39 @@ generate_random_dense_matrix(size_type num_rows, size_type num_cols, } +/** + * Generates a random dense matrix. + * + * @tparam ValueType value type of the generated matrix + * @tparam ValueDistribution type of value distribution + * @tparam Engine type of random engine + * + * @param num_rows number of rows + * @param num_cols number of columns + * @param value_dist distribution of matrix values + * @param engine a random engine + * @param exec executor where the matrix should be allocated + * @param args additional arguments for the matrix constructor + * + * @return the unique pointer of gko::matrix::Dense + */ +template +std::unique_ptr> generate_random_dense_matrix( + size_type num_rows, size_type num_cols, ValueDistribution&& value_dist, + Engine&& engine, std::shared_ptr exec, MatrixArgs&&... args) +{ + auto result = gko::matrix::Dense::create( + exec, gko::dim<2>{num_rows, num_cols}, + std::forward(args)...); + result->read( + matrix_data{gko::dim<2>{num_rows, num_cols}, + std::forward(value_dist), + std::forward(engine)}); + return result; +} + + /** * Generates a random triangular matrix. * diff --git a/core/test/utils/matrix_generator_test.cpp b/core/test/utils/matrix_generator_test.cpp index 450062f767f..7a1b8745739 100644 --- a/core/test/utils/matrix_generator_test.cpp +++ b/core/test/utils/matrix_generator_test.cpp @@ -22,7 +22,8 @@ class MatrixGenerator : public ::testing::Test { using value_type = T; using check_type = double; using real_type = gko::remove_complex; - using mtx_type = gko::matrix::MultiVector; + using mtx_type = gko::matrix::Dense; + using vec_type = gko::matrix::MultiVector; MatrixGenerator() : exec(gko::ReferenceExecutor::create()), @@ -33,6 +34,9 @@ class MatrixGenerator : public ::testing::Test { dense_mtx(gko::test::generate_random_dense_matrix( 500, 100, std::normal_distribution<>(20.0, 5.0), std::default_random_engine(41), exec)), + multi_vector(gko::test::generate_random_multi_vector( + 500, 100, std::normal_distribution<>(20.0, 5.0), + std::default_random_engine(41), exec)), l_mtx(gko::test::generate_random_lower_triangular_matrix( 4, true, std::normal_distribution<>(50, 5), std::normal_distribution<>(20.0, 5.0), @@ -67,6 +71,9 @@ class MatrixGenerator : public ::testing::Test { for (int col = 0; col < dense_mtx->get_size()[1]; ++col) { auto val = dense_mtx->at(row, col); dense_values_sample.push_back(val); + + auto vval = multi_vector->at(row, col); + multi_vector_values_sample.push_back(vval); } } @@ -87,12 +94,14 @@ class MatrixGenerator : public ::testing::Test { int upper_bandwidth; std::unique_ptr mtx; std::unique_ptr dense_mtx; + std::unique_ptr multi_vector; std::unique_ptr l_mtx; std::unique_ptr u_mtx; std::unique_ptr band_mtx; std::vector nnz_per_row_sample; std::vector values_sample; std::vector dense_values_sample; + std::vector multi_vector_values_sample; std::vector band_values_sample; @@ -171,6 +180,24 @@ TYPED_TEST(MatrixGenerator, OutputHasCorrectValuesAverageAndDeviation) TYPED_TEST(MatrixGenerator, MultiVectorOutputHasCorrectValuesAverageAndDeviation) +{ + using T = typename TestFixture::value_type; + // check the real part + this->template check_average_and_deviation( + begin(this->multi_vector_values_sample), + end(this->multi_vector_values_sample), 20.0, 5.0, + [](T& val) { return gko::real(val); }); + // check the imag part when the type is complex + if (!std::is_same>::value) { + this->template check_average_and_deviation( + begin(this->multi_vector_values_sample), + end(this->multi_vector_values_sample), 20.0, 5.0, + [](T& val) { return gko::imag(val); }); + } +} + + +TYPED_TEST(MatrixGenerator, DenseOutputHasCorrectValuesAverageAndDeviation) { using T = typename TestFixture::value_type; // check the real part @@ -274,7 +301,8 @@ TYPED_TEST(MatrixGenerator, CanGenerateTridiagMatrix) TYPED_TEST(MatrixGenerator, CanGenerateTridiagInverseMatrix) { using T = typename TestFixture::value_type; - using MultiVector = typename TestFixture::mtx_type; + using Mtx = typename TestFixture::mtx_type; + using MultiVector = typename TestFixture::vec_type; auto dist = std::normal_distribution<>(0, 1); auto engine = std::default_random_engine(42); auto lower = gko::test::detail::get_rand_value(dist, engine); @@ -288,13 +316,13 @@ TYPED_TEST(MatrixGenerator, CanGenerateTridiagInverseMatrix) size = 5; } - auto mtx = gko::test::generate_tridiag_matrix( + auto mtx = gko::test::generate_tridiag_matrix( size, {lower, diag, upper}, this->exec); - auto inv_mtx = gko::test::generate_tridiag_inverse_matrix( + auto inv_mtx = gko::test::generate_tridiag_inverse_matrix( size, {lower, diag, upper}, this->exec); auto result = MultiVector::create(this->exec, mtx->get_size()); - inv_mtx->apply(mtx, result); + inv_mtx->apply(mtx->as_const_multivector_view(), result); auto id = MultiVector::create(this->exec, mtx->get_size()); id->fill(0.0); for (gko::size_type i = 0; i < mtx->get_size()[0]; ++i) { diff --git a/core/test/utils/matrix_utils_test.cpp b/core/test/utils/matrix_utils_test.cpp index bbaa007d2e1..d16a6390432 100644 --- a/core/test/utils/matrix_utils_test.cpp +++ b/core/test/utils/matrix_utils_test.cpp @@ -11,6 +11,7 @@ #include #include +#include #include "core/test/utils.hpp" #include "core/test/utils/matrix_generator.hpp" diff --git a/cuda/test/solver/lower_trs_kernels.cu b/cuda/test/solver/lower_trs_kernels.cu index 6f141008c2c..212f3bd65e8 100644 --- a/cuda/test/solver/lower_trs_kernels.cu +++ b/cuda/test/solver/lower_trs_kernels.cu @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -26,13 +27,14 @@ namespace { class LowerTrs : public CudaTestFixture { protected: using CsrMtx = gko::matrix::Csr; - using Mtx = gko::matrix::MultiVector<>; + using Mtx = gko::matrix::Dense<>; + using Vec = gko::matrix::MultiVector<>; LowerTrs() : rand_engine(30) {} - std::unique_ptr gen_mtx(int num_rows, int num_cols) + std::unique_ptr gen_vec(int num_rows, int num_cols) { - return gko::test::generate_random_matrix( + return gko::test::generate_random_matrix( num_rows, num_cols, std::uniform_int_distribution<>(num_cols, num_cols), std::normal_distribution<>(-1.0, 1.0), rand_engine, ref); @@ -48,26 +50,26 @@ protected: void initialize_data(int m, int n) { mtx = gen_l_mtx(m); - b = gen_mtx(m, n); - x = gen_mtx(m, n); + b = gen_vec(m, n); + x = gen_vec(m, n); csr_mtx = CsrMtx::create(ref); mtx->convert_to(csr_mtx); d_csr_mtx = CsrMtx::create(exec); d_x = gko::clone(exec, x); d_csr_mtx->copy_from(csr_mtx); - b2 = Mtx::create(ref); + b2 = Vec::create(ref); d_b2 = gko::clone(exec, b); b2->copy_from(b); } - std::shared_ptr b; - std::shared_ptr b2; - std::shared_ptr x; + std::shared_ptr b; + std::shared_ptr b2; + std::shared_ptr x; std::shared_ptr mtx; std::shared_ptr csr_mtx; - std::shared_ptr d_b; - std::shared_ptr d_b2; - std::shared_ptr d_x; + std::shared_ptr d_b; + std::shared_ptr d_b2; + std::shared_ptr d_x; std::shared_ptr d_csr_mtx; std::default_random_engine rand_engine; }; @@ -129,9 +131,9 @@ TEST_F(LowerTrs, CudaMultipleRhsApplySyncfreeIsEquivalentToRef) .on(exec); auto solver = lower_trs_factory->generate(csr_mtx); auto d_solver = d_lower_trs_factory->generate(d_csr_mtx); - auto db2_strided = Mtx::create(exec, b->get_size(), 4); + auto db2_strided = Vec::create(exec, b->get_size(), 4); d_b2->convert_to(db2_strided); - auto dx_strided = Mtx::create(exec, x->get_size(), 5); + auto dx_strided = Vec::create(exec, x->get_size(), 5); solver->apply(b2, x); d_solver->apply(db2_strided, dx_strided); @@ -165,9 +167,10 @@ TEST_F(LowerTrs, CudaMultipleRhsApplyIsEquivalentToRef) gko::solver::LowerTrs<>::build().with_num_rhs(3u).on(exec); auto solver = lower_trs_factory->generate(csr_mtx); auto d_solver = d_lower_trs_factory->generate(d_csr_mtx); - auto db2_strided = Mtx::create(exec, b->get_size(), in_stride); + auto db2_strided = Vec::create(exec, b->get_size(), in_stride); d_b2->convert_to(db2_strided); - auto dx_strided = Mtx::create(exec, x->get_size(), out_stride); + + auto dx_strided = Vec::create(exec, x->get_size(), out_stride); solver->apply(b2, x); d_solver->apply(db2_strided, dx_strided); diff --git a/cuda/test/solver/upper_trs_kernels.cu b/cuda/test/solver/upper_trs_kernels.cu index fb0ab576f45..546a340bbe0 100644 --- a/cuda/test/solver/upper_trs_kernels.cu +++ b/cuda/test/solver/upper_trs_kernels.cu @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -26,13 +27,14 @@ namespace { class UpperTrs : public CudaTestFixture { protected: using CsrMtx = gko::matrix::Csr; - using Mtx = gko::matrix::MultiVector<>; + using Mtx = gko::matrix::Dense<>; + using Vec = gko::matrix::MultiVector<>; UpperTrs() : rand_engine(30) {} - std::unique_ptr gen_mtx(int num_rows, int num_cols) + std::unique_ptr gen_vec(int num_rows, int num_cols) { - return gko::test::generate_random_matrix( + return gko::test::generate_random_matrix( num_rows, num_cols, std::uniform_int_distribution<>(num_cols, num_cols), std::normal_distribution<>(-1.0, 1.0), rand_engine, ref); @@ -48,26 +50,26 @@ protected: void initialize_data(int m, int n) { mtx = gen_u_mtx(m); - b = gen_mtx(m, n); - x = gen_mtx(m, n); + b = gen_vec(m, n); + x = gen_vec(m, n); csr_mtx = CsrMtx::create(ref); mtx->convert_to(csr_mtx); d_csr_mtx = CsrMtx::create(exec); d_x = gko::clone(exec, x); d_csr_mtx->copy_from(csr_mtx); - b2 = Mtx::create(ref); + b2 = Vec::create(ref); d_b2 = gko::clone(exec, b); b2->copy_from(b); } - std::shared_ptr b; - std::shared_ptr b2; - std::shared_ptr x; + std::shared_ptr b; + std::shared_ptr b2; + std::shared_ptr x; std::shared_ptr mtx; std::shared_ptr csr_mtx; - std::shared_ptr d_b; - std::shared_ptr d_b2; - std::shared_ptr d_x; + std::shared_ptr d_b; + std::shared_ptr d_b2; + std::shared_ptr d_x; std::shared_ptr d_csr_mtx; std::default_random_engine rand_engine; }; @@ -129,9 +131,9 @@ TEST_F(UpperTrs, CudaMultipleRhsApplySyncfreeIsEquivalentToRef) .on(exec); auto solver = upper_trs_factory->generate(csr_mtx); auto d_solver = d_upper_trs_factory->generate(d_csr_mtx); - auto db2_strided = Mtx::create(exec, b->get_size(), 4); + auto db2_strided = Vec::create(exec, b->get_size(), 4); d_b2->convert_to(db2_strided); - auto dx_strided = Mtx::create(exec, x->get_size(), 5); + auto dx_strided = Vec::create(exec, x->get_size(), 5); solver->apply(b2, x); d_solver->apply(db2_strided, dx_strided); @@ -165,9 +167,9 @@ TEST_F(UpperTrs, CudaMultipleRhsApplyIsEquivalentToRef) gko::solver::UpperTrs<>::build().with_num_rhs(3u).on(exec); auto solver = upper_trs_factory->generate(csr_mtx); auto d_solver = d_upper_trs_factory->generate(d_csr_mtx); - auto db2_strided = Mtx::create(exec, b->get_size(), in_stride); + auto db2_strided = Vec::create(exec, b->get_size(), in_stride); d_b2->convert_to(db2_strided); - auto dx_strided = Mtx::create(exec, x->get_size(), out_stride); + auto dx_strided = Vec::create(exec, x->get_size(), out_stride); solver->apply(b2, x); d_solver->apply(db2_strided, dx_strided); diff --git a/cuda/test/utils/assertions_test.cu b/cuda/test/utils/assertions_test.cu index d30de413684..76f5f402372 100644 --- a/cuda/test/utils/assertions_test.cu +++ b/cuda/test/utils/assertions_test.cu @@ -20,7 +20,7 @@ class MatricesNear : public CudaTestFixture {}; TEST_F(MatricesNear, CanPassCudaMatrix) { - auto mtx = gko::initialize>( + auto mtx = gko::initialize>( {{1.0, 2.0, 3.0}, {0.0, 4.0, 0.0}}, ref); auto csr_ref = gko::matrix::Csr<>::create(ref); csr_ref->copy_from(mtx); diff --git a/dpcpp/CMakeLists.txt b/dpcpp/CMakeLists.txt index 12ee187bfc7..e066d2315a5 100644 --- a/dpcpp/CMakeLists.txt +++ b/dpcpp/CMakeLists.txt @@ -64,6 +64,7 @@ target_sources( matrix/batch_ell_kernels.dp.cpp matrix/coo_kernels.dp.cpp matrix/csr_kernels.dp.cpp + matrix/dense_kernels.dp.cpp matrix/diagonal_kernels.dp.cpp matrix/ell_kernels.dp.cpp matrix/fbcsr_kernels.dp.cpp diff --git a/dpcpp/matrix/dense_kernels.dp.cpp b/dpcpp/matrix/dense_kernels.dp.cpp new file mode 100644 index 00000000000..55cc3bdf710 --- /dev/null +++ b/dpcpp/matrix/dense_kernels.dp.cpp @@ -0,0 +1,360 @@ +// SPDX-FileCopyrightText: 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include "core/matrix/dense_kernels.hpp" + +#include + +#include + +#include "core/components/prefix_sum_kernels.hpp" +#include "core/matrix/multivector_kernels.hpp" +#include "dpcpp/base/config.hpp" +#include "dpcpp/base/dim3.dp.hpp" +#include "dpcpp/base/helper.hpp" +#include "dpcpp/base/math.hpp" +#include "dpcpp/base/onemkl_bindings.hpp" +#include "dpcpp/base/types.hpp" +#include "dpcpp/components/cooperative_groups.dp.hpp" +#include "dpcpp/components/reduction.dp.hpp" +#include "dpcpp/components/thread_ids.dp.hpp" +#include "dpcpp/components/uninitialized_array.hpp" +#include "dpcpp/synthesizer/implementation_selection.hpp" + + +namespace gko { +namespace kernels { +namespace dpcpp { +/** + * @brief The dense matrix format namespace. + * + * @ingroup dense + */ +namespace dense { + + +template +void simple_apply(std::shared_ptr exec, + matrix::view::dense a, + matrix::view::dense b, + matrix::view::dense c) +{ + using namespace oneapi::mkl; + if constexpr (onemkl::is_supported::value) { + if (b.stride != 0 && c.stride != 0) { + if (a.size[1] > 0 && a.values && b.values && c.values) { + oneapi::mkl::blas::row_major::gemm( + *exec->get_queue(), transpose::nontrans, + transpose::nontrans, c.size[0], c.size[1], a.size[1], + one(), as_device_type(a.values), a.stride, + as_device_type(b.values), b.stride, zero(), + as_device_type(c.values), c.stride); + } else { + multivector::fill(exec, c, zero()); + } + } + } else { + GKO_NOT_IMPLEMENTED; + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL); + + +template +void apply(std::shared_ptr exec, + matrix::view::dense alpha, + matrix::view::dense a, + matrix::view::dense b, + matrix::view::dense beta, + matrix::view::dense c) +{ + using namespace oneapi::mkl; + if constexpr (onemkl::is_supported::value) { + if (b.stride != 0 && c.stride != 0) { + if (a.size[1] > 0 && a.values && b.values && c.values) { + oneapi::mkl::blas::row_major::gemm( + *exec->get_queue(), transpose::nontrans, + transpose::nontrans, c.size[0], c.size[1], a.size[1], + exec->copy_val_to_host(alpha.values), + as_device_type(a.values), a.stride, + as_device_type(b.values), b.stride, + exec->copy_val_to_host(beta.values), + as_device_type(c.values), c.stride); + } else { + dense::scale(exec, beta, c); + } + } + } else { + GKO_NOT_IMPLEMENTED; + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL); + + +template +void convert_to_coo(std::shared_ptr exec, + matrix::view::dense source, + const int64* row_ptrs, + matrix::view::coo result) +{ + const auto num_rows = result.size[0]; + const auto num_cols = result.size[1]; + const auto in_vals = as_device_type(source.values); + const auto stride = source.stride; + + auto rows = result.row_idxs; + auto cols = result.col_idxs; + auto vals = as_device_type(result.values); + + exec->get_queue()->submit([&](sycl::handler& cgh) { + cgh.parallel_for(num_rows, [=](sycl::item<1> item) { + const auto row = static_cast(item[0]); + auto write_to = row_ptrs[row]; + + for (size_type col = 0; col < num_cols; col++) { + if (is_nonzero(in_vals[stride * row + col])) { + vals[write_to] = in_vals[stride * row + col]; + cols[write_to] = static_cast(col); + rows[write_to] = static_cast(row); + write_to++; + } + } + }); + }); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL); + + +template +void convert_to_csr(std::shared_ptr exec, + matrix::view::dense source, + matrix::Csr* result) +{ + const auto num_rows = result->get_size()[0]; + const auto num_cols = result->get_size()[1]; + const auto in_vals = as_device_type(source.values); + const auto stride = source.stride; + + const auto row_ptrs = result->get_const_row_ptrs(); + auto cols = result->get_col_idxs(); + auto vals = as_device_type(result->get_values()); + + exec->get_queue()->submit([&](sycl::handler& cgh) { + cgh.parallel_for(num_rows, [=](sycl::item<1> item) { + const auto row = static_cast(item[0]); + auto write_to = row_ptrs[row]; + + for (size_type col = 0; col < num_cols; col++) { + if (is_nonzero(in_vals[stride * row + col])) { + vals[write_to] = in_vals[stride * row + col]; + cols[write_to] = static_cast(col); + write_to++; + } + } + }); + }); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL); + + +template +void convert_to_ell(std::shared_ptr exec, + matrix::view::dense source, + matrix::view::ell result) +{ + const auto num_rows = result.size[0]; + const auto num_cols = result.size[1]; + const auto max_nnz_per_row = result.num_stored_elements_per_row; + const auto in_vals = as_device_type(source.values); + const auto in_stride = source.stride; + + auto cols = result.col_idxs; + auto vals = as_device_type(result.values); + const auto stride = result.stride; + + exec->get_queue()->submit([&](sycl::handler& cgh) { + cgh.parallel_for(num_rows, [=](sycl::item<1> item) { + const auto row = static_cast(item[0]); + size_type col_idx = 0; + for (size_type col = 0; col < num_cols; col++) { + if (is_nonzero(in_vals[row * in_stride + col])) { + cols[col_idx * stride + row] = col; + vals[col_idx * stride + row] = + in_vals[row * in_stride + col]; + col_idx++; + } + } + for (; col_idx < max_nnz_per_row; col_idx++) { + cols[col_idx * stride + row] = invalid_index(); + vals[col_idx * stride + row] = zero>(); + } + }); + }); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL); + + +template +void convert_to_fbcsr(std::shared_ptr exec, + matrix::view::dense source, + matrix::Fbcsr* result) + GKO_NOT_IMPLEMENTED; + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_FBCSR_KERNEL); + + +template +void count_nonzero_blocks_per_row(std::shared_ptr exec, + matrix::view::dense source, + int bs, + IndexType* result) GKO_NOT_IMPLEMENTED; + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL); + + +template +void convert_to_hybrid(std::shared_ptr exec, + matrix::view::dense source, + const int64* coo_row_ptrs, + matrix::Hybrid* result) +{ + const auto num_rows = result->get_size()[0]; + const auto num_cols = result->get_size()[1]; + const auto ell_lim = result->get_ell_num_stored_elements_per_row(); + const auto in_vals = as_device_type(source.values); + const auto in_stride = source.stride; + const auto ell_stride = result->get_ell_stride(); + auto ell_cols = result->get_ell_col_idxs(); + auto ell_vals = as_device_type(result->get_ell_values()); + auto coo_rows = result->get_coo_row_idxs(); + auto coo_cols = result->get_coo_col_idxs(); + auto coo_vals = as_device_type(result->get_coo_values()); + + exec->get_queue()->submit([&](sycl::handler& cgh) { + cgh.parallel_for(num_rows, [=](sycl::item<1> item) { + const auto row = static_cast(item[0]); + size_type ell_count = 0; + size_type col = 0; + auto ell_idx = row; + for (; col < num_cols && ell_count < ell_lim; col++) { + const auto val = in_vals[row * in_stride + col]; + if (is_nonzero(val)) { + ell_vals[ell_idx] = val; + ell_cols[ell_idx] = static_cast(col); + ell_count++; + ell_idx += ell_stride; + } + } + for (; ell_count < ell_lim; ell_count++) { + ell_vals[ell_idx] = zero>(); + ell_cols[ell_idx] = invalid_index(); + ell_idx += ell_stride; + } + auto coo_idx = coo_row_ptrs[row]; + for (; col < num_cols; col++) { + const auto val = in_vals[row * in_stride + col]; + if (is_nonzero(val)) { + coo_vals[coo_idx] = val; + coo_cols[coo_idx] = static_cast(col); + coo_rows[coo_idx] = static_cast(row); + coo_idx++; + } + } + }); + }); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL); + + +template +void convert_to_sellp(std::shared_ptr exec, + matrix::view::dense source, + matrix::view::sellp result) +{ + const auto num_rows = result.size[0]; + const auto num_cols = result.size[1]; + const auto stride = source.stride; + const auto in_vals = as_device_type(source.values); + + const auto slice_sets = result.slice_sets; + const auto slice_size = result.slice_size; + auto vals = as_device_type(result.values); + auto col_idxs = result.col_idxs; + + exec->get_queue()->submit([&](sycl::handler& cgh) { + cgh.parallel_for(num_rows, [=](sycl::item<1> item) { + const auto row = static_cast(item[0]); + const auto local_row = row % slice_size; + const auto slice = row / slice_size; + const auto slice_end = slice_sets[slice + 1] * slice_size; + auto out_idx = slice_sets[slice] * slice_size + local_row; + + for (size_type col = 0; col < num_cols; col++) { + const auto val = in_vals[row * stride + col]; + if (is_nonzero(val)) { + col_idxs[out_idx] = static_cast(col); + vals[out_idx] = val; + out_idx += slice_size; + } + } + for (; out_idx < slice_end; out_idx += slice_size) { + col_idxs[out_idx] = invalid_index(); + vals[out_idx] = zero>(); + } + }); + }); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL); + + +template +void convert_to_sparsity_csr(std::shared_ptr exec, + matrix::view::dense source, + matrix::SparsityCsr* result) +{ + const auto num_rows = result->get_size()[0]; + const auto num_cols = result->get_size()[1]; + const auto in_vals = as_device_type(source.values); + const auto stride = source.stride; + + const auto row_ptrs = result->get_const_row_ptrs(); + auto cols = result->get_col_idxs(); + + exec->get_queue()->submit([&](sycl::handler& cgh) { + cgh.parallel_for(num_rows, [=](sycl::item<1> item) { + const auto row = static_cast(item[0]); + auto write_to = row_ptrs[row]; + + for (size_type col = 0; col < num_cols; col++) { + if (is_nonzero(in_vals[stride * row + col])) { + cols[write_to] = static_cast(col); + write_to++; + } + } + }); + }); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL); + + +} // namespace dense +} // namespace dpcpp +} // namespace kernels +} // namespace gko diff --git a/dpcpp/matrix/multivector_kernels.dp.cpp b/dpcpp/matrix/multivector_kernels.dp.cpp index 9078e2d0d78..ad11b0cd8ea 100644 --- a/dpcpp/matrix/multivector_kernels.dp.cpp +++ b/dpcpp/matrix/multivector_kernels.dp.cpp @@ -9,13 +9,7 @@ #include #include -#include -#include -#include -#include -#include -#include "core/components/prefix_sum_kernels.hpp" #include "dpcpp/base/config.hpp" #include "dpcpp/base/dim3.dp.hpp" #include "dpcpp/base/helper.hpp" @@ -198,341 +192,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( GKO_DECLARE_MULTIVECTOR_COMPUTE_CONJ_DOT_DISPATCH_KERNEL); -template -void compute_norm2_dispatch( - std::shared_ptr exec, - matrix::view::dense x, - matrix::view::dense> result, array& tmp) -{ - // TODO Add onemkl for single column ? - compute_norm2(exec, x, result, tmp); -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_COMPUTE_NORM2_DISPATCH_KERNEL); - - -template -void simple_apply(std::shared_ptr exec, - matrix::view::dense a, - matrix::view::dense b, - matrix::view::dense c) -{ - using namespace oneapi::mkl; - if constexpr (onemkl::is_supported::value) { - if (b.stride != 0 && c.stride != 0) { - if (a.size[1] > 0 && a.values && b.values && c.values) { - oneapi::mkl::blas::row_major::gemm( - *exec->get_queue(), transpose::nontrans, - transpose::nontrans, c.size[0], c.size[1], a.size[1], - one(), as_device_type(a.values), a.stride, - as_device_type(b.values), b.stride, zero(), - as_device_type(c.values), c.stride); - } else { - multivector::fill(exec, c, zero()); - } - } - } else { - GKO_NOT_IMPLEMENTED; - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_SIMPLE_APPLY_KERNEL); - - -template -void apply(std::shared_ptr exec, - matrix::view::dense alpha, - matrix::view::dense a, - matrix::view::dense b, - matrix::view::dense beta, - matrix::view::dense c) -{ - using namespace oneapi::mkl; - if constexpr (onemkl::is_supported::value) { - if (b.stride != 0 && c.stride != 0) { - if (a.size[1] > 0 && a.values && b.values && c.values) { - oneapi::mkl::blas::row_major::gemm( - *exec->get_queue(), transpose::nontrans, - transpose::nontrans, c.size[0], c.size[1], a.size[1], - exec->copy_val_to_host(alpha.values), - as_device_type(a.values), a.stride, - as_device_type(b.values), b.stride, - exec->copy_val_to_host(beta.values), - as_device_type(c.values), c.stride); - } else { - multivector::scale(exec, beta, c); - } - } - } else { - GKO_NOT_IMPLEMENTED; - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_APPLY_KERNEL); - - -template -void convert_to_coo(std::shared_ptr exec, - matrix::view::dense source, - const int64* row_ptrs, - matrix::view::coo result) -{ - const auto num_rows = result.size[0]; - const auto num_cols = result.size[1]; - const auto in_vals = as_device_type(source.values); - const auto stride = source.stride; - - auto rows = result.row_idxs; - auto cols = result.col_idxs; - auto vals = as_device_type(result.values); - - exec->get_queue()->submit([&](sycl::handler& cgh) { - cgh.parallel_for(num_rows, [=](sycl::item<1> item) { - const auto row = static_cast(item[0]); - auto write_to = row_ptrs[row]; - - for (size_type col = 0; col < num_cols; col++) { - if (is_nonzero(in_vals[stride * row + col])) { - vals[write_to] = in_vals[stride * row + col]; - cols[write_to] = static_cast(col); - rows[write_to] = static_cast(row); - write_to++; - } - } - }); - }); -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_COO_KERNEL); - - -template -void convert_to_csr(std::shared_ptr exec, - matrix::view::dense source, - matrix::Csr* result) -{ - const auto num_rows = result->get_size()[0]; - const auto num_cols = result->get_size()[1]; - const auto in_vals = as_device_type(source.values); - const auto stride = source.stride; - - const auto row_ptrs = result->get_const_row_ptrs(); - auto cols = result->get_col_idxs(); - auto vals = as_device_type(result->get_values()); - - exec->get_queue()->submit([&](sycl::handler& cgh) { - cgh.parallel_for(num_rows, [=](sycl::item<1> item) { - const auto row = static_cast(item[0]); - auto write_to = row_ptrs[row]; - - for (size_type col = 0; col < num_cols; col++) { - if (is_nonzero(in_vals[stride * row + col])) { - vals[write_to] = in_vals[stride * row + col]; - cols[write_to] = static_cast(col); - write_to++; - } - } - }); - }); -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_CSR_KERNEL); - - -template -void convert_to_ell(std::shared_ptr exec, - matrix::view::dense source, - matrix::view::ell result) -{ - const auto num_rows = result.size[0]; - const auto num_cols = result.size[1]; - const auto max_nnz_per_row = result.num_stored_elements_per_row; - const auto in_vals = as_device_type(source.values); - const auto in_stride = source.stride; - - auto cols = result.col_idxs; - auto vals = as_device_type(result.values); - const auto stride = result.stride; - - exec->get_queue()->submit([&](sycl::handler& cgh) { - cgh.parallel_for(num_rows, [=](sycl::item<1> item) { - const auto row = static_cast(item[0]); - size_type col_idx = 0; - for (size_type col = 0; col < num_cols; col++) { - if (is_nonzero(in_vals[row * in_stride + col])) { - cols[col_idx * stride + row] = col; - vals[col_idx * stride + row] = - in_vals[row * in_stride + col]; - col_idx++; - } - } - for (; col_idx < max_nnz_per_row; col_idx++) { - cols[col_idx * stride + row] = invalid_index(); - vals[col_idx * stride + row] = zero>(); - } - }); - }); -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_ELL_KERNEL); - - -template -void convert_to_fbcsr(std::shared_ptr exec, - matrix::view::dense source, - matrix::Fbcsr* result) - GKO_NOT_IMPLEMENTED; - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_FBCSR_KERNEL); - - -template -void count_nonzero_blocks_per_row(std::shared_ptr exec, - matrix::view::dense source, - int bs, - IndexType* result) GKO_NOT_IMPLEMENTED; - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL); - - -template -void convert_to_hybrid(std::shared_ptr exec, - matrix::view::dense source, - const int64* coo_row_ptrs, - matrix::view::hybrid result) -{ - const auto num_rows = result.size[0]; - const auto num_cols = result.size[1]; - const auto ell_lim = result.ell_part.num_stored_elements_per_row; - const auto in_vals = as_device_type(source.values); - const auto in_stride = source.stride; - const auto ell_stride = result.ell_part.stride; - auto ell_cols = result.ell_part.col_idxs; - auto ell_vals = as_device_type(result.ell_part.values); - auto coo_rows = result.coo_part.row_idxs; - auto coo_cols = result.coo_part.col_idxs; - auto coo_vals = as_device_type(result.coo_part.values); - - exec->get_queue()->submit([&](sycl::handler& cgh) { - cgh.parallel_for(num_rows, [=](sycl::item<1> item) { - const auto row = static_cast(item[0]); - size_type ell_count = 0; - size_type col = 0; - auto ell_idx = row; - for (; col < num_cols && ell_count < ell_lim; col++) { - const auto val = in_vals[row * in_stride + col]; - if (is_nonzero(val)) { - ell_vals[ell_idx] = val; - ell_cols[ell_idx] = static_cast(col); - ell_count++; - ell_idx += ell_stride; - } - } - for (; ell_count < ell_lim; ell_count++) { - ell_vals[ell_idx] = zero>(); - ell_cols[ell_idx] = invalid_index(); - ell_idx += ell_stride; - } - auto coo_idx = coo_row_ptrs[row]; - for (; col < num_cols; col++) { - const auto val = in_vals[row * in_stride + col]; - if (is_nonzero(val)) { - coo_vals[coo_idx] = val; - coo_cols[coo_idx] = static_cast(col); - coo_rows[coo_idx] = static_cast(row); - coo_idx++; - } - } - }); - }); -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_HYBRID_KERNEL); - - -template -void convert_to_sellp(std::shared_ptr exec, - matrix::view::dense source, - matrix::view::sellp result) -{ - const auto num_rows = result.size[0]; - const auto num_cols = result.size[1]; - const auto stride = source.stride; - const auto in_vals = as_device_type(source.values); - - const auto slice_sets = result.slice_sets; - const auto slice_size = result.slice_size; - auto vals = as_device_type(result.values); - auto col_idxs = result.col_idxs; - - exec->get_queue()->submit([&](sycl::handler& cgh) { - cgh.parallel_for(num_rows, [=](sycl::item<1> item) { - const auto row = static_cast(item[0]); - const auto local_row = row % slice_size; - const auto slice = row / slice_size; - const auto slice_end = slice_sets[slice + 1] * slice_size; - auto out_idx = slice_sets[slice] * slice_size + local_row; - - for (size_type col = 0; col < num_cols; col++) { - const auto val = in_vals[row * stride + col]; - if (is_nonzero(val)) { - col_idxs[out_idx] = static_cast(col); - vals[out_idx] = val; - out_idx += slice_size; - } - } - for (; out_idx < slice_end; out_idx += slice_size) { - col_idxs[out_idx] = invalid_index(); - vals[out_idx] = zero>(); - } - }); - }); -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SELLP_KERNEL); - - -template -void convert_to_sparsity_csr(std::shared_ptr exec, - matrix::view::dense source, - matrix::SparsityCsr* result) -{ - const auto num_rows = result->get_size()[0]; - const auto num_cols = result->get_size()[1]; - const auto in_vals = as_device_type(source.values); - const auto stride = source.stride; - - const auto row_ptrs = result->get_const_row_ptrs(); - auto cols = result->get_col_idxs(); - - exec->get_queue()->submit([&](sycl::handler& cgh) { - cgh.parallel_for(num_rows, [=](sycl::item<1> item) { - const auto row = static_cast(item[0]); - auto write_to = row_ptrs[row]; - - for (size_type col = 0; col < num_cols; col++) { - if (is_nonzero(in_vals[stride * row + col])) { - cols[write_to] = static_cast(col); - write_to++; - } - } - }); - }); -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SPARSITY_CSR_KERNEL); - - template void transpose(std::shared_ptr exec, matrix::view::dense orig, diff --git a/dpcpp/test/preconditioner/jacobi_kernels.dp.cpp b/dpcpp/test/preconditioner/jacobi_kernels.dp.cpp index cc492ac872f..eb8e31a7771 100644 --- a/dpcpp/test/preconditioner/jacobi_kernels.dp.cpp +++ b/dpcpp/test/preconditioner/jacobi_kernels.dp.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include diff --git a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp index 6411a523a6a..28b07e4c14c 100644 --- a/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp +++ b/examples/adaptiveprecision-blockjacobi/adaptiveprecision-blockjacobi.cpp @@ -10,6 +10,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/cb-gmres/cb-gmres.cpp b/examples/cb-gmres/cb-gmres.cpp index d046e5cf6f8..eee0ecdc16a 100644 --- a/examples/cb-gmres/cb-gmres.cpp +++ b/examples/cb-gmres/cb-gmres.cpp @@ -12,6 +12,8 @@ #include +#include + // Helper function which measures the time of `solver->apply(b, x)` in seconds // To get an accurate result, the solve is repeated multiple times (while diff --git a/examples/custom-logger/custom-logger.cpp b/examples/custom-logger/custom-logger.cpp index 9a8c8883151..9c2e9adfbc8 100644 --- a/examples/custom-logger/custom-logger.cpp +++ b/examples/custom-logger/custom-logger.cpp @@ -7,6 +7,8 @@ // This is the main ginkgo header file. #include +#include + // Add the fstream header to read from data from files. #include // Add the map header for storing the executor map. diff --git a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp index 26049168887..b7d8876c269 100644 --- a/examples/custom-stopping-criterion/custom-stopping-criterion.cpp +++ b/examples/custom-stopping-criterion/custom-stopping-criterion.cpp @@ -10,6 +10,8 @@ #include +#include + /** * The ByInteraction class is a criterion which asks for user input to stop diff --git a/examples/file-config-solver/file-config-solver.cpp b/examples/file-config-solver/file-config-solver.cpp index 8115b4172fb..ef7a79e793e 100644 --- a/examples/file-config-solver/file-config-solver.cpp +++ b/examples/file-config-solver/file-config-solver.cpp @@ -11,6 +11,8 @@ #include +#include + // the header in extensions is not shipped with ginkgo.hpp #include diff --git a/examples/ginkgo-overhead/ginkgo-overhead.cpp b/examples/ginkgo-overhead/ginkgo-overhead.cpp index 6a2ed86575c..78e049cddf5 100644 --- a/examples/ginkgo-overhead/ginkgo-overhead.cpp +++ b/examples/ginkgo-overhead/ginkgo-overhead.cpp @@ -8,6 +8,8 @@ #include +#include + [[noreturn]] void print_usage_and_exit(const char* name) { diff --git a/examples/heat-equation/heat-equation.cpp b/examples/heat-equation/heat-equation.cpp index a62ec8d6067..6b955c4c6cc 100644 --- a/examples/heat-equation/heat-equation.cpp +++ b/examples/heat-equation/heat-equation.cpp @@ -45,6 +45,8 @@ setting. #include +#include + // This function implements a simple Ginkgo-themed clamped color mapping for // values in the range [0,5]. diff --git a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp index fbd761a849b..07bf70bbbf9 100644 --- a/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp +++ b/examples/ilu-preconditioned-solver/ilu-preconditioned-solver.cpp @@ -10,6 +10,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/inverse-iteration/inverse-iteration.cpp b/examples/inverse-iteration/inverse-iteration.cpp index d8ee046f0bf..75f14651576 100644 --- a/examples/inverse-iteration/inverse-iteration.cpp +++ b/examples/inverse-iteration/inverse-iteration.cpp @@ -12,6 +12,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/iterative-refinement/iterative-refinement.cpp b/examples/iterative-refinement/iterative-refinement.cpp index 94538bbad83..7c8c7897c93 100644 --- a/examples/iterative-refinement/iterative-refinement.cpp +++ b/examples/iterative-refinement/iterative-refinement.cpp @@ -10,6 +10,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/mixed-multigrid-preconditioned-solver/mixed-multigrid-preconditioned-solver.cpp b/examples/mixed-multigrid-preconditioned-solver/mixed-multigrid-preconditioned-solver.cpp index c9810e81471..3c428618b16 100644 --- a/examples/mixed-multigrid-preconditioned-solver/mixed-multigrid-preconditioned-solver.cpp +++ b/examples/mixed-multigrid-preconditioned-solver/mixed-multigrid-preconditioned-solver.cpp @@ -10,6 +10,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/mixed-multigrid-solver/mixed-multigrid-solver.cpp b/examples/mixed-multigrid-solver/mixed-multigrid-solver.cpp index 2ffbc9b0bb7..a976389c9de 100644 --- a/examples/mixed-multigrid-solver/mixed-multigrid-solver.cpp +++ b/examples/mixed-multigrid-solver/mixed-multigrid-solver.cpp @@ -10,6 +10,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/mixed-precision-ir/mixed-precision-ir.cpp b/examples/mixed-precision-ir/mixed-precision-ir.cpp index 1357518d0d0..88d48750976 100644 --- a/examples/mixed-precision-ir/mixed-precision-ir.cpp +++ b/examples/mixed-precision-ir/mixed-precision-ir.cpp @@ -10,6 +10,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/mixed-spmv/mixed-spmv.cpp b/examples/mixed-spmv/mixed-spmv.cpp index ea6bab32b81..3d9cebd2daf 100644 --- a/examples/mixed-spmv/mixed-spmv.cpp +++ b/examples/mixed-spmv/mixed-spmv.cpp @@ -7,6 +7,8 @@ // This is the main ginkgo header file. #include +#include + // Add the fstream header to read from data from files. #include // Add the C++ iostream header to output information to the console. diff --git a/examples/multigrid-preconditioned-solver-customized/multigrid-preconditioned-solver-customized.cpp b/examples/multigrid-preconditioned-solver-customized/multigrid-preconditioned-solver-customized.cpp index 327f57506f9..f22d26b0dd8 100644 --- a/examples/multigrid-preconditioned-solver-customized/multigrid-preconditioned-solver-customized.cpp +++ b/examples/multigrid-preconditioned-solver-customized/multigrid-preconditioned-solver-customized.cpp @@ -10,6 +10,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/multigrid-preconditioned-solver/multigrid-preconditioned-solver.cpp b/examples/multigrid-preconditioned-solver/multigrid-preconditioned-solver.cpp index a01172541af..1c99a46bf6e 100644 --- a/examples/multigrid-preconditioned-solver/multigrid-preconditioned-solver.cpp +++ b/examples/multigrid-preconditioned-solver/multigrid-preconditioned-solver.cpp @@ -10,6 +10,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/papi-logging/papi-logging.cpp b/examples/papi-logging/papi-logging.cpp index bc969ec50eb..162e31b7dcd 100644 --- a/examples/papi-logging/papi-logging.cpp +++ b/examples/papi-logging/papi-logging.cpp @@ -12,6 +12,8 @@ #include +#include + namespace { diff --git a/examples/par-ilu-convergence/par-ilu-convergence.cpp b/examples/par-ilu-convergence/par-ilu-convergence.cpp index b972931ac0a..e2b188be344 100644 --- a/examples/par-ilu-convergence/par-ilu-convergence.cpp +++ b/examples/par-ilu-convergence/par-ilu-convergence.cpp @@ -11,6 +11,8 @@ #include +#include + const std::map()>> executors{ @@ -72,6 +74,7 @@ int main(int argc, char* argv[]) { using ValueType = double; using IndexType = int; + using Csr = gko::matrix::Csr; // print usage message if (argc < 2 || executors.find(argv[1]) == executors.end()) { @@ -143,9 +146,11 @@ int main(int argc, char* argv[]) gko::as>(factory->generate(mtx)); exec->synchronize(); auto toc = std::chrono::high_resolution_clock::now(); - auto residual = gko::clone(exec, mtx); - result->get_operators()[0]->apply(one, result->get_operators()[1], - minus_one, residual); + auto residual = + gko::as(result->get_operators()[0]) + ->multiply_add(one, + gko::as(result->get_operators()[1]), + minus_one, mtx); times.push_back( std::chrono::duration_cast(toc - tic) .count()); diff --git a/examples/performance-debugging/performance-debugging.cpp b/examples/performance-debugging/performance-debugging.cpp index e6b4e1a7208..533c5c7b544 100644 --- a/examples/performance-debugging/performance-debugging.cpp +++ b/examples/performance-debugging/performance-debugging.cpp @@ -19,6 +19,8 @@ #include +#include + template using vec = gko::matrix::MultiVector; diff --git a/examples/preconditioned-solver/preconditioned-solver.cpp b/examples/preconditioned-solver/preconditioned-solver.cpp index 20603fecf46..cda7cb8a6c3 100644 --- a/examples/preconditioned-solver/preconditioned-solver.cpp +++ b/examples/preconditioned-solver/preconditioned-solver.cpp @@ -9,6 +9,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/reordered-preconditioned-solver/reordered-preconditioned-solver.cpp b/examples/reordered-preconditioned-solver/reordered-preconditioned-solver.cpp index 53434f83811..1791b91e340 100644 --- a/examples/reordered-preconditioned-solver/reordered-preconditioned-solver.cpp +++ b/examples/reordered-preconditioned-solver/reordered-preconditioned-solver.cpp @@ -9,6 +9,8 @@ #include +#include + int main(int argc, char* argv[]) { diff --git a/examples/simple-solver/simple-solver.cpp b/examples/simple-solver/simple-solver.cpp index fae103dd50e..195f6233a70 100644 --- a/examples/simple-solver/simple-solver.cpp +++ b/examples/simple-solver/simple-solver.cpp @@ -7,6 +7,8 @@ // This is the main ginkgo header file. #include +#include + // Add the fstream header to read from data from files. #include // Add the C++ iostream header to output information to the console. diff --git a/extensions/test/kokkos/types.cpp b/extensions/test/kokkos/types.cpp index 1c0c914410e..78fca95ce07 100644 --- a/extensions/test/kokkos/types.cpp +++ b/extensions/test/kokkos/types.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include diff --git a/hip/test/solver/lower_trs_kernels.cpp b/hip/test/solver/lower_trs_kernels.cpp index 137ec766987..e99b34ad626 100644 --- a/hip/test/solver/lower_trs_kernels.cpp +++ b/hip/test/solver/lower_trs_kernels.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -24,13 +25,14 @@ namespace { class LowerTrs : public HipTestFixture { protected: using CsrMtx = gko::matrix::Csr; - using Mtx = gko::matrix::MultiVector<>; + using Mtx = gko::matrix::Dense<>; + using Vec = gko::matrix::MultiVector<>; LowerTrs() : rand_engine(30) {} - std::unique_ptr gen_mtx(int num_rows, int num_cols) + std::unique_ptr gen_vec(int num_rows, int num_cols) { - return gko::test::generate_random_matrix( + return gko::test::generate_random_matrix( num_rows, num_cols, std::uniform_int_distribution<>(num_cols, num_cols), std::normal_distribution<>(-1.0, 1.0), rand_engine, ref); @@ -46,26 +48,26 @@ class LowerTrs : public HipTestFixture { void initialize_data(int m, int n) { mtx = gen_l_mtx(m); - b = gen_mtx(m, n); - x = gen_mtx(m, n); + b = gen_vec(m, n); + x = gen_vec(m, n); csr_mtx = CsrMtx::create(ref); mtx->convert_to(csr_mtx); d_csr_mtx = CsrMtx::create(exec); d_x = gko::clone(exec, x); d_csr_mtx->copy_from(csr_mtx); - b2 = Mtx::create(ref); + b2 = Vec::create(ref); d_b2 = gko::clone(exec, b); b2->copy_from(b); } - std::shared_ptr b; - std::shared_ptr b2; - std::shared_ptr x; + std::shared_ptr b; + std::shared_ptr b2; + std::shared_ptr x; std::shared_ptr mtx; std::shared_ptr csr_mtx; - std::shared_ptr d_b; - std::shared_ptr d_b2; - std::shared_ptr d_x; + std::shared_ptr d_b; + std::shared_ptr d_b2; + std::shared_ptr d_x; std::shared_ptr d_csr_mtx; std::default_random_engine rand_engine; }; diff --git a/hip/test/solver/upper_trs_kernels.cpp b/hip/test/solver/upper_trs_kernels.cpp index ca34c489202..479d4f8dfe0 100644 --- a/hip/test/solver/upper_trs_kernels.cpp +++ b/hip/test/solver/upper_trs_kernels.cpp @@ -24,13 +24,14 @@ namespace { class UpperTrs : public HipTestFixture { protected: using CsrMtx = gko::matrix::Csr; - using Mtx = gko::matrix::MultiVector<>; + using Mtx = gko::matrix::Dense<>; + using Vec = gko::matrix::MultiVector<>; UpperTrs() : rand_engine(30) {} - std::unique_ptr gen_mtx(int num_rows, int num_cols) + std::unique_ptr gen_vec(int num_rows, int num_cols) { - return gko::test::generate_random_matrix( + return gko::test::generate_random_matrix( num_rows, num_cols, std::uniform_int_distribution<>(num_cols, num_cols), std::normal_distribution<>(-1.0, 1.0), rand_engine, ref); @@ -46,26 +47,26 @@ class UpperTrs : public HipTestFixture { void initialize_data(int m, int n) { mtx = gen_u_mtx(m); - b = gen_mtx(m, n); - x = gen_mtx(m, n); + b = gen_vec(m, n); + x = gen_vec(m, n); csr_mtx = CsrMtx::create(ref); mtx->convert_to(csr_mtx); d_csr_mtx = CsrMtx::create(exec); d_x = gko::clone(exec, x); d_csr_mtx->copy_from(csr_mtx); - b2 = Mtx::create(ref); + b2 = Vec::create(ref); d_b2 = gko::clone(exec, b); b2->copy_from(b); } - std::shared_ptr b; - std::shared_ptr b2; - std::shared_ptr x; + std::shared_ptr b; + std::shared_ptr b2; + std::shared_ptr x; std::shared_ptr mtx; std::shared_ptr csr_mtx; - std::shared_ptr d_b; - std::shared_ptr d_b2; - std::shared_ptr d_x; + std::shared_ptr d_b; + std::shared_ptr d_b2; + std::shared_ptr d_x; std::shared_ptr d_csr_mtx; std::default_random_engine rand_engine; }; diff --git a/hip/test/utils/assertions_test.cpp b/hip/test/utils/assertions_test.cpp index a25d2315544..02c8b7f55ed 100644 --- a/hip/test/utils/assertions_test.cpp +++ b/hip/test/utils/assertions_test.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include "hip/test/utils.hip.hpp" @@ -20,7 +21,7 @@ class MatricesNear : public HipTestFixture {}; TEST_F(MatricesNear, CanPassHipMatrix) { - auto mtx = gko::initialize>( + auto mtx = gko::initialize>( {{1.0, 2.0, 3.0}, {0.0, 4.0, 0.0}}, ref); auto csr_ref = gko::matrix::Csr<>::create(ref); csr_ref->copy_from(mtx); diff --git a/include/ginkgo/core/matrix/coo.hpp b/include/ginkgo/core/matrix/coo.hpp index da033091093..0f61f6c1584 100644 --- a/include/ginkgo/core/matrix/coo.hpp +++ b/include/ginkgo/core/matrix/coo.hpp @@ -24,7 +24,7 @@ template class Csr; template -class MultiVector; +class Dense; template class CooBuilder; @@ -58,7 +58,7 @@ class Coo : public LinOp, public ConvertibleTo, IndexType>>, #endif public ConvertibleTo>, - public ConvertibleTo>, + public ConvertibleTo>, public DiagonalExtractable, public ReadableFromMatrixData, public WritableToMatrixData, @@ -67,7 +67,7 @@ class Coo : public LinOp, remove_complex>> { friend class EnableCloneable; friend class Csr; - friend class MultiVector; + friend class Dense; friend class CooBuilder; friend class Coo, IndexType>; friend class Hybrid; @@ -80,8 +80,8 @@ class Coo : public LinOp, using ConvertibleTo, IndexType>>::move_to; using ConvertibleTo>::convert_to; using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; using ReadableFromMatrixData::read; using value_type = ValueType; @@ -129,9 +129,9 @@ class Coo : public LinOp, void move_to(Csr* other) override; - void convert_to(MultiVector* other) const override; + void convert_to(Dense* other) const override; - void move_to(MultiVector* other) override; + void move_to(Dense* other) override; void read(const mat_data& data) override; diff --git a/include/ginkgo/core/matrix/csr.hpp b/include/ginkgo/core/matrix/csr.hpp index 08585dd3e22..db175a401f3 100644 --- a/include/ginkgo/core/matrix/csr.hpp +++ b/include/ginkgo/core/matrix/csr.hpp @@ -21,6 +21,9 @@ namespace matrix { template class MultiVector; +template +class Dense; + template class Diagonal; @@ -148,7 +151,7 @@ class Csr : public LinOp, #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 public ConvertibleTo, IndexType>>, #endif - public ConvertibleTo>, + public ConvertibleTo>, public ConvertibleTo>, public ConvertibleTo>, public ConvertibleTo>, @@ -165,7 +168,7 @@ class Csr : public LinOp, public ScaledIdentityAddable { friend class EnableCloneable; friend class Coo; - friend class MultiVector; + friend class Dense; friend class Diagonal; friend class Ell; friend class Hybrid; @@ -181,8 +184,8 @@ class Csr : public LinOp, using EnableCloneable::move_to; using ConvertibleTo, IndexType>>::convert_to; using ConvertibleTo, IndexType>>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; using ConvertibleTo>::convert_to; using ConvertibleTo>::move_to; using ConvertibleTo>::convert_to; @@ -310,9 +313,9 @@ class Csr : public LinOp, void move_to(Csr, IndexType>* result) override; #endif - void convert_to(MultiVector* other) const override; + void convert_to(Dense* other) const override; - void move_to(MultiVector* other) override; + void move_to(Dense* other) override; void convert_to(Coo* result) const override; diff --git a/include/ginkgo/core/matrix/dense.hpp b/include/ginkgo/core/matrix/dense.hpp new file mode 100644 index 00000000000..43e8f4b7646 --- /dev/null +++ b/include/ginkgo/core/matrix/dense.hpp @@ -0,0 +1,570 @@ +// SPDX-FileCopyrightText: 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#pragma once + + +#include +#include + + +namespace gko { +namespace matrix { + + +template +class MultiVector; + + +template +class Coo; + +template +class Csr; + +template +class Diagonal; + +template +class Ell; + +template +class Fbcsr; + +template +class Hybrid; + +template +class Sellp; + +template +class SparsityCsr; + + +/** + * Dense is a matrix format which explicitly stores all values of the + * matrix. + * + * The values are stored in row-major format (values belonging to the same row + * appear consecutive in the memory). Optionally, rows can be padded for better + * memory access. + * + * @tparam ValueType precision of matrix elements + * + * @ingroup dense + * @ingroup mat_formats + * @ingroup LinOp + */ +template +class Dense : public LinOp, + public EnableCloneable>, + public ConvertibleTo>, + public ConvertibleTo>>, +#if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16 + public ConvertibleTo>>, +#endif +#if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 + public ConvertibleTo>>, +#endif + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public ConvertibleTo>, + public DiagonalExtractable, + public ReadableFromMatrixData, + public ReadableFromMatrixData, + public WritableToMatrixData, + public WritableToMatrixData, + public Transposable, + public ScaledIdentityAddable { + friend class EnableCloneable; + friend class Dense>; + friend class Dense>; + friend class MultiVector; + friend class Coo; + friend class Coo; + friend class Csr; + friend class Csr; + friend class Ell; + friend class Ell; + friend class Fbcsr; + friend class Fbcsr; + friend class Hybrid; + friend class Hybrid; + friend class Sellp; + friend class Sellp; + friend class SparsityCsr; + friend class SparsityCsr; + GKO_ASSERT_SUPPORTED_VALUE_TYPE; + +public: + using EnableCloneable::convert_to; + using EnableCloneable::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; + + using value_type = ValueType; + using index_type = int64; + using transposed_type = Dense; + using mat_data64 = matrix_data; + using mat_data32 = matrix_data; + using device_mat_data64 = device_matrix_data; + using device_mat_data32 = device_matrix_data; + using device_view = view::dense; + using const_device_view = view::dense; + + using row_major_range = gko::range>; + + void convert_to(MultiVector* result) const override; + + void move_to(MultiVector* result) override; + + void convert_to(Dense>* result) const override; + + void move_to(Dense>* result) override; + +#if GINKGO_ENABLE_HALF || GINKGO_ENABLE_BFLOAT16 + friend class Dense>; + using ConvertibleTo>>::convert_to; + using ConvertibleTo>>::move_to; + + void convert_to(Dense>* result) const override; + + void move_to(Dense>* result) override; +#endif + +#if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 + friend class Dense>; + using ConvertibleTo>>::convert_to; + using ConvertibleTo>>::move_to; + + void convert_to(Dense>* result) const override; + + void move_to(Dense>* result) override; +#endif + + void convert_to(Coo* result) const override; + + void move_to(Coo* result) override; + + void convert_to(Coo* result) const override; + + void move_to(Coo* result) override; + + void convert_to(Csr* result) const override; + + void move_to(Csr* result) override; + + void convert_to(Csr* result) const override; + + void move_to(Csr* result) override; + + void convert_to(Ell* result) const override; + + void move_to(Ell* result) override; + + void convert_to(Ell* result) const override; + + void move_to(Ell* result) override; + + void convert_to(Fbcsr* result) const override; + + void move_to(Fbcsr* result) override; + + void convert_to(Fbcsr* result) const override; + + void move_to(Fbcsr* result) override; + + void convert_to(Hybrid* result) const override; + + void move_to(Hybrid* result) override; + + void convert_to(Hybrid* result) const override; + + void move_to(Hybrid* result) override; + + void convert_to(Sellp* result) const override; + + void move_to(Sellp* result) override; + + void convert_to(Sellp* result) const override; + + void move_to(Sellp* result) override; + + void convert_to(SparsityCsr* result) const override; + + void move_to(SparsityCsr* result) override; + + void convert_to(SparsityCsr* result) const override; + + void move_to(SparsityCsr* result) override; + + void read(const mat_data32& data) override; + + void read(const mat_data64& data) override; + + void read(const device_mat_data32& data) override; + + void read(const device_mat_data64& data) override; + + void read(device_mat_data32&& data) override; + + void read(device_mat_data64&& data) override; + + void write(mat_data32& data) const override; + + void write(mat_data64& data) const override; + + void fill(ValueType value); + + /** + * Writes the diagonal of this matrix into an existing diagonal matrix. + * + * @param output The output matrix. Its size must match the size of this + * matrix's diagonal. + * @see Dense::extract_diagonal() + */ + void extract_diagonal(ptr_param> output) const; + + std::unique_ptr> extract_diagonal() const override; + + [[nodiscard]] std::unique_ptr transpose() const override; + + [[nodiscard]] std::unique_ptr conj_transpose() const override; + + /** + * Writes the transposed matrix into the given output matrix. + * + * @param output The output matrix. It must have the dimensions + * `gko::transpose(this->get_size())` + */ + void transpose(ptr_param output) const; + + /** + * Writes the conjugate-transposed matrix into the given output matrix. + * + * @param output The output matrix. It must have the dimensions + * `gko::transpose(this->get_size())` + */ + void conj_transpose(ptr_param output) const; + + void add_scaled(ptr_param alpha, + ptr_param> diag); + + void sub_scaled(ptr_param alpha, + ptr_param> diag); + + [[nodiscard]] static std::unique_ptr create( + std::shared_ptr exec, const dim<2>& size = dim<2>{}, + size_type stride = 0); + + [[nodiscard]] static std::unique_ptr create( + std::shared_ptr exec, const dim<2>& size, + array values, size_type stride); + + [[nodiscard]] static std::unique_ptr create_const( + std::shared_ptr exec, const dim<2>& size, + ::gko::detail::const_array_view&& values, size_type stride); + + [[nodiscard]] std::unique_ptr create_subview(span rows, span cols); + + [[nodiscard]] std::unique_ptr create_subview(span rows, + span cols) const; + + [[nodiscard]] std::unique_ptr create_const_subview( + span rows, span cols) const; + + [[nodiscard]] std::unique_ptr> + as_const_multivector_view() const; + + [[nodiscard]] std::unique_ptr> as_multivector_view(); + + [[nodiscard]] device_view get_device_view(); + + [[nodiscard]] const_device_view get_const_device_view() const; + + ValueType* get_values() noexcept { return values_.get_data(); } + + const ValueType* get_const_values() const noexcept + { + return values_.get_const_data(); + } + + ValueType& at(size_type row, size_type col); + + ValueType at(size_type row, size_type col) const; + + [[nodiscard]] size_type get_stride() const noexcept; + + [[nodiscard]] size_type get_num_stored_elements() const noexcept; + + Dense(const Dense& other); + + Dense(Dense&& other); + + Dense& operator=(const Dense& other); + + Dense& operator=(Dense&& other); + +protected: + Dense(std::shared_ptr exec, const dim<2>& size = dim<2>{}, + size_type stride = 0); + + Dense(std::shared_ptr exec, const dim<2>& size, + array values, size_type stride); + + void apply_impl(const LinOp* b, LinOp* x) const override; + + void apply_impl(const LinOp* alpha, const LinOp* b, const LinOp* beta, + LinOp* x) const override; + + [[nodiscard]] size_type linearize_index(size_type row, + size_type col) const noexcept; + + void resize(dim<2> new_size); + + template + void convert_impl(Coo* result) const; + + template + void convert_impl(Csr* result) const; + + template + void convert_impl(Ell* result) const; + + template + void convert_impl(Fbcsr* result) const; + + template + void convert_impl(Hybrid* result) const; + + template + void convert_impl(Sellp* result) const; + + template + void convert_impl(SparsityCsr* result) const; + +private: + size_type stride_; + array values_; + + void add_scaled_identity_impl(const LinOp* a, const LinOp* b) override; +}; + + +} // namespace matrix + + +namespace detail { + + +template +struct temporary_clone_helper> { + static std::unique_ptr> create( + std::shared_ptr exec, matrix::Dense* ptr, + bool copy_data) + { + if (copy_data) { + return gko::clone(std::move(exec), ptr); + } else { + return matrix::Dense::create(exec, ptr->get_size()); + } + } +}; + + +} // namespace detail + + +/** + * Creates and initializes a column-vector. + * + * This function first creates a temporary Dense matrix, fills it with + * passed in values, and then converts the matrix to the requested type. + * + * @tparam Matrix matrix type to initialize + * (Dense has to implement the ConvertibleTo + * interface) + * @tparam TArgs argument types for Matrix::create method + * (not including the implied Executor as the first argument) + * + * @param stride row stride for the temporary Dense matrix + * @param vals values used to initialize the vector + * @param exec Executor associated to the vector + * @param create_args additional arguments passed to Matrix::create, not + * including the Executor, which is passed as the first + * argument + * + * @ingroup LinOp + */ +template +std::unique_ptr initialize( + size_type stride, std::initializer_list vals, + std::shared_ptr exec, TArgs&&... create_args) +{ + using dense = matrix::Dense; + size_type num_rows = vals.size(); + auto tmp = dense::create(exec->get_master(), dim<2>{num_rows, 1}, stride); + size_type idx = 0; + for (const auto& elem : vals) { + tmp->at(idx, 0) = elem; + ++idx; + } + auto mtx = Matrix::create(exec, std::forward(create_args)...); + tmp->move_to(mtx); + return mtx; +} + +/** + * Creates and initializes a column-vector. + * + * This function first creates a temporary Dense matrix, fills it with + * passed in values, and then converts the matrix to the requested type. The + * stride of the intermediate Dense matrix is set to 1. + * + * @tparam Matrix matrix type to initialize + * (Dense has to implement the ConvertibleTo + * interface) + * @tparam TArgs argument types for Matrix::create method + * (not including the implied Executor as the first argument) + * + * @param vals values used to initialize the vector + * @param exec Executor associated to the vector + * @param create_args additional arguments passed to Matrix::create, not + * including the Executor, which is passed as the first + * argument + * + * @ingroup LinOp + */ +template +std::unique_ptr initialize( + std::initializer_list vals, + std::shared_ptr exec, TArgs&&... create_args) +{ + return initialize(1, vals, std::move(exec), + std::forward(create_args)...); +} + + +/** + * Creates and initializes a matrix. + * + * This function first creates a temporary Dense matrix, fills it with + * passed in values, and then converts the matrix to the requested type. + * + * @tparam Matrix matrix type to initialize + * (Dense has to implement the ConvertibleTo + * interface) + * @tparam TArgs argument types for Matrix::create method + * (not including the implied Executor as the first argument) + * + * @param stride row stride for the temporary Dense matrix + * @param vals values used to initialize the matrix + * @param exec Executor associated to the matrix + * @param create_args additional arguments passed to Matrix::create, not + * including the Executor, which is passed as the first + * argument + * + * @ingroup LinOp + */ +template +std::unique_ptr initialize( + size_type stride, + std::initializer_list> + vals, + std::shared_ptr exec, TArgs&&... create_args) +{ + using dense = matrix::Dense; + size_type num_rows = vals.size(); + size_type num_cols = num_rows > 0 ? begin(vals)->size() : 1; + auto tmp = + dense::create(exec->get_master(), dim<2>{num_rows, num_cols}, stride); + size_type ridx = 0; + for (const auto& row : vals) { + size_type cidx = 0; + for (const auto& elem : row) { + tmp->at(ridx, cidx) = elem; + ++cidx; + } + ++ridx; + } + auto mtx = Matrix::create(exec, std::forward(create_args)...); + tmp->move_to(mtx); + return mtx; +} + + +/** + * Creates and initializes a matrix. + * + * This function first creates a temporary Dense matrix, fills it with + * passed in values, and then converts the matrix to the requested type. The + * stride of the intermediate Dense matrix is set to the number of columns + * of the initializer list. + * + * @tparam Matrix matrix type to initialize + * (Dense has to implement the ConvertibleTo + * interface) + * @tparam TArgs argument types for Matrix::create method + * (not including the implied Executor as the first argument) + * + * @param vals values used to initialize the matrix + * @param exec Executor associated to the matrix + * @param create_args additional arguments passed to Matrix::create, not + * including the Executor, which is passed as the first + * argument + * + * @ingroup LinOp + */ +template +std::unique_ptr initialize( + std::initializer_list> + vals, + std::shared_ptr exec, TArgs&&... create_args) +{ + return initialize(vals.size() > 0 ? begin(vals)->size() : 0, vals, + std::move(exec), + std::forward(create_args)...); +} + + +} // namespace gko diff --git a/include/ginkgo/core/matrix/diagonal.hpp b/include/ginkgo/core/matrix/diagonal.hpp index 020e359303c..3faa2c49c80 100644 --- a/include/ginkgo/core/matrix/diagonal.hpp +++ b/include/ginkgo/core/matrix/diagonal.hpp @@ -17,9 +17,6 @@ namespace matrix { template class Csr; -template -class MultiVector; - /** * This class is a utility which efficiently implements the diagonal matrix (a diff --git a/include/ginkgo/core/matrix/ell.hpp b/include/ginkgo/core/matrix/ell.hpp index 9029bb1e96f..f213f8c9946 100644 --- a/include/ginkgo/core/matrix/ell.hpp +++ b/include/ginkgo/core/matrix/ell.hpp @@ -16,7 +16,7 @@ namespace matrix { template -class MultiVector; +class Dense; template class Coo; @@ -59,7 +59,7 @@ class Ell : public LinOp, #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 public ConvertibleTo, IndexType>>, #endif - public ConvertibleTo>, + public ConvertibleTo>, public ConvertibleTo>, public DiagonalExtractable, public ReadableFromMatrixData, @@ -67,7 +67,7 @@ class Ell : public LinOp, public EnableAbsoluteComputation< remove_complex>> { friend class EnableCloneable; - friend class MultiVector; + friend class Dense; friend class Coo; friend class Csr; friend class Ell, IndexType>; @@ -80,8 +80,8 @@ class Ell : public LinOp, using EnableCloneable::move_to; using ConvertibleTo, IndexType>>::convert_to; using ConvertibleTo, IndexType>>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; using ConvertibleTo>::convert_to; using ConvertibleTo>::move_to; using ReadableFromMatrixData::read; @@ -124,9 +124,9 @@ class Ell : public LinOp, void move_to(Ell, IndexType>* result) override; #endif - void convert_to(MultiVector* other) const override; + void convert_to(Dense* other) const override; - void move_to(MultiVector* other) override; + void move_to(Dense* other) override; void convert_to(Csr* other) const override; diff --git a/include/ginkgo/core/matrix/fbcsr.hpp b/include/ginkgo/core/matrix/fbcsr.hpp index f7e2c70e5bf..83ea205ffb5 100644 --- a/include/ginkgo/core/matrix/fbcsr.hpp +++ b/include/ginkgo/core/matrix/fbcsr.hpp @@ -16,7 +16,7 @@ namespace matrix { template -class MultiVector; +class Dense; template class Csr; @@ -106,7 +106,7 @@ class Fbcsr #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 public ConvertibleTo, IndexType>>, #endif - public ConvertibleTo>, + public ConvertibleTo>, public ConvertibleTo>, public ConvertibleTo>, public DiagonalExtractable, @@ -117,7 +117,7 @@ class Fbcsr remove_complex>> { friend class EnableCloneable; friend class Csr; - friend class MultiVector; + friend class Dense; friend class SparsityCsr; friend class FbcsrBuilder; friend class Fbcsr, IndexType>; @@ -147,8 +147,8 @@ class Fbcsr using ConvertibleTo< Fbcsr, IndexType>>::convert_to; using ConvertibleTo, IndexType>>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; using ConvertibleTo>::convert_to; using ConvertibleTo>::move_to; using ConvertibleTo>::convert_to; @@ -189,9 +189,9 @@ class Fbcsr Fbcsr, IndexType>* result) override; #endif - void convert_to(MultiVector* other) const override; + void convert_to(Dense* other) const override; - void move_to(MultiVector* other) override; + void move_to(Dense* other) override; /** * Converts the matrix to CSR format diff --git a/include/ginkgo/core/matrix/hybrid.hpp b/include/ginkgo/core/matrix/hybrid.hpp index 6bc0003bd3f..e47da1786c9 100644 --- a/include/ginkgo/core/matrix/hybrid.hpp +++ b/include/ginkgo/core/matrix/hybrid.hpp @@ -20,7 +20,7 @@ namespace matrix { template -class MultiVector; +class Dense; template class Csr; @@ -49,7 +49,7 @@ class Hybrid #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 public ConvertibleTo, IndexType>>, #endif - public ConvertibleTo>, + public ConvertibleTo>, public ConvertibleTo>, public DiagonalExtractable, public ReadableFromMatrixData, @@ -57,7 +57,7 @@ class Hybrid public EnableAbsoluteComputation< remove_complex>> { friend class EnableCloneable; - friend class MultiVector; + friend class Dense; friend class Csr; friend class Hybrid, IndexType>; GKO_ASSERT_SUPPORTED_VALUE_AND_INDEX_TYPE; @@ -68,8 +68,8 @@ class Hybrid using ConvertibleTo< Hybrid, IndexType>>::convert_to; using ConvertibleTo, IndexType>>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; using ConvertibleTo>::convert_to; using ConvertibleTo>::move_to; using ReadableFromMatrixData::read; @@ -399,9 +399,9 @@ class Hybrid Hybrid, IndexType>* result) override; #endif - void convert_to(MultiVector* other) const override; + void convert_to(Dense* other) const override; - void move_to(MultiVector* other) override; + void move_to(Dense* other) override; void convert_to(Csr* other) const override; diff --git a/include/ginkgo/core/matrix/multivector.hpp b/include/ginkgo/core/matrix/multivector.hpp index fb049aba1de..5b4922a2572 100644 --- a/include/ginkgo/core/matrix/multivector.hpp +++ b/include/ginkgo/core/matrix/multivector.hpp @@ -45,29 +45,8 @@ class VectorCache; namespace matrix { -template -class Coo; - -template -class Csr; - template -class Diagonal; - -template -class Ell; - -template -class Fbcsr; - -template -class Hybrid; - -template -class Sellp; - -template -class SparsityCsr; +class Dense; /** @@ -97,21 +76,7 @@ class MultiVector #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 public ConvertibleTo>>, #endif - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public ConvertibleTo>, - public DiagonalExtractable, + public ConvertibleTo>, public ReadableFromMatrixData, public ReadableFromMatrixData, public WritableToMatrixData, @@ -119,24 +84,8 @@ class MultiVector public Transposable, public Permutable, public Permutable, - public EnableAbsoluteComputation>>, - public ScaledIdentityAddable { - friend class EnableCloneable; - friend class Coo; - friend class Coo; - friend class Csr; - friend class Csr; - friend class Diagonal; - friend class Ell; - friend class Ell; - friend class Fbcsr; - friend class Fbcsr; - friend class Hybrid; - friend class Hybrid; - friend class Sellp; - friend class Sellp; - friend class SparsityCsr; - friend class SparsityCsr; + public EnableAbsoluteComputation>> { + friend class Dense; friend class MultiVector>; friend class EnableCloneable; friend class experimental::distributed::Vector; @@ -148,34 +97,8 @@ class MultiVector using EnableCloneable::move_to; using ConvertibleTo>>::convert_to; using ConvertibleTo>>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; using ReadableFromMatrixData::read; using ReadableFromMatrixData::read; @@ -321,61 +244,9 @@ class MultiVector void move_to(MultiVector>* result) override; #endif - void convert_to(Coo* result) const override; - - void move_to(Coo* result) override; - - void convert_to(Coo* result) const override; - - void move_to(Coo* result) override; - - void convert_to(Csr* result) const override; - - void move_to(Csr* result) override; - - void convert_to(Csr* result) const override; - - void move_to(Csr* result) override; - - void convert_to(Ell* result) const override; - - void move_to(Ell* result) override; - - void convert_to(Ell* result) const override; - - void move_to(Ell* result) override; - - void convert_to(Fbcsr* result) const override; - - void move_to(Fbcsr* result) override; - - void convert_to(Fbcsr* result) const override; - - void move_to(Fbcsr* result) override; - - void convert_to(Hybrid* result) const override; - - void move_to(Hybrid* result) override; - - void convert_to(Hybrid* result) const override; - - void move_to(Hybrid* result) override; + void convert_to(Dense* result) const override; - void convert_to(Sellp* result) const override; - - void move_to(Sellp* result) override; - - void convert_to(Sellp* result) const override; - - void move_to(Sellp* result) override; - - void convert_to(SparsityCsr* result) const override; - - void move_to(SparsityCsr* result) override; - - void convert_to(SparsityCsr* result) const override; - - void move_to(SparsityCsr* result) override; + void move_to(Dense* result) override; void read(const mat_data64& data) override; @@ -810,17 +681,6 @@ class MultiVector void inverse_column_permute(const array* permutation_indices, ptr_param output) const; - std::unique_ptr> extract_diagonal() const override; - - /** - * Writes the diagonal of this matrix into an existing diagonal matrix. - * - * @param output The output matrix. Its size must match the size of this - * matrix's diagonal. - * @see MultiVector::extract_diagonal() - */ - void extract_diagonal(ptr_param> output) const; - std::unique_ptr compute_absolute() const override; /** @@ -1270,6 +1130,11 @@ class MultiVector std::shared_ptr exec, const dim<2>& size, gko::detail::const_array_view&& values, size_type stride); + [[nodiscard]] std::unique_ptr> as_const_dense_view() + const; + + [[nodiscard]] std::unique_ptr> as_dense_view(); + /** * Copy-assigns a MultiVector. Preserves the executor, reallocates * the matrix with minimal stride if the dimensions don't match, then copies @@ -1362,27 +1227,6 @@ class MultiVector this->get_stride()); } - template - void convert_impl(Coo* result) const; - - template - void convert_impl(Csr* result) const; - - template - void convert_impl(Ell* result) const; - - template - void convert_impl(Fbcsr* result) const; - - template - void convert_impl(Hybrid* result) const; - - template - void convert_impl(Sellp* result) const; - - template - void convert_impl(SparsityCsr* result) const; - /** * @copydoc scale(const LinOp *) * @@ -1529,15 +1373,9 @@ class MultiVector private: size_type stride_; array values_; - - void add_scaled_identity_impl(const LinOp* a, const LinOp* b) override; }; -template -using Dense = MultiVector; - - } // namespace matrix @@ -1597,159 +1435,6 @@ make_const_dense_view(VecPtr&& vector) } -/** - * Creates and initializes a column-vector. - * - * This function first creates a temporary MultiVector, fills it with - * passed in values, and then converts the matrix to the requested type. - * - * @tparam Matrix matrix type to initialize - * (MultiVector has to implement the ConvertibleTo - * interface) - * @tparam TArgs argument types for Matrix::create method - * (not including the implied Executor as the first argument) - * - * @param stride row stride for the temporary MultiVector - * @param vals values used to initialize the vector - * @param exec Executor associated to the vector - * @param create_args additional arguments passed to Matrix::create, not - * including the Executor, which is passed as the first - * argument - * - * @ingroup LinOp - */ -template -std::unique_ptr initialize( - size_type stride, std::initializer_list vals, - std::shared_ptr exec, TArgs&&... create_args) -{ - using multi_vector = matrix::MultiVector; - size_type num_rows = vals.size(); - auto tmp = - multi_vector::create(exec->get_master(), dim<2>{num_rows, 1}, stride); - size_type idx = 0; - for (const auto& elem : vals) { - tmp->at(idx) = elem; - ++idx; - } - auto mtx = Matrix::create(exec, std::forward(create_args)...); - tmp->move_to(mtx); - return mtx; -} - -/** - * Creates and initializes a column-vector. - * - * This function first creates a temporary MultiVector, fills it with - * passed in values, and then converts the matrix to the requested type. The - * stride of the intermediate MultiVector is set to 1. - * - * @tparam Matrix matrix type to initialize - * (MultiVector has to implement the ConvertibleTo - * interface) - * @tparam TArgs argument types for Matrix::create method - * (not including the implied Executor as the first argument) - * - * @param vals values used to initialize the vector - * @param exec Executor associated to the vector - * @param create_args additional arguments passed to Matrix::create, not - * including the Executor, which is passed as the first - * argument - * - * @ingroup LinOp - */ -template -std::unique_ptr initialize( - std::initializer_list vals, - std::shared_ptr exec, TArgs&&... create_args) -{ - return initialize(1, vals, std::move(exec), - std::forward(create_args)...); -} - - -/** - * Creates and initializes a matrix. - * - * This function first creates a temporary MultiVector, fills it with - * passed in values, and then converts the matrix to the requested type. - * - * @tparam Matrix matrix type to initialize - * (MultiVector has to implement the ConvertibleTo - * interface) - * @tparam TArgs argument types for Matrix::create method - * (not including the implied Executor as the first argument) - * - * @param stride row stride for the temporary MultiVector - * @param vals values used to initialize the matrix - * @param exec Executor associated to the matrix - * @param create_args additional arguments passed to Matrix::create, not - * including the Executor, which is passed as the first - * argument - * - * @ingroup LinOp - */ -template -std::unique_ptr initialize( - size_type stride, - std::initializer_list> - vals, - std::shared_ptr exec, TArgs&&... create_args) -{ - using multi_vector = matrix::MultiVector; - size_type num_rows = vals.size(); - size_type num_cols = num_rows > 0 ? begin(vals)->size() : 1; - auto tmp = multi_vector::create(exec->get_master(), - dim<2>{num_rows, num_cols}, stride); - size_type ridx = 0; - for (const auto& row : vals) { - size_type cidx = 0; - for (const auto& elem : row) { - tmp->at(ridx, cidx) = elem; - ++cidx; - } - ++ridx; - } - auto mtx = Matrix::create(exec, std::forward(create_args)...); - tmp->move_to(mtx); - return mtx; -} - - -/** - * Creates and initializes a matrix. - * - * This function first creates a temporary MultiVector, fills it with - * passed in values, and then converts the matrix to the requested type. The - * stride of the intermediate MultiVector is set to the number of columns - * of the initializer list. - * - * @tparam Matrix matrix type to initialize - * (MultiVector has to implement the ConvertibleTo - * interface) - * @tparam TArgs argument types for Matrix::create method - * (not including the implied Executor as the first argument) - * - * @param vals values used to initialize the matrix - * @param exec Executor associated to the matrix - * @param create_args additional arguments passed to Matrix::create, not - * including the Executor, which is passed as the first - * argument - * - * @ingroup LinOp - */ -template -std::unique_ptr initialize( - std::initializer_list> - vals, - std::shared_ptr exec, TArgs&&... create_args) -{ - return initialize(vals.size() > 0 ? begin(vals)->size() : 0, vals, - std::move(exec), - std::forward(create_args)...); -} - - } // namespace gko diff --git a/include/ginkgo/core/matrix/sellp.hpp b/include/ginkgo/core/matrix/sellp.hpp index ace6bc252a3..52dd077caaf 100644 --- a/include/ginkgo/core/matrix/sellp.hpp +++ b/include/ginkgo/core/matrix/sellp.hpp @@ -20,7 +20,7 @@ constexpr int default_stride_factor = 1; template -class MultiVector; +class Dense; template class Csr; @@ -51,7 +51,7 @@ class Sellp #if GINKGO_ENABLE_HALF && GINKGO_ENABLE_BFLOAT16 public ConvertibleTo, IndexType>>, #endif - public ConvertibleTo>, + public ConvertibleTo>, public ConvertibleTo>, public DiagonalExtractable, public ReadableFromMatrixData, @@ -59,7 +59,7 @@ class Sellp public EnableAbsoluteComputation< remove_complex>> { friend class EnableCloneable; - friend class MultiVector; + friend class Dense; friend class Csr; friend class Sellp, IndexType>; GKO_ASSERT_SUPPORTED_VALUE_AND_INDEX_TYPE; @@ -70,8 +70,8 @@ class Sellp using ConvertibleTo< Sellp, IndexType>>::convert_to; using ConvertibleTo, IndexType>>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; using ConvertibleTo>::convert_to; using ConvertibleTo>::move_to; using ReadableFromMatrixData::read; @@ -120,9 +120,9 @@ class Sellp Sellp, IndexType>* result) override; #endif - void convert_to(MultiVector* other) const override; + void convert_to(Dense* other) const override; - void move_to(MultiVector* other) override; + void move_to(Dense* other) override; void convert_to(Csr* other) const override; diff --git a/include/ginkgo/core/matrix/sparsity_csr.hpp b/include/ginkgo/core/matrix/sparsity_csr.hpp index b61d65dfc35..36bfa0cbdb7 100644 --- a/include/ginkgo/core/matrix/sparsity_csr.hpp +++ b/include/ginkgo/core/matrix/sparsity_csr.hpp @@ -22,7 +22,7 @@ class Csr; template -class MultiVector; +class Dense; template @@ -51,13 +51,13 @@ template class SparsityCsr : public LinOp, public EnableCloneable>, public ConvertibleTo>, - public ConvertibleTo>, + public ConvertibleTo>, public ReadableFromMatrixData, public WritableToMatrixData, public Transposable { friend class EnableCloneable; friend class Csr; - friend class MultiVector; + friend class Dense; friend class Fbcsr; GKO_ASSERT_SUPPORTED_VALUE_AND_INDEX_TYPE; @@ -66,8 +66,8 @@ class SparsityCsr : public LinOp, using EnableCloneable::move_to; using ConvertibleTo>::convert_to; using ConvertibleTo>::move_to; - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; using ReadableFromMatrixData::read; using value_type = ValueType; @@ -80,9 +80,9 @@ class SparsityCsr : public LinOp, void move_to(Csr* result) override; - void convert_to(MultiVector* result) const override; + void convert_to(Dense* result) const override; - void move_to(MultiVector* result) override; + void move_to(Dense* result) override; void read(const mat_data& data) override; diff --git a/include/ginkgo/core/preconditioner/jacobi.hpp b/include/ginkgo/core/preconditioner/jacobi.hpp index 635e6979a9f..0b24558abd3 100644 --- a/include/ginkgo/core/preconditioner/jacobi.hpp +++ b/include/ginkgo/core/preconditioner/jacobi.hpp @@ -186,15 +186,15 @@ struct block_interleaved_storage_scheme { template class Jacobi : public LinOp, public EnableCloneable>, - public ConvertibleTo>, + public ConvertibleTo>, public WritableToMatrixData, public Transposable { friend class EnableCloneable>; GKO_ASSERT_SUPPORTED_VALUE_AND_INDEX_TYPE; public: - using ConvertibleTo>::convert_to; - using ConvertibleTo>::move_to; + using ConvertibleTo>::convert_to; + using ConvertibleTo>::move_to; using value_type = ValueType; using index_type = IndexType; using mat_data = matrix_data; @@ -264,9 +264,9 @@ class Jacobi : public LinOp, return blocks_.get_size(); } - void convert_to(matrix::MultiVector* result) const override; + void convert_to(matrix::Dense* result) const override; - void move_to(matrix::MultiVector* result) override; + void move_to(matrix::Dense* result) override; void write(mat_data& data) const override; diff --git a/include/ginkgo/core/solver/ir.hpp b/include/ginkgo/core/solver/ir.hpp index f83c76757bc..a76a3d247cb 100644 --- a/include/ginkgo/core/solver/ir.hpp +++ b/include/ginkgo/core/solver/ir.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include diff --git a/omp/CMakeLists.txt b/omp/CMakeLists.txt index cf1ea2c1e7b..d3ea77bf786 100644 --- a/omp/CMakeLists.txt +++ b/omp/CMakeLists.txt @@ -38,6 +38,7 @@ target_sources( matrix/batch_ell_kernels.cpp matrix/coo_kernels.cpp matrix/csr_kernels.cpp + matrix/dense_kernels.cpp matrix/diagonal_kernels.cpp matrix/ell_kernels.cpp matrix/fbcsr_kernels.cpp diff --git a/omp/matrix/dense_kernels.cpp b/omp/matrix/dense_kernels.cpp new file mode 100644 index 00000000000..422cbd7737e --- /dev/null +++ b/omp/matrix/dense_kernels.cpp @@ -0,0 +1,375 @@ +// SPDX-FileCopyrightText: 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include "core/matrix/dense_kernels.hpp" + +#include "accessor/block_col_major.hpp" + +namespace gko { +namespace kernels { +namespace omp { +namespace dense { + + +template +void simple_apply(std::shared_ptr exec, + matrix::view::dense a, + matrix::view::dense b, + matrix::view::dense c) +{ +#pragma omp parallel for + for (size_type row = 0; row < c.size[0]; ++row) { + for (size_type col = 0; col < c.size[1]; ++col) { + c(row, col) = zero(); + } + } + +#pragma omp parallel for + for (size_type row = 0; row < c.size[0]; ++row) { + for (size_type inner = 0; inner < a.size[1]; ++inner) { + for (size_type col = 0; col < c.size[1]; ++col) { + c(row, col) += a(row, inner) * b(inner, col); + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL); + + +template +void apply(std::shared_ptr exec, + matrix::view::dense alpha, + matrix::view::dense a, + matrix::view::dense b, + matrix::view::dense beta, + matrix::view::dense c) +{ + if (is_nonzero(beta(0, 0))) { +#pragma omp parallel for + for (size_type row = 0; row < c.size[0]; ++row) { + for (size_type col = 0; col < c.size[1]; ++col) { + c(row, col) *= beta(0, 0); + } + } + } else { +#pragma omp parallel for + for (size_type row = 0; row < c.size[0]; ++row) { + for (size_type col = 0; col < c.size[1]; ++col) { + c(row, col) = zero(); + } + } + } + +#pragma omp parallel for + for (size_type row = 0; row < c.size[0]; ++row) { + for (size_type inner = 0; inner < a.size[1]; ++inner) { + for (size_type col = 0; col < c.size[1]; ++col) { + c(row, col) += alpha(0, 0) * a(row, inner) * b(inner, col); + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL); + + +template +void convert_to_coo(std::shared_ptr exec, + matrix::view::dense source, + const int64* row_ptrs, + matrix::view::coo result) +{ + auto num_rows = result.size[0]; + auto num_cols = result.size[1]; + auto num_nonzeros = result.num_stored_elements; + + auto row_idxs = result.row_idxs; + auto col_idxs = result.col_idxs; + auto values = result.values; + +#pragma omp parallel for + for (size_type row = 0; row < num_rows; ++row) { + auto idxs = row_ptrs[row]; + for (size_type col = 0; col < num_cols; ++col) { + auto val = source(row, col); + if (is_nonzero(val)) { + row_idxs[idxs] = row; + col_idxs[idxs] = col; + values[idxs] = val; + ++idxs; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL); + + +template +void convert_to_csr(std::shared_ptr exec, + matrix::view::dense source, + matrix::Csr* result) +{ + auto num_rows = result->get_size()[0]; + auto num_cols = result->get_size()[1]; + auto num_nonzeros = result->get_num_stored_elements(); + + auto row_ptrs = result->get_row_ptrs(); + auto col_idxs = result->get_col_idxs(); + auto values = result->get_values(); + +#pragma omp parallel for + for (size_type row = 0; row < num_rows; ++row) { + auto cur_ptr = row_ptrs[row]; + for (size_type col = 0; col < num_cols; ++col) { + auto val = source(row, col); + if (is_nonzero(val)) { + col_idxs[cur_ptr] = col; + values[cur_ptr] = val; + ++cur_ptr; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL); + + +template +void convert_to_ell(std::shared_ptr exec, + matrix::view::dense source, + matrix::view::ell result) +{ + auto num_rows = result.size[0]; + auto num_cols = result.size[1]; + auto max_nnz_per_row = result.num_stored_elements_per_row; +#pragma omp parallel for + for (size_type i = 0; i < max_nnz_per_row; i++) { + for (size_type j = 0; j < num_rows; j++) { + result.val_at(j, i) = zero(); + result.col_at(j, i) = invalid_index(); + } + } +#pragma omp parallel for + for (size_type row = 0; row < num_rows; row++) { + size_type col_idx = 0; + for (size_type col = 0; col < num_cols; col++) { + auto val = source(row, col); + if (is_nonzero(val)) { + result.val_at(row, col_idx) = val; + result.col_at(row, col_idx) = col; + col_idx++; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL); + + +template +void convert_to_fbcsr(std::shared_ptr exec, + matrix::view::dense source, + matrix::Fbcsr* result) +{ + const auto num_rows = source.size[0]; + const auto num_cols = source.size[1]; + const auto bs = result->get_block_size(); + const auto nzbs = result->get_num_stored_blocks(); + const auto num_block_rows = num_rows / bs; + const auto num_block_cols = num_cols / bs; + acc::range> blocks( + std::array{static_cast(nzbs), + static_cast(bs), + static_cast(bs)}, + result->get_values()); + auto col_idxs = result->get_col_idxs(); +#pragma omp parallel for + for (size_type brow = 0; brow < num_block_rows; ++brow) { + auto block = result->get_const_row_ptrs()[brow]; + for (size_type bcol = 0; bcol < num_block_cols; ++bcol) { + bool block_nz = false; + for (int lrow = 0; lrow < bs; ++lrow) { + for (int lcol = 0; lcol < bs; ++lcol) { + const auto row = lrow + bs * brow; + const auto col = lcol + bs * bcol; + block_nz = block_nz || is_nonzero(source(row, col)); + } + } + if (block_nz) { + col_idxs[block] = bcol; + for (int lrow = 0; lrow < bs; ++lrow) { + for (int lcol = 0; lcol < bs; ++lcol) { + const auto row = lrow + bs * brow; + const auto col = lcol + bs * bcol; + blocks(block, lrow, lcol) = source(row, col); + } + } + block++; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_FBCSR_KERNEL); + + +template +void convert_to_hybrid(std::shared_ptr exec, + matrix::view::dense source, + const int64* coo_row_ptrs, + matrix::view::hybrid result) +{ + auto num_rows = result.size[0]; + auto num_cols = result.size[1]; + auto ell_lim = result.ell_part.num_stored_elements_per_row; + auto coo_val = result.coo_part.values; + auto coo_col = result.coo_part.col_idxs; + auto coo_row = result.coo_part.row_idxs; + +#pragma omp parallel for + for (size_type row = 0; row < num_rows; row++) { + size_type ell_count = 0; + size_type col = 0; + for (; col < num_cols && ell_count < ell_lim; col++) { + auto val = source(row, col); + if (is_nonzero(val)) { + result.ell_part.val_at(row, ell_count) = val; + result.ell_part.col_at(row, ell_count) = col; + ell_count++; + } + } + for (; ell_count < ell_lim; ell_count++) { + result.ell_part.val_at(row, ell_count) = zero(); + result.ell_part.col_at(row, ell_count) = invalid_index(); + } + auto coo_idx = coo_row_ptrs[row]; + for (; col < num_cols; col++) { + auto val = source(row, col); + if (is_nonzero(val)) { + coo_val[coo_idx] = val; + coo_col[coo_idx] = col; + coo_row[coo_idx] = row; + coo_idx++; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL); + + +template +void convert_to_sellp(std::shared_ptr exec, + matrix::view::dense source, + matrix::view::sellp result) +{ + const auto num_rows = result.size[0]; + const auto num_cols = result.size[1]; + const auto vals = result.values; + const auto col_idxs = result.col_idxs; + const auto slice_sets = result.slice_sets; + const auto slice_size = result.slice_size; + const auto num_slices = ceildiv(num_rows, slice_size); +#pragma omp parallel for + for (size_type slice = 0; slice < num_slices; slice++) { + for (size_type local_row = 0; local_row < slice_size; local_row++) { + const auto row = slice * slice_size + local_row; + if (row >= num_rows) { + break; + } + auto sellp_idx = slice_sets[slice] * slice_size + local_row; + const auto sellp_end = + slice_sets[slice + 1] * slice_size + local_row; + for (size_type col = 0; col < num_cols; col++) { + auto val = source(row, col); + if (is_nonzero(val)) { + col_idxs[sellp_idx] = col; + vals[sellp_idx] = val; + sellp_idx += slice_size; + } + } + for (; sellp_idx < sellp_end; sellp_idx += slice_size) { + col_idxs[sellp_idx] = invalid_index(); + vals[sellp_idx] = zero(); + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL); + + +template +void convert_to_sparsity_csr(std::shared_ptr exec, + matrix::view::dense source, + matrix::SparsityCsr* result) +{ + auto num_rows = result->get_size()[0]; + auto num_cols = result->get_size()[1]; + + auto row_ptrs = result->get_row_ptrs(); + auto col_idxs = result->get_col_idxs(); + auto value = result->get_value(); + value[0] = one(); + +#pragma omp parallel for + for (size_type row = 0; row < num_rows; ++row) { + auto cur_ptr = row_ptrs[row]; + for (size_type col = 0; col < num_cols; ++col) { + auto val = source(row, col); + if (is_nonzero(val)) { + col_idxs[cur_ptr] = col; + ++cur_ptr; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL); + + +template +void count_nonzero_blocks_per_row(std::shared_ptr exec, + matrix::view::dense source, + int bs, IndexType* result) +{ + const auto num_rows = source.size[0]; + const auto num_cols = source.size[1]; + const auto num_block_rows = num_rows / bs; + const auto num_block_cols = num_cols / bs; +#pragma omp parallel for + for (size_type brow = 0; brow < num_block_rows; ++brow) { + IndexType num_nonzero_blocks{}; + for (size_type bcol = 0; bcol < num_block_cols; ++bcol) { + bool block_nz = false; + for (int lrow = 0; lrow < bs; ++lrow) { + for (int lcol = 0; lcol < bs; ++lcol) { + const auto row = lrow + bs * brow; + const auto col = lcol + bs * bcol; + block_nz = block_nz || is_nonzero(source(row, col)); + } + } + num_nonzero_blocks += block_nz ? 1 : 0; + } + result[brow] = num_nonzero_blocks; + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL); + + +} // namespace dense +} // namespace omp +} // namespace kernels +} // namespace gko diff --git a/omp/matrix/multivector_kernels.cpp b/omp/matrix/multivector_kernels.cpp index ffb1d238594..24ee29e5a1b 100644 --- a/omp/matrix/multivector_kernels.cpp +++ b/omp/matrix/multivector_kernels.cpp @@ -10,16 +10,6 @@ #include #include -#include -#include -#include -#include -#include -#include - -#include "accessor/block_col_major.hpp" -#include "accessor/range.hpp" -#include "core/components/prefix_sum_kernels.hpp" namespace gko { @@ -75,333 +65,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( GKO_DECLARE_MULTIVECTOR_COMPUTE_NORM2_DISPATCH_KERNEL); -template -void simple_apply(std::shared_ptr exec, - matrix::view::dense a, - matrix::view::dense b, - matrix::view::dense c) -{ -#pragma omp parallel for - for (size_type row = 0; row < c.size[0]; ++row) { - for (size_type col = 0; col < c.size[1]; ++col) { - c(row, col) = zero(); - } - } - -#pragma omp parallel for - for (size_type row = 0; row < c.size[0]; ++row) { - for (size_type inner = 0; inner < a.size[1]; ++inner) { - for (size_type col = 0; col < c.size[1]; ++col) { - c(row, col) += a(row, inner) * b(inner, col); - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_SIMPLE_APPLY_KERNEL); - - -template -void apply(std::shared_ptr exec, - matrix::view::dense alpha, - matrix::view::dense a, - matrix::view::dense b, - matrix::view::dense beta, - matrix::view::dense c) -{ - if (is_nonzero(beta(0, 0))) { -#pragma omp parallel for - for (size_type row = 0; row < c.size[0]; ++row) { - for (size_type col = 0; col < c.size[1]; ++col) { - c(row, col) *= beta(0, 0); - } - } - } else { -#pragma omp parallel for - for (size_type row = 0; row < c.size[0]; ++row) { - for (size_type col = 0; col < c.size[1]; ++col) { - c(row, col) = zero(); - } - } - } - -#pragma omp parallel for - for (size_type row = 0; row < c.size[0]; ++row) { - for (size_type inner = 0; inner < a.size[1]; ++inner) { - for (size_type col = 0; col < c.size[1]; ++col) { - c(row, col) += alpha(0, 0) * a(row, inner) * b(inner, col); - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_APPLY_KERNEL); - - -template -void convert_to_coo(std::shared_ptr exec, - matrix::view::dense source, - const int64* row_ptrs, - matrix::view::coo result) -{ - auto num_rows = result.size[0]; - auto num_cols = result.size[1]; - auto num_nonzeros = result.num_stored_elements; - - auto row_idxs = result.row_idxs; - auto col_idxs = result.col_idxs; - auto values = result.values; - -#pragma omp parallel for - for (size_type row = 0; row < num_rows; ++row) { - auto idxs = row_ptrs[row]; - for (size_type col = 0; col < num_cols; ++col) { - auto val = source(row, col); - if (is_nonzero(val)) { - row_idxs[idxs] = row; - col_idxs[idxs] = col; - values[idxs] = val; - ++idxs; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_COO_KERNEL); - - -template -void convert_to_csr(std::shared_ptr exec, - matrix::view::dense source, - matrix::Csr* result) -{ - auto num_rows = result->get_size()[0]; - auto num_cols = result->get_size()[1]; - auto num_nonzeros = result->get_num_stored_elements(); - - auto row_ptrs = result->get_row_ptrs(); - auto col_idxs = result->get_col_idxs(); - auto values = result->get_values(); - -#pragma omp parallel for - for (size_type row = 0; row < num_rows; ++row) { - auto cur_ptr = row_ptrs[row]; - for (size_type col = 0; col < num_cols; ++col) { - auto val = source(row, col); - if (is_nonzero(val)) { - col_idxs[cur_ptr] = col; - values[cur_ptr] = val; - ++cur_ptr; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_CSR_KERNEL); - - -template -void convert_to_ell(std::shared_ptr exec, - matrix::view::dense source, - matrix::view::ell result) -{ - auto num_rows = result.size[0]; - auto num_cols = result.size[1]; - auto max_nnz_per_row = result.num_stored_elements_per_row; -#pragma omp parallel for - for (size_type i = 0; i < max_nnz_per_row; i++) { - for (size_type j = 0; j < num_rows; j++) { - result.val_at(j, i) = zero(); - result.col_at(j, i) = invalid_index(); - } - } -#pragma omp parallel for - for (size_type row = 0; row < num_rows; row++) { - size_type col_idx = 0; - for (size_type col = 0; col < num_cols; col++) { - auto val = source(row, col); - if (is_nonzero(val)) { - result.val_at(row, col_idx) = val; - result.col_at(row, col_idx) = col; - col_idx++; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_ELL_KERNEL); - - -template -void convert_to_fbcsr(std::shared_ptr exec, - matrix::view::dense source, - matrix::Fbcsr* result) -{ - const auto num_rows = source.size[0]; - const auto num_cols = source.size[1]; - const auto bs = result->get_block_size(); - const auto nzbs = result->get_num_stored_blocks(); - const auto num_block_rows = num_rows / bs; - const auto num_block_cols = num_cols / bs; - acc::range> blocks( - std::array{static_cast(nzbs), - static_cast(bs), - static_cast(bs)}, - result->get_values()); - auto col_idxs = result->get_col_idxs(); -#pragma omp parallel for - for (size_type brow = 0; brow < num_block_rows; ++brow) { - auto block = result->get_const_row_ptrs()[brow]; - for (size_type bcol = 0; bcol < num_block_cols; ++bcol) { - bool block_nz = false; - for (int lrow = 0; lrow < bs; ++lrow) { - for (int lcol = 0; lcol < bs; ++lcol) { - const auto row = lrow + bs * brow; - const auto col = lcol + bs * bcol; - block_nz = block_nz || is_nonzero(source(row, col)); - } - } - if (block_nz) { - col_idxs[block] = bcol; - for (int lrow = 0; lrow < bs; ++lrow) { - for (int lcol = 0; lcol < bs; ++lcol) { - const auto row = lrow + bs * brow; - const auto col = lcol + bs * bcol; - blocks(block, lrow, lcol) = source(row, col); - } - } - block++; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_FBCSR_KERNEL); - - -template -void convert_to_hybrid(std::shared_ptr exec, - matrix::view::dense source, - const int64* coo_row_ptrs, - matrix::view::hybrid result) -{ - auto num_rows = result.size[0]; - auto num_cols = result.size[1]; - auto ell_lim = result.ell_part.num_stored_elements_per_row; - auto coo_val = result.coo_part.values; - auto coo_col = result.coo_part.col_idxs; - auto coo_row = result.coo_part.row_idxs; - -#pragma omp parallel for - for (size_type row = 0; row < num_rows; row++) { - size_type ell_count = 0; - size_type col = 0; - for (; col < num_cols && ell_count < ell_lim; col++) { - auto val = source(row, col); - if (is_nonzero(val)) { - result.ell_part.val_at(row, ell_count) = val; - result.ell_part.col_at(row, ell_count) = col; - ell_count++; - } - } - for (; ell_count < ell_lim; ell_count++) { - result.ell_part.val_at(row, ell_count) = zero(); - result.ell_part.col_at(row, ell_count) = invalid_index(); - } - auto coo_idx = coo_row_ptrs[row]; - for (; col < num_cols; col++) { - auto val = source(row, col); - if (is_nonzero(val)) { - coo_val[coo_idx] = val; - coo_col[coo_idx] = col; - coo_row[coo_idx] = row; - coo_idx++; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_HYBRID_KERNEL); - - -template -void convert_to_sellp(std::shared_ptr exec, - matrix::view::dense source, - matrix::view::sellp result) -{ - const auto num_rows = result.size[0]; - const auto num_cols = result.size[1]; - const auto vals = result.values; - const auto col_idxs = result.col_idxs; - const auto slice_sets = result.slice_sets; - const auto slice_size = result.slice_size; - const auto num_slices = ceildiv(num_rows, slice_size); -#pragma omp parallel for - for (size_type slice = 0; slice < num_slices; slice++) { - for (size_type local_row = 0; local_row < slice_size; local_row++) { - const auto row = slice * slice_size + local_row; - if (row >= num_rows) { - break; - } - auto sellp_idx = slice_sets[slice] * slice_size + local_row; - const auto sellp_end = - slice_sets[slice + 1] * slice_size + local_row; - for (size_type col = 0; col < num_cols; col++) { - auto val = source(row, col); - if (is_nonzero(val)) { - col_idxs[sellp_idx] = col; - vals[sellp_idx] = val; - sellp_idx += slice_size; - } - } - for (; sellp_idx < sellp_end; sellp_idx += slice_size) { - col_idxs[sellp_idx] = invalid_index(); - vals[sellp_idx] = zero(); - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SELLP_KERNEL); - - -template -void convert_to_sparsity_csr(std::shared_ptr exec, - matrix::view::dense source, - matrix::SparsityCsr* result) -{ - auto num_rows = result->get_size()[0]; - auto num_cols = result->get_size()[1]; - - auto row_ptrs = result->get_row_ptrs(); - auto col_idxs = result->get_col_idxs(); - auto value = result->get_value(); - value[0] = one(); - -#pragma omp parallel for - for (size_type row = 0; row < num_rows; ++row) { - auto cur_ptr = row_ptrs[row]; - for (size_type col = 0; col < num_cols; ++col) { - auto val = source(row, col); - if (is_nonzero(val)) { - col_idxs[cur_ptr] = col; - ++cur_ptr; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SPARSITY_CSR_KERNEL); - - template void transpose(std::shared_ptr exec, matrix::view::dense orig, @@ -435,37 +98,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( GKO_DECLARE_MULTIVECTOR_CONJ_TRANSPOSE_KERNEL); -template -void count_nonzero_blocks_per_row(std::shared_ptr exec, - matrix::view::dense source, - int bs, IndexType* result) -{ - const auto num_rows = source.size[0]; - const auto num_cols = source.size[1]; - const auto num_block_rows = num_rows / bs; - const auto num_block_cols = num_cols / bs; -#pragma omp parallel for - for (size_type brow = 0; brow < num_block_rows; ++brow) { - IndexType num_nonzero_blocks{}; - for (size_type bcol = 0; bcol < num_block_cols; ++bcol) { - bool block_nz = false; - for (int lrow = 0; lrow < bs; ++lrow) { - for (int lcol = 0; lcol < bs; ++lcol) { - const auto row = lrow + bs * brow; - const auto col = lcol + bs * bcol; - block_nz = block_nz || is_nonzero(source(row, col)); - } - } - num_nonzero_blocks += block_nz ? 1 : 0; - } - result[brow] = num_nonzero_blocks; - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL); - - } // namespace multivector } // namespace omp } // namespace kernels diff --git a/omp/test/matrix/fbcsr_kernels.cpp b/omp/test/matrix/fbcsr_kernels.cpp index 0c7c6093b5a..9885341a0fe 100644 --- a/omp/test/matrix/fbcsr_kernels.cpp +++ b/omp/test/matrix/fbcsr_kernels.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include diff --git a/reference/CMakeLists.txt b/reference/CMakeLists.txt index db2b88a2062..cb009dd229c 100644 --- a/reference/CMakeLists.txt +++ b/reference/CMakeLists.txt @@ -36,6 +36,7 @@ target_sources( matrix/batch_ell_kernels.cpp matrix/coo_kernels.cpp matrix/csr_kernels.cpp + matrix/dense_kernels.cpp matrix/diagonal_kernels.cpp matrix/ell_kernels.cpp matrix/fbcsr_kernels.cpp diff --git a/reference/matrix/dense_kernels.cpp b/reference/matrix/dense_kernels.cpp new file mode 100644 index 00000000000..24cd2b0b27a --- /dev/null +++ b/reference/matrix/dense_kernels.cpp @@ -0,0 +1,509 @@ +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include "core/matrix/dense_kernels.hpp" + +#include "accessor/block_col_major.hpp" +#include "core/components/prefix_sum_kernels.hpp" + +namespace gko { +namespace kernels { +namespace reference { +namespace dense { + + +template +void simple_apply(std::shared_ptr exec, + matrix::view::dense a, + matrix::view::dense b, + matrix::view::dense c) +{ + for (size_type row = 0; row < c.size[0]; ++row) { + for (size_type col = 0; col < c.size[1]; ++col) { + c(row, col) = zero(); + } + } + + for (size_type row = 0; row < c.size[0]; ++row) { + for (size_type inner = 0; inner < a.size[1]; ++inner) { + for (size_type col = 0; col < c.size[1]; ++col) { + c(row, col) += a(row, inner) * b(inner, col); + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SIMPLE_APPLY_KERNEL); + + +template +void apply(std::shared_ptr exec, + matrix::view::dense alpha, + matrix::view::dense a, + matrix::view::dense b, + matrix::view::dense beta, + matrix::view::dense c) +{ + if (is_nonzero(beta(0, 0))) { + for (size_type row = 0; row < c.size[0]; ++row) { + for (size_type col = 0; col < c.size[1]; ++col) { + c(row, col) *= beta(0, 0); + } + } + } else { + for (size_type row = 0; row < c.size[0]; ++row) { + for (size_type col = 0; col < c.size[1]; ++col) { + c(row, col) = zero(); + } + } + } + + for (size_type row = 0; row < c.size[0]; ++row) { + for (size_type inner = 0; inner < a.size[1]; ++inner) { + for (size_type col = 0; col < c.size[1]; ++col) { + c(row, col) += alpha(0, 0) * a(row, inner) * b(inner, col); + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_APPLY_KERNEL); + + +template +void convert_to_coo(std::shared_ptr exec, + matrix::view::dense source, const int64*, + matrix::view::coo result) +{ + auto num_rows = result.size[0]; + auto num_cols = result.size[1]; + auto num_nonzeros = result.num_stored_elements; + + auto row_idxs = result.row_idxs; + auto col_idxs = result.col_idxs; + auto values = result.values; + + size_type idxs = 0; + for (size_type row = 0; row < num_rows; ++row) { + for (size_type col = 0; col < num_cols; ++col) { + auto val = source(row, col); + if (is_nonzero(val)) { + row_idxs[idxs] = row; + col_idxs[idxs] = col; + values[idxs] = val; + ++idxs; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_COO_KERNEL); + + +template +void convert_to_csr(std::shared_ptr exec, + matrix::view::dense source, + matrix::Csr* result) +{ + auto num_rows = result->get_size()[0]; + auto num_cols = result->get_size()[1]; + auto num_nonzeros = result->get_num_stored_elements(); + + auto row_ptrs = result->get_row_ptrs(); + auto col_idxs = result->get_col_idxs(); + auto values = result->get_values(); + + size_type cur_ptr = 0; + row_ptrs[0] = cur_ptr; + for (size_type row = 0; row < num_rows; ++row) { + for (size_type col = 0; col < num_cols; ++col) { + auto val = source(row, col); + if (is_nonzero(val)) { + col_idxs[cur_ptr] = col; + values[cur_ptr] = val; + ++cur_ptr; + } + } + row_ptrs[row + 1] = cur_ptr; + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_CSR_KERNEL); + + +template +void convert_to_ell(std::shared_ptr exec, + matrix::view::dense source, + matrix::view::ell result) +{ + auto num_rows = result.size[0]; + auto num_cols = result.size[1]; + auto max_nnz_per_row = result.num_stored_elements_per_row; + for (size_type i = 0; i < max_nnz_per_row; i++) { + for (size_type j = 0; j < num_rows; j++) { + result.val_at(j, i) = zero(); + result.col_at(j, i) = invalid_index(); + } + } + size_type col_idx = 0; + for (size_type row = 0; row < num_rows; row++) { + col_idx = 0; + for (size_type col = 0; col < num_cols; col++) { + auto val = source(row, col); + if (is_nonzero(val)) { + result.val_at(row, col_idx) = val; + result.col_at(row, col_idx) = col; + col_idx++; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_ELL_KERNEL); + + +template +void convert_to_fbcsr(std::shared_ptr exec, + matrix::view::dense source, + matrix::Fbcsr* result) +{ + const auto num_rows = source.size[0]; + const auto num_cols = source.size[1]; + const auto bs = result->get_block_size(); + const auto nzbs = result->get_num_stored_blocks(); + const auto num_block_rows = num_rows / bs; + const auto num_block_cols = num_cols / bs; + acc::range> blocks( + std::array{static_cast(nzbs), + static_cast(bs), + static_cast(bs)}, + result->get_values()); + auto col_idxs = result->get_col_idxs(); + for (size_type brow = 0; brow < num_block_rows; ++brow) { + auto block = result->get_const_row_ptrs()[brow]; + for (size_type bcol = 0; bcol < num_block_cols; ++bcol) { + bool block_nz = false; + for (int lrow = 0; lrow < bs; ++lrow) { + for (int lcol = 0; lcol < bs; ++lcol) { + const auto row = lrow + bs * brow; + const auto col = lcol + bs * bcol; + block_nz = block_nz || is_nonzero(source(row, col)); + } + } + if (block_nz) { + col_idxs[block] = bcol; + for (int lrow = 0; lrow < bs; ++lrow) { + for (int lcol = 0; lcol < bs; ++lcol) { + const auto row = lrow + bs * brow; + const auto col = lcol + bs * bcol; + blocks(block, lrow, lcol) = source(row, col); + } + } + block++; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_FBCSR_KERNEL); + + +template +void convert_to_hybrid(std::shared_ptr exec, + matrix::view::dense source, + const int64*, + matrix::view::hybrid result) +{ + auto num_rows = result.size[0]; + auto num_cols = result.size[1]; + auto ell_lim = result.ell_part.num_stored_elements_per_row; + auto coo_lim = result.coo_part.num_stored_elements; + auto coo_val = result.coo_part.values; + auto coo_col = result.coo_part.col_idxs; + auto coo_row = result.coo_part.row_idxs; + std::fill_n( + result.ell_part.values, + result.ell_part.stride * result.ell_part.num_stored_elements_per_row, + zero()); + std::fill_n( + result.ell_part.col_idxs, + result.ell_part.stride * result.ell_part.num_stored_elements_per_row, + invalid_index()); + + size_type coo_idx = 0; + for (size_type row = 0; row < num_rows; row++) { + size_type col = 0; + for (size_type col_idx = 0; col < num_cols && col_idx < ell_lim; + col++) { + auto val = source(row, col); + if (is_nonzero(val)) { + result.ell_part.val_at(row, col_idx) = val; + result.ell_part.col_at(row, col_idx) = col; + col_idx++; + } + } + for (; col < num_cols; col++) { + auto val = source(row, col); + if (is_nonzero(val)) { + coo_val[coo_idx] = val; + coo_col[coo_idx] = col; + coo_row[coo_idx] = row; + coo_idx++; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_HYBRID_KERNEL); + + +template +void convert_to_sellp(std::shared_ptr exec, + matrix::view::dense source, + matrix::view::sellp result) +{ + auto num_rows = result.size[0]; + auto num_cols = result.size[1]; + auto vals = result.values; + auto col_idxs = result.col_idxs; + auto slice_lengths = result.slice_lengths; + auto slice_sets = result.slice_sets; + auto slice_size = result.slice_size; + for (size_type row = 0; row < num_rows; row++) { + const auto slice = row / slice_size; + const auto local_row = row % slice_size; + auto sellp_ind = slice_sets[slice] * slice_size + local_row; + const auto sellp_end = slice_sets[slice + 1] * slice_size + local_row; + for (size_type col = 0; col < num_cols; col++) { + auto val = source(row, col); + if (is_nonzero(val)) { + col_idxs[sellp_ind] = col; + vals[sellp_ind] = val; + sellp_ind += slice_size; + } + } + for (; sellp_ind < sellp_end; sellp_ind += slice_size) { + col_idxs[sellp_ind] = invalid_index(); + vals[sellp_ind] = zero(); + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_SELLP_KERNEL); + + +template +void convert_to_sparsity_csr(std::shared_ptr exec, + matrix::view::dense source, + matrix::SparsityCsr* result) +{ + auto num_rows = result->get_size()[0]; + auto num_cols = result->get_size()[1]; + + auto row_ptrs = result->get_row_ptrs(); + auto col_idxs = result->get_col_idxs(); + auto value = result->get_value(); + value[0] = one(); + size_type cur_ptr = 0; + row_ptrs[0] = cur_ptr; + for (size_type row = 0; row < num_rows; ++row) { + for (size_type col = 0; col < num_cols; ++col) { + auto val = source(row, col); + if (is_nonzero(val)) { + col_idxs[cur_ptr] = col; + ++cur_ptr; + } + } + row_ptrs[row + 1] = cur_ptr; + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_CONVERT_TO_SPARSITY_CSR_KERNEL); + + +template +void compute_max_nnz_per_row(std::shared_ptr exec, + matrix::view::dense source, + size_type& result) +{ + auto num_rows = source.size[0]; + auto num_cols = source.size[1]; + result = 0; + for (size_type row = 0; row < num_rows; ++row) { + size_type num_nonzeros = 0; + for (size_type col = 0; col < num_cols; ++col) { + num_nonzeros += is_nonzero(source(row, col)); + } + result = std::max(num_nonzeros, result); + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( + GKO_DECLARE_DENSE_COMPUTE_MAX_NNZ_PER_ROW_KERNEL); + + +template +void compute_slice_sets(std::shared_ptr exec, + matrix::view::dense source, + size_type slice_size, size_type stride_factor, + size_type* slice_sets, size_type* slice_lengths) +{ + const auto num_rows = source.size[0]; + const auto num_cols = source.size[1]; + const auto num_slices = ceildiv(num_rows, slice_size); + for (size_type slice = 0; slice < num_slices; slice++) { + size_type slice_length = 0; + for (size_type local_row = 0; local_row < slice_size; local_row++) { + const auto row = slice * slice_size + local_row; + size_type row_nnz{}; + if (row < num_rows) { + for (size_type col = 0; col < num_cols; col++) { + row_nnz += is_nonzero(source(row, col)); + } + } + slice_length = std::max( + slice_length, ceildiv(row_nnz, stride_factor) * stride_factor); + } + slice_lengths[slice] = slice_length; + } + exec->copy(num_slices, slice_lengths, slice_sets); + components::prefix_sum_nonnegative(exec, slice_sets, num_slices + 1); +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( + GKO_DECLARE_DENSE_COMPUTE_SLICE_SETS_KERNEL); + + +template +void count_nonzeros_per_row(std::shared_ptr exec, + matrix::view::dense source, + IndexType* result) +{ + auto num_rows = source.size[0]; + auto num_cols = source.size[1]; + for (size_type row = 0; row < num_rows; ++row) { + IndexType num_nonzeros{}; + for (size_type col = 0; col < num_cols; ++col) { + num_nonzeros += is_nonzero(source(row, col)); + } + result[row] = num_nonzeros; + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_COUNT_NONZEROS_PER_ROW_KERNEL); +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( + GKO_DECLARE_DENSE_COUNT_NONZEROS_PER_ROW_KERNEL_SIZE_T); + + +template +void count_nonzero_blocks_per_row(std::shared_ptr exec, + matrix::view::dense source, + int bs, IndexType* result) +{ + const auto num_rows = source.size[0]; + const auto num_cols = source.size[1]; + const auto num_block_rows = num_rows / bs; + const auto num_block_cols = num_cols / bs; + for (size_type brow = 0; brow < num_block_rows; ++brow) { + IndexType num_nonzero_blocks{}; + for (size_type bcol = 0; bcol < num_block_cols; ++bcol) { + bool block_nz = false; + for (int lrow = 0; lrow < bs; ++lrow) { + for (int lcol = 0; lcol < bs; ++lcol) { + const auto row = lrow + bs * brow; + const auto col = lcol + bs * bcol; + block_nz = block_nz || is_nonzero(source(row, col)); + } + } + num_nonzero_blocks += block_nz ? 1 : 0; + } + result[brow] = num_nonzero_blocks; + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( + GKO_DECLARE_DENSE_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL); + + +template +void extract_diagonal(std::shared_ptr exec, + matrix::view::dense orig, + matrix::Diagonal* diag) +{ + auto diag_values = diag->get_values(); + for (size_type i = 0; i < diag->get_size()[0]; ++i) { + diag_values[i] = orig(i, i); + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_EXTRACT_DIAGONAL_KERNEL); + + +template +void add_scaled_diag(std::shared_ptr exec, + matrix::view::dense alpha, + const matrix::Diagonal* x, + matrix::view::dense y) +{ + const auto diag_values = x->get_const_values(); + if (is_nonzero(alpha(0, 0))) { + for (size_type i = 0; i < x->get_size()[0]; i++) { + y(i, i) += alpha(0, 0) * diag_values[i]; + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_ADD_SCALED_DIAG_KERNEL); + + +template +void sub_scaled_diag(std::shared_ptr exec, + matrix::view::dense alpha, + const matrix::Diagonal* x, + matrix::view::dense y) +{ + const auto diag_values = x->get_const_values(); + if (is_nonzero(alpha(0, 0))) { + for (size_type i = 0; i < x->get_size()[0]; i++) { + y(i, i) -= alpha(0, 0) * diag_values[i]; + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_DENSE_SUB_SCALED_DIAG_KERNEL); + + +template +void add_scaled_identity(std::shared_ptr exec, + matrix::view::dense alpha, + matrix::view::dense beta, + matrix::view::dense mtx) +{ + const auto dim = mtx.size; + for (size_type row = 0; row < dim[0]; row++) { + for (size_type col = 0; col < dim[1]; col++) { + mtx(row, col) = beta.values[0] * mtx(row, col); + if (row == col) { + mtx(row, row) += alpha.values[0]; + } + } + } +} + +GKO_INSTANTIATE_FOR_EACH_VALUE_AND_SCALAR_TYPE( + GKO_DECLARE_DENSE_ADD_SCALED_IDENTITY_KERNEL); + + +} // namespace dense +} // namespace reference +} // namespace kernels +} // namespace gko diff --git a/reference/matrix/multivector_kernels.cpp b/reference/matrix/multivector_kernels.cpp index a704274c365..c22a47eceaa 100644 --- a/reference/matrix/multivector_kernels.cpp +++ b/reference/matrix/multivector_kernels.cpp @@ -9,11 +9,7 @@ #include #include #include -#include #include -#include -#include -#include #include "accessor/block_col_major.hpp" #include "accessor/range.hpp" @@ -32,65 +28,6 @@ namespace reference { namespace multivector { -template -void simple_apply(std::shared_ptr exec, - matrix::view::dense a, - matrix::view::dense b, - matrix::view::dense c) -{ - for (size_type row = 0; row < c.size[0]; ++row) { - for (size_type col = 0; col < c.size[1]; ++col) { - c(row, col) = zero(); - } - } - - for (size_type row = 0; row < c.size[0]; ++row) { - for (size_type inner = 0; inner < a.size[1]; ++inner) { - for (size_type col = 0; col < c.size[1]; ++col) { - c(row, col) += a(row, inner) * b(inner, col); - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_SIMPLE_APPLY_KERNEL); - - -template -void apply(std::shared_ptr exec, - matrix::view::dense alpha, - matrix::view::dense a, - matrix::view::dense b, - matrix::view::dense beta, - matrix::view::dense c) -{ - if (is_nonzero(beta(0, 0))) { - for (size_type row = 0; row < c.size[0]; ++row) { - for (size_type col = 0; col < c.size[1]; ++col) { - c(row, col) *= beta(0, 0); - } - } - } else { - for (size_type row = 0; row < c.size[0]; ++row) { - for (size_type col = 0; col < c.size[1]; ++col) { - c(row, col) = zero(); - } - } - } - - for (size_type row = 0; row < c.size[0]; ++row) { - for (size_type inner = 0; inner < a.size[1]; ++inner) { - for (size_type col = 0; col < c.size[1]; ++col) { - c(row, col) += alpha(0, 0) * a(row, inner) * b(inner, col); - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_MULTIVECTOR_APPLY_KERNEL); - - template void copy(std::shared_ptr exec, matrix::view::dense input, @@ -227,42 +164,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_SCALAR_TYPE( GKO_DECLARE_MULTIVECTOR_SUB_SCALED_KERNEL); -template -void add_scaled_diag(std::shared_ptr exec, - matrix::view::dense alpha, - const matrix::Diagonal* x, - matrix::view::dense y) -{ - const auto diag_values = x->get_const_values(); - if (is_nonzero(alpha(0, 0))) { - for (size_type i = 0; i < x->get_size()[0]; i++) { - y(i, i) += alpha(0, 0) * diag_values[i]; - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_ADD_SCALED_DIAG_KERNEL); - - -template -void sub_scaled_diag(std::shared_ptr exec, - matrix::view::dense alpha, - const matrix::Diagonal* x, - matrix::view::dense y) -{ - const auto diag_values = x->get_const_values(); - if (is_nonzero(alpha(0, 0))) { - for (size_type i = 0; i < x->get_size()[0]; i++) { - y(i, i) -= alpha(0, 0) * diag_values[i]; - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_SUB_SCALED_DIAG_KERNEL); - - template void compute_dot(std::shared_ptr exec, matrix::view::dense x, @@ -460,369 +361,6 @@ GKO_INSTANTIATE_FOR_EACH_NON_COMPLEX_VALUE_TYPE( GKO_DECLARE_MULTIVECTOR_COMPUTE_SQRT_KERNEL); -template -void convert_to_coo(std::shared_ptr exec, - matrix::view::dense source, const int64*, - matrix::view::coo result) -{ - auto num_rows = result.size[0]; - auto num_cols = result.size[1]; - auto num_nonzeros = result.num_stored_elements; - - auto row_idxs = result.row_idxs; - auto col_idxs = result.col_idxs; - auto values = result.values; - - size_type idxs = 0; - for (size_type row = 0; row < num_rows; ++row) { - for (size_type col = 0; col < num_cols; ++col) { - auto val = source(row, col); - if (is_nonzero(val)) { - row_idxs[idxs] = row; - col_idxs[idxs] = col; - values[idxs] = val; - ++idxs; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_COO_KERNEL); - - -template -void convert_to_csr(std::shared_ptr exec, - matrix::view::dense source, - matrix::Csr* result) -{ - auto num_rows = result->get_size()[0]; - auto num_cols = result->get_size()[1]; - auto num_nonzeros = result->get_num_stored_elements(); - - auto row_ptrs = result->get_row_ptrs(); - auto col_idxs = result->get_col_idxs(); - auto values = result->get_values(); - - size_type cur_ptr = 0; - row_ptrs[0] = cur_ptr; - for (size_type row = 0; row < num_rows; ++row) { - for (size_type col = 0; col < num_cols; ++col) { - auto val = source(row, col); - if (is_nonzero(val)) { - col_idxs[cur_ptr] = col; - values[cur_ptr] = val; - ++cur_ptr; - } - } - row_ptrs[row + 1] = cur_ptr; - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_CSR_KERNEL); - - -template -void convert_to_ell(std::shared_ptr exec, - matrix::view::dense source, - matrix::view::ell result) -{ - auto num_rows = result.size[0]; - auto num_cols = result.size[1]; - auto max_nnz_per_row = result.num_stored_elements_per_row; - for (size_type i = 0; i < max_nnz_per_row; i++) { - for (size_type j = 0; j < num_rows; j++) { - result.val_at(j, i) = zero(); - result.col_at(j, i) = invalid_index(); - } - } - size_type col_idx = 0; - for (size_type row = 0; row < num_rows; row++) { - col_idx = 0; - for (size_type col = 0; col < num_cols; col++) { - auto val = source(row, col); - if (is_nonzero(val)) { - result.val_at(row, col_idx) = val; - result.col_at(row, col_idx) = col; - col_idx++; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_ELL_KERNEL); - - -template -void convert_to_fbcsr(std::shared_ptr exec, - matrix::view::dense source, - matrix::Fbcsr* result) -{ - const auto num_rows = source.size[0]; - const auto num_cols = source.size[1]; - const auto bs = result->get_block_size(); - const auto nzbs = result->get_num_stored_blocks(); - const auto num_block_rows = num_rows / bs; - const auto num_block_cols = num_cols / bs; - acc::range> blocks( - std::array{static_cast(nzbs), - static_cast(bs), - static_cast(bs)}, - result->get_values()); - auto col_idxs = result->get_col_idxs(); - for (size_type brow = 0; brow < num_block_rows; ++brow) { - auto block = result->get_const_row_ptrs()[brow]; - for (size_type bcol = 0; bcol < num_block_cols; ++bcol) { - bool block_nz = false; - for (int lrow = 0; lrow < bs; ++lrow) { - for (int lcol = 0; lcol < bs; ++lcol) { - const auto row = lrow + bs * brow; - const auto col = lcol + bs * bcol; - block_nz = block_nz || is_nonzero(source(row, col)); - } - } - if (block_nz) { - col_idxs[block] = bcol; - for (int lrow = 0; lrow < bs; ++lrow) { - for (int lcol = 0; lcol < bs; ++lcol) { - const auto row = lrow + bs * brow; - const auto col = lcol + bs * bcol; - blocks(block, lrow, lcol) = source(row, col); - } - } - block++; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_FBCSR_KERNEL); - - -template -void convert_to_hybrid(std::shared_ptr exec, - matrix::view::dense source, - const int64*, - matrix::view::hybrid result) -{ - auto num_rows = result.size[0]; - auto num_cols = result.size[1]; - auto ell_lim = result.ell_part.num_stored_elements_per_row; - auto coo_lim = result.coo_part.num_stored_elements; - auto coo_val = result.coo_part.values; - auto coo_col = result.coo_part.col_idxs; - auto coo_row = result.coo_part.row_idxs; - std::fill_n( - result.ell_part.values, - result.ell_part.stride * result.ell_part.num_stored_elements_per_row, - zero()); - std::fill_n( - result.ell_part.col_idxs, - result.ell_part.stride * result.ell_part.num_stored_elements_per_row, - invalid_index()); - - size_type coo_idx = 0; - for (size_type row = 0; row < num_rows; row++) { - size_type col = 0; - for (size_type col_idx = 0; col < num_cols && col_idx < ell_lim; - col++) { - auto val = source(row, col); - if (is_nonzero(val)) { - result.ell_part.val_at(row, col_idx) = val; - result.ell_part.col_at(row, col_idx) = col; - col_idx++; - } - } - for (; col < num_cols; col++) { - auto val = source(row, col); - if (is_nonzero(val)) { - coo_val[coo_idx] = val; - coo_col[coo_idx] = col; - coo_row[coo_idx] = row; - coo_idx++; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_HYBRID_KERNEL); - - -template -void convert_to_sellp(std::shared_ptr exec, - matrix::view::dense source, - matrix::view::sellp result) -{ - auto num_rows = result.size[0]; - auto num_cols = result.size[1]; - auto vals = result.values; - auto col_idxs = result.col_idxs; - auto slice_lengths = result.slice_lengths; - auto slice_sets = result.slice_sets; - auto slice_size = result.slice_size; - for (size_type row = 0; row < num_rows; row++) { - const auto slice = row / slice_size; - const auto local_row = row % slice_size; - auto sellp_ind = slice_sets[slice] * slice_size + local_row; - const auto sellp_end = slice_sets[slice + 1] * slice_size + local_row; - for (size_type col = 0; col < num_cols; col++) { - auto val = source(row, col); - if (is_nonzero(val)) { - col_idxs[sellp_ind] = col; - vals[sellp_ind] = val; - sellp_ind += slice_size; - } - } - for (; sellp_ind < sellp_end; sellp_ind += slice_size) { - col_idxs[sellp_ind] = invalid_index(); - vals[sellp_ind] = zero(); - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SELLP_KERNEL); - - -template -void convert_to_sparsity_csr(std::shared_ptr exec, - matrix::view::dense source, - matrix::SparsityCsr* result) -{ - auto num_rows = result->get_size()[0]; - auto num_cols = result->get_size()[1]; - - auto row_ptrs = result->get_row_ptrs(); - auto col_idxs = result->get_col_idxs(); - auto value = result->get_value(); - value[0] = one(); - size_type cur_ptr = 0; - row_ptrs[0] = cur_ptr; - for (size_type row = 0; row < num_rows; ++row) { - for (size_type col = 0; col < num_cols; ++col) { - auto val = source(row, col); - if (is_nonzero(val)) { - col_idxs[cur_ptr] = col; - ++cur_ptr; - } - } - row_ptrs[row + 1] = cur_ptr; - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_CONVERT_TO_SPARSITY_CSR_KERNEL); - - -template -void compute_max_nnz_per_row(std::shared_ptr exec, - matrix::view::dense source, - size_type& result) -{ - auto num_rows = source.size[0]; - auto num_cols = source.size[1]; - result = 0; - for (size_type row = 0; row < num_rows; ++row) { - size_type num_nonzeros = 0; - for (size_type col = 0; col < num_cols; ++col) { - num_nonzeros += is_nonzero(source(row, col)); - } - result = std::max(num_nonzeros, result); - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_COMPUTE_MAX_NNZ_PER_ROW_KERNEL); - - -template -void compute_slice_sets(std::shared_ptr exec, - matrix::view::dense source, - size_type slice_size, size_type stride_factor, - size_type* slice_sets, size_type* slice_lengths) -{ - const auto num_rows = source.size[0]; - const auto num_cols = source.size[1]; - const auto num_slices = ceildiv(num_rows, slice_size); - for (size_type slice = 0; slice < num_slices; slice++) { - size_type slice_length = 0; - for (size_type local_row = 0; local_row < slice_size; local_row++) { - const auto row = slice * slice_size + local_row; - size_type row_nnz{}; - if (row < num_rows) { - for (size_type col = 0; col < num_cols; col++) { - row_nnz += is_nonzero(source(row, col)); - } - } - slice_length = std::max( - slice_length, ceildiv(row_nnz, stride_factor) * stride_factor); - } - slice_lengths[slice] = slice_length; - } - exec->copy(num_slices, slice_lengths, slice_sets); - components::prefix_sum_nonnegative(exec, slice_sets, num_slices + 1); -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_COMPUTE_SLICE_SETS_KERNEL); - - -template -void count_nonzeros_per_row(std::shared_ptr exec, - matrix::view::dense source, - IndexType* result) -{ - auto num_rows = source.size[0]; - auto num_cols = source.size[1]; - for (size_type row = 0; row < num_rows; ++row) { - IndexType num_nonzeros{}; - for (size_type col = 0; col < num_cols; ++col) { - num_nonzeros += is_nonzero(source(row, col)); - } - result[row] = num_nonzeros; - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZEROS_PER_ROW_KERNEL); -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZEROS_PER_ROW_KERNEL_SIZE_T); - - -template -void count_nonzero_blocks_per_row(std::shared_ptr exec, - matrix::view::dense source, - int bs, IndexType* result) -{ - const auto num_rows = source.size[0]; - const auto num_cols = source.size[1]; - const auto num_block_rows = num_rows / bs; - const auto num_block_cols = num_cols / bs; - for (size_type brow = 0; brow < num_block_rows; ++brow) { - IndexType num_nonzero_blocks{}; - for (size_type bcol = 0; bcol < num_block_cols; ++bcol) { - bool block_nz = false; - for (int lrow = 0; lrow < bs; ++lrow) { - for (int lcol = 0; lcol < bs; ++lcol) { - const auto row = lrow + bs * brow; - const auto col = lcol + bs * bcol; - block_nz = block_nz || is_nonzero(source(row, col)); - } - } - num_nonzero_blocks += block_nz ? 1 : 0; - } - result[brow] = num_nonzero_blocks; - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( - GKO_DECLARE_MULTIVECTOR_COUNT_NONZERO_BLOCKS_PER_ROW_KERNEL); - - template void transpose(std::shared_ptr exec, matrix::view::dense orig, @@ -1171,21 +709,6 @@ GKO_INSTANTIATE_FOR_EACH_VALUE_AND_INDEX_TYPE( GKO_DECLARE_MULTIVECTOR_INV_COL_SCALE_PERMUTE_KERNEL); -template -void extract_diagonal(std::shared_ptr exec, - matrix::view::dense orig, - matrix::Diagonal* diag) -{ - auto diag_values = diag->get_values(); - for (size_type i = 0; i < diag->get_size()[0]; ++i) { - diag_values[i] = orig(i, i); - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE( - GKO_DECLARE_MULTIVECTOR_EXTRACT_DIAGONAL_KERNEL); - - template void inplace_absolute_dense(std::shared_ptr exec, matrix::view::dense source) @@ -1266,27 +789,6 @@ void get_imag(std::shared_ptr exec, GKO_INSTANTIATE_FOR_EACH_VALUE_TYPE(GKO_DECLARE_GET_IMAG_KERNEL); -template -void add_scaled_identity(std::shared_ptr exec, - matrix::view::dense alpha, - matrix::view::dense beta, - matrix::view::dense mtx) -{ - const auto dim = mtx.size; - for (size_type row = 0; row < dim[0]; row++) { - for (size_type col = 0; col < dim[1]; col++) { - mtx(row, col) = beta.values[0] * mtx(row, col); - if (row == col) { - mtx(row, row) += alpha.values[0]; - } - } - } -} - -GKO_INSTANTIATE_FOR_EACH_VALUE_AND_SCALAR_TYPE( - GKO_DECLARE_MULTIVECTOR_ADD_SCALED_IDENTITY_KERNEL); - - } // namespace multivector } // namespace reference } // namespace kernels diff --git a/reference/test/base/batch_multi_vector_kernels.cpp b/reference/test/base/batch_multi_vector_kernels.cpp index c42ec3333d3..3eb004e2a60 100644 --- a/reference/test/base/batch_multi_vector_kernels.cpp +++ b/reference/test/base/batch_multi_vector_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "core/base/batch_utilities.hpp" diff --git a/reference/test/base/combination.cpp b/reference/test/base/combination.cpp index 6dfed4855ec..f4a7a09f030 100644 --- a/reference/test/base/combination.cpp +++ b/reference/test/base/combination.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include "core/test/utils.hpp" @@ -18,21 +19,21 @@ namespace { template class Combination : public ::testing::Test { protected: - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; Combination() : exec{gko::ReferenceExecutor::create()}, - coefficients{gko::initialize({1}, exec), - gko::initialize({2}, exec)} + coefficients{gko::initialize({1}, exec), + gko::initialize({2}, exec)} { operators = { gko::initialize({I({2.0, 3.0}), I({1.0, 4.0})}, exec), gko::initialize({I({3.0, 2.0}), I({2.0, 0.0})}, exec)}; } - std::shared_ptr exec; - std::vector> coefficients; + std::vector> coefficients; std::vector> operators; }; @@ -45,11 +46,11 @@ TYPED_TEST(Combination, AppliesToVector) cmb = [ 8 7 ] [ 5 4 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmb = gko::Combination::create( this->coefficients[0], this->operators[0], this->coefficients[1], this->operators[1]); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmb->apply(x, res); @@ -65,11 +66,11 @@ TYPED_TEST(Combination, AppliesToMixedVector) [ 5 4 ] */ using value_type = gko::next_precision; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; auto cmb = gko::Combination::create( this->coefficients[0], this->operators[0], this->coefficients[1], this->operators[1]); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmb->apply(x, res); @@ -85,12 +86,12 @@ TYPED_TEST(Combination, AppliesToComplexVector) cmb = [ 8 7 ] [ 5 4 ] */ - using Mtx = gko::to_complex; - using T = typename Mtx::value_type; + using Vec = gko::to_complex; + using T = typename Vec::value_type; auto cmb = gko::Combination::create( this->coefficients[0], this->operators[0], this->coefficients[1], this->operators[1]); - auto x = gko::initialize({T{1.0, -2.0}, T{2.0, -4.0}}, this->exec); + auto x = gko::initialize({T{1.0, -2.0}, T{2.0, -4.0}}, this->exec); auto res = clone(x); cmb->apply(x, res); @@ -100,42 +101,19 @@ TYPED_TEST(Combination, AppliesToComplexVector) } -TYPED_TEST(Combination, AppliesToMixedComplexVector) -{ - /* - cmb = [ 8 7 ] - [ 5 4 ] - */ - using value_type = gko::to_complex>; - using Mtx = gko::matrix::MultiVector; - auto cmb = gko::Combination::create( - this->coefficients[0], this->operators[0], this->coefficients[1], - this->operators[1]); - auto x = gko::initialize( - {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec); - auto res = clone(x); - - cmb->apply(x, res); - - GKO_ASSERT_MTX_NEAR(res, - l({value_type{22.0, -44.0}, value_type{13.0, -26.0}}), - (r_mixed())); -} - - TYPED_TEST(Combination, AppliesLinearCombinationToVector) { /* cmb = [ 8 7 ] [ 5 4 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmb = gko::Combination::create( this->coefficients[0], this->operators[0], this->coefficients[1], this->operators[1]); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmb->apply(alpha, x, beta, res); @@ -151,13 +129,13 @@ TYPED_TEST(Combination, AppliesLinearCombinationToMixedVector) [ 5 4 ] */ using value_type = gko::next_precision; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; auto cmb = gko::Combination::create( this->coefficients[0], this->operators[0], this->coefficients[1], this->operators[1]); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmb->apply(alpha, x, beta, res); @@ -173,7 +151,7 @@ TYPED_TEST(Combination, AppliesLinearCombinationToComplexVector) cmb = [ 8 7 ] [ 5 4 ] */ - using MultiVector = typename TestFixture::Mtx; + using MultiVector = typename TestFixture::Vec; using MultiVectorComplex = gko::to_complex; using T = typename MultiVectorComplex::value_type; auto cmb = gko::Combination::create( @@ -192,31 +170,4 @@ TYPED_TEST(Combination, AppliesLinearCombinationToComplexVector) } -TYPED_TEST(Combination, AppliesLinearCombinationToMixedComplexVector) -{ - /* - cmb = [ 8 7 ] - [ 5 4 ] - */ - using MixedMultiVector = - gko::matrix::MultiVector>; - using MixedMultiVectorComplex = gko::to_complex; - using value_type = typename MixedMultiVectorComplex::value_type; - auto cmb = gko::Combination::create( - this->coefficients[0], this->operators[0], this->coefficients[1], - this->operators[1]); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize( - {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec); - auto res = clone(x); - - cmb->apply(alpha, x, beta, res); - - GKO_ASSERT_MTX_NEAR(res, - l({value_type{65.0, -130.0}, value_type{37.0, -74.0}}), - (r_mixed())); -} - - } // namespace diff --git a/reference/test/base/composition.cpp b/reference/test/base/composition.cpp index f56244521da..4a014439c8e 100644 --- a/reference/test/base/composition.cpp +++ b/reference/test/base/composition.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include "core/test/utils.hpp" @@ -46,7 +47,8 @@ class DummyLinOp : public gko::LinOp, template class Composition : public ::testing::Test { protected: - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using value_type = T; Composition() : exec{gko::ReferenceExecutor::create()} @@ -67,7 +69,7 @@ class Composition : public ::testing::Test { } std::shared_ptr exec; - std::vector> coefficients; + std::vector> coefficients; std::vector> operators; std::shared_ptr identity; std::shared_ptr product; @@ -82,9 +84,9 @@ TYPED_TEST(Composition, AppliesSingleToVector) cmp = [ -9 -2 ] [ 27 26 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Composition::create(this->product); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(x, res); @@ -99,10 +101,10 @@ TYPED_TEST(Composition, AppliesSingleToMixedVector) cmp = [ -9 -2 ] [ 27 26 ] */ - using Mtx = gko::matrix::MultiVector>; - using value_type = typename Mtx::value_type; + using Vec = gko::matrix::MultiVector>; + using value_type = typename Vec::value_type; auto cmp = gko::Composition::create(this->product); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(x, res); @@ -119,9 +121,9 @@ TYPED_TEST(Composition, AppliesSingleToComplexVector) [ 27 26 ] */ using value_type = gko::to_complex; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; auto cmp = gko::Composition::create(this->product); - auto x = gko::initialize( + auto x = gko::initialize( {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec); auto res = clone(x); @@ -133,38 +135,17 @@ TYPED_TEST(Composition, AppliesSingleToComplexVector) } -TYPED_TEST(Composition, AppliesSingleToMixedComplexVector) -{ - /* - cmp = [ -9 -2 ] - [ 27 26 ] - */ - using value_type = gko::next_precision>; - using Mtx = gko::matrix::MultiVector; - auto cmp = gko::Composition::create(this->product); - auto x = gko::initialize( - {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec); - auto res = clone(x); - - cmp->apply(x, res); - - GKO_ASSERT_MTX_NEAR(res, - l({value_type{-13.0, 26.0}, value_type{79.0, -158.0}}), - (r_mixed())); -} - - TYPED_TEST(Composition, AppliesSingleLinearCombinationToVector) { /* cmp = [ -9 -2 ] [ 27 26 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Composition::create(this->product); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(alpha, x, beta, res); @@ -180,11 +161,11 @@ TYPED_TEST(Composition, AppliesSingleLinearCombinationToMixedVector) [ 27 26 ] */ using value_type = gko::next_precision; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; auto cmp = gko::Composition::create(this->product); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(alpha, x, beta, res); @@ -200,7 +181,7 @@ TYPED_TEST(Composition, AppliesSingleLinearCombinationToComplexVector) cmp = [ -9 -2 ] [ 27 26 ] */ - using MultiVector = typename TestFixture::Mtx; + using MultiVector = typename TestFixture::Vec; using MultiVectorComplex = gko::to_complex; using value_type = typename MultiVectorComplex::value_type; auto cmp = gko::Composition::create(this->product); @@ -218,41 +199,16 @@ TYPED_TEST(Composition, AppliesSingleLinearCombinationToComplexVector) } -TYPED_TEST(Composition, AppliesSingleLinearCombinationToMixedComplexVector) -{ - /* - cmp = [ -9 -2 ] - [ 27 26 ] - */ - using MixedMultiVector = - gko::matrix::MultiVector>; - using MixedMultiVectorComplex = gko::to_complex; - using value_type = typename MixedMultiVectorComplex::value_type; - auto cmp = gko::Composition::create(this->product); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize( - {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec); - auto res = clone(x); - - cmp->apply(alpha, x, beta, res); - - GKO_ASSERT_MTX_NEAR(res, - l({value_type{-40.0, 80.0}, value_type{235.0, -470.0}}), - (r_mixed())); -} - - TYPED_TEST(Composition, AppliesToVector) { /* cmp = [ 2 ] * [ 3 2 ] [ 1 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Composition::create(this->operators[0], this->operators[1]); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(x, res); @@ -267,12 +223,12 @@ TYPED_TEST(Composition, AppliesLinearCombinationToVector) cmp = [ 2 ] * [ 3 2 ] [ 1 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Composition::create(this->operators[0], this->operators[1]); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(alpha, x, beta, res); @@ -287,10 +243,10 @@ TYPED_TEST(Composition, AppliesLongerToVector) cmp = [ 2 ] * [ 3 2 ] * [ -9 -2 ] [ 1 ] [ 27 26 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Composition::create( this->operators[0], this->operators[1], this->product); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(x, res); @@ -305,12 +261,12 @@ TYPED_TEST(Composition, AppliesLongerLinearCombinationToVector) cmp = [ 2 ] * [ 3 2 ] * [ -9 -2 ] [ 1 ] [ 27 26 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Composition::create( this->operators[0], this->operators[1], this->product); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(alpha, x, beta, res); @@ -326,10 +282,10 @@ TYPED_TEST(Composition, AppliesLongestToVector) [ 1 ] [ 5 -3 0 ] [ 6 -2 ] [ 0 1 ] [ -3 2 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Composition::create(this->operators.begin(), this->operators.end()); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(x, res); @@ -345,12 +301,12 @@ TYPED_TEST(Composition, AppliesLongestLinearCombinationToVector) [ 1 ] [ 5 -3 0 ] [ 6 -2 ] [ 0 1 ] [ -3 2 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Composition::create(this->operators.begin(), this->operators.end()); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(alpha, x, beta, res); @@ -366,10 +322,10 @@ TYPED_TEST(Composition, AppliesLongestToVectorMultipleRhs) [ 1 ] [ 5 -3 0 ] [ 6 -2 ] [ 0 1 ] [ -3 2 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Composition::create(this->operators.begin(), this->operators.end()); - auto x = clone(this->identity); + auto x = clone(this->identity->as_multivector_view()); auto res = clone(x); cmp->apply(x, res); @@ -386,12 +342,12 @@ TYPED_TEST(Composition, AppliesLongestLinearCombinationToVectorMultipleRhs) [ 1 ] [ 5 -3 0 ] [ 6 -2 ] [ 0 1 ] [ -3 2 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Composition::create(this->operators.begin(), this->operators.end()); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = clone(this->identity); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = clone(this->identity->as_multivector_view()); auto res = clone(x); cmp->apply(alpha, x, beta, res); @@ -406,13 +362,13 @@ TYPED_TEST(Composition, AppliesToVectorWithInitialGuess) /* cmp = I * DummyLinOp * I */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto cmp = gko::Composition::create( this->identity, DummyLinOp::create(this->exec, this->identity->get_size()), this->identity); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(x, res); @@ -426,14 +382,14 @@ TYPED_TEST(Composition, AppliesToVectorWithInitialGuess2) /* cmp = I * DummyLinOp(2x3) * DummyLinOp(3x2) * I */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto size1 = gko::dim<2>(3, 2); auto size2 = gko::dim<2>(2, 3); auto cmp = gko::Composition::create( this->identity, DummyLinOp::create(this->exec, size2), DummyLinOp::create(this->exec, size1), this->identity); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(x, res); @@ -447,12 +403,12 @@ TYPED_TEST(Composition, AppliesToVectorWithInitialGuess3) /* cmp = I * DummyLinOp */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto cmp = gko::Composition::create( DummyLinOp::create(this->exec, this->identity->get_size()), this->identity); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(x, res); @@ -466,14 +422,14 @@ TYPED_TEST(Composition, AppliesToVectorWithInitialGuess4) /* cmp = I * DummyLinOp(2x3) * DummyLinOp(3x2) */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto size1 = gko::dim<2>(3, 2); auto size2 = gko::dim<2>(2, 3); auto cmp = gko::Composition::create( this->identity, DummyLinOp::create(this->exec, size2), DummyLinOp::create(this->exec, size1)); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(x, res); @@ -487,14 +443,14 @@ TYPED_TEST(Composition, AppliesToVectorWithInitialGuess5) /* cmp = DummyLinOp(2x3) * DummyLinOp(3x2) * I */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto size1 = gko::dim<2>(3, 2); auto size2 = gko::dim<2>(2, 3); auto cmp = gko::Composition::create( DummyLinOp::create(this->exec, size2), DummyLinOp::create(this->exec, size1), this->identity); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = clone(x); cmp->apply(x, res); diff --git a/reference/test/base/perturbation.cpp b/reference/test/base/perturbation.cpp index 121562e20ef..bba5ade38f8 100644 --- a/reference/test/base/perturbation.cpp +++ b/reference/test/base/perturbation.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include "core/test/utils.hpp" @@ -18,20 +19,21 @@ namespace { template class Perturbation : public ::testing::Test { protected: - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; Perturbation() : exec{gko::ReferenceExecutor::create()}, - basis{gko::initialize({2.0, 1.0}, exec)}, - scalar{gko::initialize({2.0}, exec)} + basis{gko::initialize({2.0, 1.0}, exec)}, + scalar{gko::initialize({2.0}, exec)} { - projector = gko::initialize({I({3.0, 2.0})}, exec); + projector = gko::initialize({I({3.0, 2.0})}, exec); } std::shared_ptr exec; - std::shared_ptr basis; - std::shared_ptr projector; - std::shared_ptr scalar; + std::shared_ptr basis; + std::shared_ptr projector; + std::shared_ptr scalar; }; TYPED_TEST_SUITE(Perturbation, gko::test::ValueTypes, TypenameNameGenerator); @@ -39,7 +41,7 @@ TYPED_TEST_SUITE(Perturbation, gko::test::ValueTypes, TypenameNameGenerator); TYPED_TEST(Perturbation, CopiesOnSameExecutor) { - using Mtx = typename TestFixture::Mtx; + using Mtx = typename TestFixture::Vec; auto per = gko::Perturbation::create(this->scalar, this->basis, this->projector); auto out = per->create_default(); @@ -56,7 +58,7 @@ TYPED_TEST(Perturbation, CopiesOnSameExecutor) TYPED_TEST(Perturbation, MovesOnSameExecutor) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto per = gko::Perturbation::create(this->scalar, this->basis, this->projector); auto per2 = per->clone(); @@ -84,11 +86,11 @@ TYPED_TEST(Perturbation, AppliesToVector) cmp = I + 2 * [ 2 ] * [ 3 2 ] [ 1 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Perturbation::create(this->scalar, this->basis, this->projector); - auto x = gko::initialize({1.0, 2.0}, this->exec); - auto res = Mtx::create_with_config_of(x); + auto x = gko::initialize({1.0, 2.0}, this->exec); + auto res = Vec::create_with_config_of(x); cmp->apply(x, res); @@ -102,12 +104,12 @@ TYPED_TEST(Perturbation, AppliesToMixedVector) cmp = I + 2 * [ 2 ] * [ 3 2 ] [ 1 ] */ - using Mtx = gko::matrix::MultiVector>; - using value_type = typename Mtx::value_type; + using Vec = gko::matrix::MultiVector>; + using value_type = typename Vec::value_type; auto cmp = gko::Perturbation::create(this->scalar, this->basis, this->projector); - auto x = gko::initialize({1.0, 2.0}, this->exec); - auto res = Mtx::create_with_config_of(x); + auto x = gko::initialize({1.0, 2.0}, this->exec); + auto res = Vec::create_with_config_of(x); cmp->apply(x, res); @@ -123,12 +125,12 @@ TYPED_TEST(Perturbation, AppliesToComplexVector) [ 1 ] */ using value_type = gko::to_complex; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; auto cmp = gko::Perturbation::create(this->scalar, this->basis, this->projector); - auto x = gko::initialize( + auto x = gko::initialize( {value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec); - auto res = Mtx::create_with_config_of(x); + auto res = Vec::create_with_config_of(x); cmp->apply(x, res); @@ -166,12 +168,12 @@ TYPED_TEST(Perturbation, AppliesLinearCombinationToVector) cmp = I + 2 * [ 2 ] * [ 3 2 ] [ 1 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Perturbation::create(this->scalar, this->basis, this->projector); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = gko::clone(x); cmp->apply(alpha, x, beta, res); @@ -187,12 +189,12 @@ TYPED_TEST(Perturbation, AppliesLinearCombinationToMixedVector) [ 1 ] */ using value_type = gko::next_precision; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; auto cmp = gko::Perturbation::create(this->scalar, this->basis, this->projector); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = gko::clone(x); cmp->apply(alpha, x, beta, res); @@ -208,7 +210,7 @@ TYPED_TEST(Perturbation, AppliesLinearCombinationToComplexVector) cmp = I + 2 * [ 2 ] * [ 3 2 ] [ 1 ] */ - using MultiVector = typename TestFixture::Mtx; + using MultiVector = typename TestFixture::Vec; using MultiVectorComplex = gko::to_complex; using value_type = typename MultiVectorComplex::value_type; auto cmp = gko::Perturbation::create(this->scalar, this->basis, @@ -259,10 +261,10 @@ TYPED_TEST(Perturbation, ConstructionByBasisAppliesToVector) cmp = I + 2 * [ 2 ] * [ 2 1 ] [ 1 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Perturbation::create(this->scalar, this->basis); - auto x = gko::initialize({1.0, 2.0}, this->exec); - auto res = Mtx::create_with_config_of(x); + auto x = gko::initialize({1.0, 2.0}, this->exec); + auto res = Vec::create_with_config_of(x); cmp->apply(x, res); @@ -276,11 +278,11 @@ TYPED_TEST(Perturbation, ConstructionByBasisAppliesLinearCombinationToVector) cmp = I + 2 * [ 2 ] * [ 2 1 ] [ 1 ] */ - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; auto cmp = gko::Perturbation::create(this->scalar, this->basis); - auto alpha = gko::initialize({3.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto x = gko::initialize({1.0, 2.0}, this->exec); + auto alpha = gko::initialize({3.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto x = gko::initialize({1.0, 2.0}, this->exec); auto res = gko::clone(x); cmp->apply(alpha, x, beta, res); diff --git a/reference/test/base/utils.cpp b/reference/test/base/utils.cpp index 2c272ffdbd6..a34a23ce04b 100644 --- a/reference/test/base/utils.cpp +++ b/reference/test/base/utils.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "core/test/utils.hpp" @@ -27,14 +28,13 @@ class ConvertToWithSorting : public ::testing::Test { protected: using value_type = double; using index_type = gko::int32; - using MultiVector = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Csr = gko::matrix::Csr; using Coo = gko::matrix::Coo; ConvertToWithSorting() : ref{gko::ReferenceExecutor::create()}, - mtx{gko::initialize({{1, 2, 3}, {6, 0, 7}, {-1, 8, 0}}, - ref)}, + mtx{gko::initialize({{1, 2, 3}, {6, 0, 7}, {-1, 8, 0}}, ref)}, unsorted_coo{ Coo::create(ref, gko::dim<2>{3, 3}, gko::array{ref, {1, 3, 2, 7, 6, -1, 8}}, @@ -49,7 +49,7 @@ class ConvertToWithSorting : public ::testing::Test { {} std::shared_ptr ref; - std::unique_ptr mtx; + std::unique_ptr mtx; std::unique_ptr unsorted_coo; std::unique_ptr unsorted_csr; }; diff --git a/reference/test/factorization/cholesky_kernels.cpp b/reference/test/factorization/cholesky_kernels.cpp index 01146fed558..19421d81029 100644 --- a/reference/test/factorization/cholesky_kernels.cpp +++ b/reference/test/factorization/cholesky_kernels.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include "core/components/prefix_sum_kernels.hpp" @@ -53,8 +54,8 @@ class Cholesky : public ::testing::Test { {gko::one()}, ref); auto id = gko::matrix::Identity::create( ref, l_factor->get_size()[0]); - auto result = gko::as(l_factor->conj_transpose()); - l_factor->apply(one, id, one, result); + auto result = l_factor->scale_add( + one, one, gko::as(l_factor->conj_transpose())); gko::matrix_data data; result->write(data); for (auto& entry : data.nonzeros) { diff --git a/reference/test/factorization/factorization.cpp b/reference/test/factorization/factorization.cpp index b015dc47727..7b73aa41a9e 100644 --- a/reference/test/factorization/factorization.cpp +++ b/reference/test/factorization/factorization.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include diff --git a/reference/test/factorization/ic_kernels.cpp b/reference/test/factorization/ic_kernels.cpp index c97b0bc2fe2..9cf1066a482 100644 --- a/reference/test/factorization/ic_kernels.cpp +++ b/reference/test/factorization/ic_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "core/test/utils.hpp" @@ -60,7 +61,7 @@ class Ic : public ::testing::Test { using factorization_type = gko::factorization::Ic; using Coo = gko::matrix::Coo; using Csr = gko::matrix::Csr; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; Ic() : ref(gko::ReferenceExecutor::create()), @@ -190,11 +191,11 @@ TYPED_TEST(Ic, GenerateIdentity) } -TYPED_TEST(Ic, GenerateMultiVectorIdentity) +TYPED_TEST(Ic, GenerateDenseIdentity) { - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; auto dense_id = - gko::share(MultiVector::create(this->exec, this->identity->get_size())); + gko::share(Dense::create(this->exec, this->identity->get_size())); this->identity->convert_to(dense_id); auto fact = this->fact_fact->generate(dense_id); diff --git a/reference/test/factorization/ilu_kernels.cpp b/reference/test/factorization/ilu_kernels.cpp index 1c497d63c60..ad3b0033bdc 100644 --- a/reference/test/factorization/ilu_kernels.cpp +++ b/reference/test/factorization/ilu_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "core/test/utils.hpp" @@ -58,6 +59,7 @@ class Ilu : public ::testing::Test { using index_type = typename std::tuple_element<1, decltype(ValueIndexType())>::type; using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Coo = gko::matrix::Coo; using Csr = gko::matrix::Csr; using ilu_type = gko::factorization::Ilu; @@ -65,59 +67,59 @@ class Ilu : public ::testing::Test { : ref(gko::ReferenceExecutor::create()), exec(std::static_pointer_cast(ref)), // clang-format off - identity(gko::initialize( + identity(gko::initialize( {{1., 0., 0.}, {0., 1., 0.}, {0., 0., 1.}}, exec)), - lower_triangular(gko::initialize( + lower_triangular(gko::initialize( {{1., 0., 0.}, {1., 1., 0.}, {1., 1., 1.}}, exec)), - upper_triangular(gko::initialize( + upper_triangular(gko::initialize( {{1., 1., 1.}, {0., 1., 1.}, {0., 0., 1.}}, exec)), - mtx_small(gko::initialize( + mtx_small(gko::initialize( {{4., 6., 8.}, {2., 2., 5.}, {1., 1., 1.}}, exec)), mtx_csr_small(nullptr), - small_l_expected(gko::initialize( + small_l_expected(gko::initialize( {{1., 0., 0.}, {0.5, 1., 0.}, {0.25, 0.5, 1.}}, exec)), - small_u_expected(gko::initialize( + small_u_expected(gko::initialize( {{4., 6., 8.}, {0., -1., 1.}, {0., 0., -1.5}}, exec)), - mtx_small2(gko::initialize( + mtx_small2(gko::initialize( {{8., 8., 0}, {2., 0., 5.}, {1., 1., 1}}, exec)), mtx_csr_small2(nullptr), - small2_l_expected(gko::initialize( + small2_l_expected(gko::initialize( {{1., 0., 0}, {.25, 1., 0.}, {.125, 0., 1}}, exec)), - small2_u_expected(gko::initialize( + small2_u_expected(gko::initialize( {{8., 8., 0}, {0., -2., 5.}, {0., 0., 1}}, exec)), - mtx_big(gko::initialize({{1., 1., 1., 0., 1., 3.}, + mtx_big(gko::initialize({{1., 1., 1., 0., 1., 3.}, {1., 2., 2., 0., 2., 0.}, {0., 2., 3., 3., 3., 5.}, {1., 0., 3., 4., 4., 4.}, {1., 2., 0., 4., 5., 6.}, {0., 2., 3., 4., 5., 8.}}, exec)), - big_l_expected(gko::initialize({{1., 0., 0., 0., 0., 0.}, + big_l_expected(gko::initialize({{1., 0., 0., 0., 0., 0.}, {1., 1., 0., 0., 0., 0.}, {0., 2., 1., 0., 0., 0.}, {1., 0., 2., 1., 0., 0.}, {1., 1., 0., -2., 1., 0.}, {0., 2., 1., -0.5, 0.5, 1.}}, exec)), - big_u_expected(gko::initialize({{1., 1., 1., 0., 1., 3.}, + big_u_expected(gko::initialize({{1., 1., 1., 0., 1., 3.}, {0., 1., 1., 0., 1., 0.}, {0., 0., 1., 3., 1., 5.}, {0., 0., 0., -2., 1., -9.}, @@ -131,7 +133,7 @@ class Ilu : public ::testing::Test { {1., 2., 0., 4., 1., 6.}, {0., 2., 3., 4., 5., 8.}}, exec)), - big_nodiag_l_expected(gko::initialize( + big_nodiag_l_expected(gko::initialize( {{1., 0., 0., 0., 0., 0.}, {1., 1., 0., 0., 0., 0.}, {0., 2., 1., 0., 0., 0.}, @@ -139,7 +141,7 @@ class Ilu : public ::testing::Test { {1., 1., 0., 0.571428571428571, 1., 0.}, {0., 2., -0.5, 0.785714285714286, -0.108695652173913, 1.}}, exec)), - big_nodiag_u_expected(gko::initialize( + big_nodiag_u_expected(gko::initialize( {{1., 1., 1., 0., 1., 3.}, {0., 1., 1., 0., 1., 0.}, {0., 0., -2., 3., 1., 5.}, @@ -161,23 +163,23 @@ class Ilu : public ::testing::Test { std::shared_ptr ref; std::shared_ptr exec; - std::shared_ptr identity; - std::shared_ptr lower_triangular; - std::shared_ptr upper_triangular; - std::shared_ptr mtx_small; + std::shared_ptr identity; + std::shared_ptr lower_triangular; + std::shared_ptr upper_triangular; + std::shared_ptr mtx_small; std::shared_ptr mtx_csr_small; - std::shared_ptr small_l_expected; - std::shared_ptr small_u_expected; - std::shared_ptr mtx_small2; + std::shared_ptr small_l_expected; + std::shared_ptr small_u_expected; + std::shared_ptr mtx_small2; std::shared_ptr mtx_csr_small2; - std::shared_ptr small2_l_expected; - std::shared_ptr small2_u_expected; - std::shared_ptr mtx_big; - std::shared_ptr big_l_expected; - std::shared_ptr big_u_expected; + std::shared_ptr small2_l_expected; + std::shared_ptr small2_u_expected; + std::shared_ptr mtx_big; + std::shared_ptr big_l_expected; + std::shared_ptr big_u_expected; std::shared_ptr mtx_big_nodiag; - std::shared_ptr big_nodiag_l_expected; - std::shared_ptr big_nodiag_u_expected; + std::shared_ptr big_nodiag_l_expected; + std::shared_ptr big_nodiag_u_expected; std::unique_ptr ilu_factory_skip; std::unique_ptr ilu_factory_sort; }; @@ -319,7 +321,7 @@ TYPED_TEST(Ilu, GenerateForCsrIdentity) } -TYPED_TEST(Ilu, GenerateForMultiVectorIdentity) +TYPED_TEST(Ilu, GenerateForDenseIdentity) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_skip->generate(this->identity); @@ -331,7 +333,7 @@ TYPED_TEST(Ilu, GenerateForMultiVectorIdentity) } -TYPED_TEST(Ilu, GenerateForMultiVectorLowerTriangular) +TYPED_TEST(Ilu, GenerateForDenseLowerTriangular) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_skip->generate(this->lower_triangular); @@ -343,7 +345,7 @@ TYPED_TEST(Ilu, GenerateForMultiVectorLowerTriangular) } -TYPED_TEST(Ilu, GenerateForMultiVectorUpperTriangular) +TYPED_TEST(Ilu, GenerateForDenseUpperTriangular) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_skip->generate(this->upper_triangular); @@ -371,7 +373,7 @@ TYPED_TEST(Ilu, ApplyMethodMultiVectorSmall) } -TYPED_TEST(Ilu, GenerateForMultiVectorSmall) +TYPED_TEST(Ilu, GenerateForDenseSmall) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_skip->generate(this->mtx_small); @@ -531,7 +533,7 @@ TYPED_TEST(Ilu, GenerateForCsrBigWithDiagonalZeros) } -TYPED_TEST(Ilu, GenerateForMultiVectorBig) +TYPED_TEST(Ilu, GenerateForDenseBig) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_skip->generate(this->mtx_big); @@ -543,7 +545,7 @@ TYPED_TEST(Ilu, GenerateForMultiVectorBig) } -TYPED_TEST(Ilu, GenerateForMultiVectorBigSort) +TYPED_TEST(Ilu, GenerateForDenseBigSort) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_sort->generate(this->mtx_big); diff --git a/reference/test/factorization/lu_kernels.cpp b/reference/test/factorization/lu_kernels.cpp index 00d4aefab70..2d1bf016ed9 100644 --- a/reference/test/factorization/lu_kernels.cpp +++ b/reference/test/factorization/lu_kernels.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "core/base/index_range.hpp" diff --git a/reference/test/factorization/par_ic_kernels.cpp b/reference/test/factorization/par_ic_kernels.cpp index a835398ea03..26c7c069f37 100644 --- a/reference/test/factorization/par_ic_kernels.cpp +++ b/reference/test/factorization/par_ic_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "core/factorization/factorization_kernels.hpp" @@ -51,7 +52,7 @@ class ParIc : public ::testing::Test { gko::factorization::ParIc; using Coo = gko::matrix::Coo; using Csr = gko::matrix::Csr; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; ParIc() : ref(gko::ReferenceExecutor::create()), @@ -201,11 +202,11 @@ TYPED_TEST(ParIc, GenerateIdentity) } -TYPED_TEST(ParIc, GenerateMultiVectorIdentity) +TYPED_TEST(ParIc, GenerateDenseIdentity) { - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; auto dense_id = - gko::share(MultiVector::create(this->exec, this->identity->get_size())); + gko::share(Dense::create(this->exec, this->identity->get_size())); this->identity->convert_to(dense_id); auto fact = this->fact_fact->generate(dense_id); diff --git a/reference/test/factorization/par_ict_kernels.cpp b/reference/test/factorization/par_ict_kernels.cpp index a0080c8c54e..b6d2cc36d9d 100644 --- a/reference/test/factorization/par_ict_kernels.cpp +++ b/reference/test/factorization/par_ict_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "core/factorization/factorization_kernels.hpp" @@ -51,7 +52,7 @@ class ParIct : public ::testing::Test { gko::factorization::ParIct; using Coo = gko::matrix::Coo; using Csr = gko::matrix::Csr; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; ParIct() : ref(gko::ReferenceExecutor::create()), @@ -268,11 +269,11 @@ TYPED_TEST(ParIct, GenerateIdentity) } -TYPED_TEST(ParIct, GenerateMultiVectorIdentity) +TYPED_TEST(ParIct, GenerateDenseIdentity) { - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; auto dense_id = - gko::share(MultiVector::create(this->exec, this->identity->get_size())); + gko::share(Dense::create(this->exec, this->identity->get_size())); this->identity->convert_to(dense_id); auto fact = this->fact_fact->generate(dense_id); diff --git a/reference/test/factorization/par_ilu_kernels.cpp b/reference/test/factorization/par_ilu_kernels.cpp index 3738623adea..1bdbb119368 100644 --- a/reference/test/factorization/par_ilu_kernels.cpp +++ b/reference/test/factorization/par_ilu_kernels.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "core/factorization/factorization_kernels.hpp" @@ -49,6 +50,7 @@ class ParIlu : public ::testing::Test { using index_type = typename std::tuple_element<1, decltype(ValueIndexType())>::type; using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Coo = gko::matrix::Coo; using Csr = gko::matrix::Csr; using par_ilu_type = gko::factorization::ParIlu; @@ -60,59 +62,59 @@ class ParIlu : public ::testing::Test { {{0., 0., 0.}, {0., 0., 0.}, {0., 0., 0.}}, exec)), - identity(gko::initialize( + identity(gko::initialize( {{1., 0., 0.}, {0., 1., 0.}, {0., 0., 1.}}, exec)), - lower_triangular(gko::initialize( + lower_triangular(gko::initialize( {{1., 0., 0.}, {1., 1., 0.}, {1., 1., 1.}}, exec)), - upper_triangular(gko::initialize( + upper_triangular(gko::initialize( {{1., 1., 1.}, {0., 1., 1.}, {0., 0., 1.}}, exec)), - mtx_small(gko::initialize( + mtx_small(gko::initialize( {{4., 6., 8.}, {2., 2., 5.}, {1., 1., 1.}}, exec)), mtx_csr_small(nullptr), - small_l_expected(gko::initialize( + small_l_expected(gko::initialize( {{1., 0., 0.}, {0.5, 1., 0.}, {0.25, 0.5, 1.}}, exec)), - small_u_expected(gko::initialize( + small_u_expected(gko::initialize( {{4., 6., 8.}, {0., -1., 1.}, {0., 0., -1.5}}, exec)), - mtx_small2(gko::initialize( + mtx_small2(gko::initialize( {{8., 8., 0}, {2., 0., 5.}, {1., 1., 1}}, exec)), mtx_csr_small2(nullptr), - small2_l_expected(gko::initialize( + small2_l_expected(gko::initialize( {{1., 0., 0}, {.25, 1., 0.}, {.125, 0., 1}}, exec)), - small2_u_expected(gko::initialize( + small2_u_expected(gko::initialize( {{8., 8., 0}, {0., -2., 5.}, {0., 0., 1}}, exec)), - mtx_big(gko::initialize({{1., 1., 1., 0., 1., 3.}, + mtx_big(gko::initialize({{1., 1., 1., 0., 1., 3.}, {1., 2., 2., 0., 2., 0.}, {0., 2., 3., 3., 3., 5.}, {1., 0., 3., 4., 4., 4.}, {1., 2., 0., 4., 5., 6.}, {0., 2., 3., 4., 5., 8.}}, exec)), - big_l_expected(gko::initialize({{1., 0., 0., 0., 0., 0.}, + big_l_expected(gko::initialize({{1., 0., 0., 0., 0., 0.}, {1., 1., 0., 0., 0., 0.}, {0., 2., 1., 0., 0., 0.}, {1., 0., 2., 1., 0., 0.}, {1., 1., 0., -2., 1., 0.}, {0., 2., 1., -0.5, 0.5, 1.}}, exec)), - big_u_expected(gko::initialize({{1., 1., 1., 0., 1., 3.}, + big_u_expected(gko::initialize({{1., 1., 1., 0., 1., 3.}, {0., 1., 1., 0., 1., 0.}, {0., 0., 1., 3., 1., 5.}, {0., 0., 0., -2., 1., -9.}, @@ -126,7 +128,7 @@ class ParIlu : public ::testing::Test { {1., 2., 0., 4., 1., 6.}, {0., 2., 3., 4., 5., 8.}}, exec)), - big_nodiag_l_expected(gko::initialize( + big_nodiag_l_expected(gko::initialize( {{1., 0., 0., 0., 0., 0.}, {1., 1., 0., 0., 0., 0.}, {0., 2., 1., 0., 0., 0.}, @@ -134,7 +136,7 @@ class ParIlu : public ::testing::Test { {1., 1., 0., 0.571428571428571, 1., 0.}, {0., 2., -0.5, 0.785714285714286, -0.108695652173913, 1.}}, exec)), - big_nodiag_u_expected(gko::initialize( + big_nodiag_u_expected(gko::initialize( {{1., 1., 1., 0., 1., 3.}, {0., 1., 1., 0., 1., 0.}, {0., 0., -2., 3., 1., 5.}, @@ -159,23 +161,23 @@ class ParIlu : public ::testing::Test { std::shared_ptr ref; std::shared_ptr exec; std::shared_ptr empty_csr; - std::shared_ptr identity; - std::shared_ptr lower_triangular; - std::shared_ptr upper_triangular; - std::shared_ptr mtx_small; + std::shared_ptr identity; + std::shared_ptr lower_triangular; + std::shared_ptr upper_triangular; + std::shared_ptr mtx_small; std::shared_ptr mtx_csr_small; - std::shared_ptr small_l_expected; - std::shared_ptr small_u_expected; - std::shared_ptr mtx_small2; + std::shared_ptr small_l_expected; + std::shared_ptr small_u_expected; + std::shared_ptr mtx_small2; std::shared_ptr mtx_csr_small2; - std::shared_ptr small2_l_expected; - std::shared_ptr small2_u_expected; - std::shared_ptr mtx_big; - std::shared_ptr big_l_expected; - std::shared_ptr big_u_expected; + std::shared_ptr small2_l_expected; + std::shared_ptr small2_u_expected; + std::shared_ptr mtx_big; + std::shared_ptr big_l_expected; + std::shared_ptr big_u_expected; std::shared_ptr mtx_big_nodiag; - std::shared_ptr big_nodiag_l_expected; - std::shared_ptr big_nodiag_u_expected; + std::shared_ptr big_nodiag_l_expected; + std::shared_ptr big_nodiag_u_expected; std::unique_ptr ilu_factory_skip; std::unique_ptr ilu_factory_sort; }; @@ -328,17 +330,17 @@ TYPED_TEST(ParIlu, KernelInitializeRowPtrsLUZeroMatrix) TYPED_TEST(ParIlu, KernelInitializeLU) { - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; using Csr = typename TestFixture::Csr; using index_type = typename TestFixture::index_type; using value_type = typename TestFixture::value_type; // clang-format off auto expected_l = - gko::initialize({{1., 0., 0.}, + gko::initialize({{1., 0., 0.}, {2., 1., 0.}, {1., 1., 1.}}, this->ref); auto expected_u = - gko::initialize({{4., 6., 8.}, + gko::initialize({{4., 6., 8.}, {0., 2., 5.}, {0., 0., 1.}}, this->ref); // clang-format on @@ -379,18 +381,18 @@ TYPED_TEST(ParIlu, KernelInitializeLUZeroMatrix) TYPED_TEST(ParIlu, KernelComputeLU) { using value_type = typename TestFixture::value_type; - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; using Coo = typename TestFixture::Coo; using Csr = typename TestFixture::Csr; // clang-format off auto l_dense = - gko::initialize({{1., 0., 0.}, + gko::initialize({{1., 0., 0.}, {2., 1., 0.}, {1., 1., 1.}}, this->ref); // U must be transposed before calling the kernel, so we simply create it // transposed auto u_dense = - gko::initialize({{4., 0., 0.}, + gko::initialize({{4., 0., 0.}, {6., 2., 0.}, {8., 5., 1.}}, this->ref); // clang-format on @@ -403,8 +405,8 @@ TYPED_TEST(ParIlu, KernelComputeLU) this->mtx_small->convert_to(mtx_coo); // The expected result of U also needs to be transposed auto u_expected_lin_op = this->small_u_expected->transpose(); - auto u_expected = std::unique_ptr( - static_cast(u_expected_lin_op.release())); + auto u_expected = std::unique_ptr( + static_cast(u_expected_lin_op.release())); gko::kernels::reference::par_ilu_factorization::compute_l_u_factors( this->ref, iterations, mtx_coo->get_const_device_view(), l_csr.get(), @@ -517,7 +519,7 @@ TYPED_TEST(ParIlu, GenerateForCsrIdentity) } -TYPED_TEST(ParIlu, GenerateForMultiVectorIdentity) +TYPED_TEST(ParIlu, GenerateForDenseIdentity) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_skip->generate(this->identity); @@ -529,7 +531,7 @@ TYPED_TEST(ParIlu, GenerateForMultiVectorIdentity) } -TYPED_TEST(ParIlu, GenerateForMultiVectorLowerTriangular) +TYPED_TEST(ParIlu, GenerateForDenseLowerTriangular) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_skip->generate(this->lower_triangular); @@ -541,7 +543,7 @@ TYPED_TEST(ParIlu, GenerateForMultiVectorLowerTriangular) } -TYPED_TEST(ParIlu, GenerateForMultiVectorUpperTriangular) +TYPED_TEST(ParIlu, GenerateForDenseUpperTriangular) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_skip->generate(this->upper_triangular); @@ -569,7 +571,7 @@ TYPED_TEST(ParIlu, ApplyMethodMultiVectorSmall) } -TYPED_TEST(ParIlu, GenerateForMultiVectorSmall) +TYPED_TEST(ParIlu, GenerateForDenseSmall) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_skip->generate(this->mtx_small); @@ -621,7 +623,7 @@ TYPED_TEST(ParIlu, GenerateForCsrBigWithDiagonalZeros) } -TYPED_TEST(ParIlu, GenerateForMultiVectorSmallWithMultipleIterations) +TYPED_TEST(ParIlu, GenerateForDenseSmallWithMultipleIterations) { using value_type = typename TestFixture::value_type; using par_ilu_type = typename TestFixture::par_ilu_type; @@ -637,7 +639,7 @@ TYPED_TEST(ParIlu, GenerateForMultiVectorSmallWithMultipleIterations) } -TYPED_TEST(ParIlu, GenerateForMultiVectorBig) +TYPED_TEST(ParIlu, GenerateForDenseBig) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_skip->generate(this->mtx_big); @@ -649,7 +651,7 @@ TYPED_TEST(ParIlu, GenerateForMultiVectorBig) } -TYPED_TEST(ParIlu, GenerateForMultiVectorBigSort) +TYPED_TEST(ParIlu, GenerateForDenseBigSort) { using value_type = typename TestFixture::value_type; auto factors = this->ilu_factory_sort->generate(this->mtx_big); diff --git a/reference/test/factorization/par_ilut_kernels.cpp b/reference/test/factorization/par_ilut_kernels.cpp index 700dec77c08..ec385e9dc7f 100644 --- a/reference/test/factorization/par_ilut_kernels.cpp +++ b/reference/test/factorization/par_ilut_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include "core/test/utils.hpp" @@ -48,7 +49,7 @@ class ParIlut : public ::testing::Test { typename std::tuple_element<1, decltype(ValueIndexType())>::type; using factorization_type = gko::factorization::ParIlut; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Coo = gko::matrix::Coo; using Csr = gko::matrix::Csr; using ComplexCsr = @@ -558,11 +559,11 @@ TYPED_TEST(ParIlut, GenerateIdentity) } -TYPED_TEST(ParIlut, GenerateMultiVectorIdentity) +TYPED_TEST(ParIlut, GenerateDenseIdentity) { - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; auto dense_id = - gko::share(MultiVector::create(this->exec, this->identity->get_size())); + gko::share(Dense::create(this->exec, this->identity->get_size())); this->identity->convert_to(dense_id); auto fact = this->fact_fact->generate(dense_id); diff --git a/reference/test/log/convergence.cpp b/reference/test/log/convergence.cpp index f6299276539..17d8d1ce723 100644 --- a/reference/test/log/convergence.cpp +++ b/reference/test/log/convergence.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include diff --git a/reference/test/log/papi.cpp b/reference/test/log/papi.cpp index 9a8d5ca092f..dc489fb58e0 100644 --- a/reference/test/log/papi.cpp +++ b/reference/test/log/papi.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include diff --git a/reference/test/matrix/CMakeLists.txt b/reference/test/matrix/CMakeLists.txt index 42efeca53a1..33e09f2dd3b 100644 --- a/reference/test/matrix/CMakeLists.txt +++ b/reference/test/matrix/CMakeLists.txt @@ -3,6 +3,7 @@ ginkgo_create_test(batch_dense_kernels) ginkgo_create_test(batch_ell_kernels) ginkgo_create_test(coo_kernels) ginkgo_create_test(csr_kernels) +ginkgo_create_test(dense_kernels) ginkgo_create_test(diagonal_kernels) ginkgo_create_test(ell_kernels) ginkgo_create_test(fbcsr_kernels) diff --git a/reference/test/matrix/batch_csr_kernels.cpp b/reference/test/matrix/batch_csr_kernels.cpp index ea42736926e..a439b456046 100644 --- a/reference/test/matrix/batch_csr_kernels.cpp +++ b/reference/test/matrix/batch_csr_kernels.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "core/test/utils.hpp" diff --git a/reference/test/matrix/batch_dense_kernels.cpp b/reference/test/matrix/batch_dense_kernels.cpp index d3eebadcbfd..39d3a2c1bc3 100644 --- a/reference/test/matrix/batch_dense_kernels.cpp +++ b/reference/test/matrix/batch_dense_kernels.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "core/test/utils.hpp" @@ -25,8 +26,9 @@ class MultiVector : public ::testing::Test { protected: using value_type = T; using size_type = gko::size_type; - using BMtx = gko::batch::matrix::MultiVector; + using BMtx = gko::batch::matrix::Dense; using BMVec = gko::batch::MultiVector; + using DenseMtx = gko::matrix::Dense; using MultiVectorMtx = gko::matrix::MultiVector; MultiVector() : exec(gko::ReferenceExecutor::create()) { @@ -34,9 +36,9 @@ class MultiVector : public ::testing::Test { {{I({1.0, -1.0, 1.5}), I({-2.0, 2.0, 3.0})}, {{1.0, -2.0, -0.5}, {1.0, -2.5, 4.0}}}, exec); - mtx_00 = gko::initialize( + mtx_00 = gko::initialize( {I({1.0, -1.0, 1.5}), I({-2.0, 2.0, 3.0})}, exec); - mtx_01 = gko::initialize( + mtx_01 = gko::initialize( {I({1.0, -2.0, -0.5}), I({1.0, -2.5, 4.0})}, exec); b_0 = gko::batch::initialize( {{I({1.0, 0.0, 1.0}), I({2.0, 0.0, 1.0}), @@ -64,8 +66,8 @@ class MultiVector : public ::testing::Test { std::shared_ptr exec; std::unique_ptr mtx_0; - std::unique_ptr mtx_00; - std::unique_ptr mtx_01; + std::unique_ptr mtx_00; + std::unique_ptr mtx_01; std::unique_ptr b_0; std::unique_ptr b_00; std::unique_ptr b_01; diff --git a/reference/test/matrix/batch_ell_kernels.cpp b/reference/test/matrix/batch_ell_kernels.cpp index 00622d59d1b..cfc23182f3f 100644 --- a/reference/test/matrix/batch_ell_kernels.cpp +++ b/reference/test/matrix/batch_ell_kernels.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include diff --git a/reference/test/matrix/coo_kernels.cpp b/reference/test/matrix/coo_kernels.cpp index 8303aef9cba..61f8051bc4e 100644 --- a/reference/test/matrix/coo_kernels.cpp +++ b/reference/test/matrix/coo_kernels.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include "core/test/utils.hpp" @@ -32,6 +32,7 @@ class Coo : public ::testing::Test { using Csr = gko::matrix::Csr; using Mtx = gko::matrix::Coo; using Vec = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using MixedVec = gko::matrix::MultiVector>; Coo() : exec(gko::ReferenceExecutor::create()), mtx(Mtx::create(exec)) @@ -164,12 +165,10 @@ TYPED_TEST(Coo, MovesToCsr) } -TYPED_TEST(Coo, ConvertsToMultiVector) +TYPED_TEST(Coo, ConvertsToDense) { - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using MultiVector = typename TestFixture::Vec; - auto dense_mtx = MultiVector::create(this->mtx->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx->get_executor()); this->mtx->convert_to(dense_mtx); @@ -181,12 +180,10 @@ TYPED_TEST(Coo, ConvertsToMultiVector) } -TYPED_TEST(Coo, ConvertsToMultiVectorUnsorted) +TYPED_TEST(Coo, ConvertsToDenseUnsorted) { - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using MultiVector = typename TestFixture::Vec; - auto dense_mtx = MultiVector::create(this->mtx->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx->get_executor()); this->uns_mtx->convert_to(dense_mtx); @@ -198,11 +195,10 @@ TYPED_TEST(Coo, ConvertsToMultiVectorUnsorted) } -TYPED_TEST(Coo, MovesToMultiVector) +TYPED_TEST(Coo, MovesToDense) { - using value_type = typename TestFixture::value_type; - using MultiVector = typename TestFixture::Vec; - auto dense_mtx = MultiVector::create(this->mtx->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx->get_executor()); this->mtx->move_to(dense_mtx); @@ -282,14 +278,12 @@ TYPED_TEST(Coo, MovesEmptyToCsr) } -TYPED_TEST(Coo, ConvertsEmptyToMultiVector) +TYPED_TEST(Coo, ConvertsEmptyToDense) { - using ValueType = typename TestFixture::value_type; - using IndexType = typename TestFixture::index_type; using Coo = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = typename TestFixture::Dense; auto empty = Coo::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); empty->convert_to(res); @@ -297,14 +291,12 @@ TYPED_TEST(Coo, ConvertsEmptyToMultiVector) } -TYPED_TEST(Coo, MovesEmptyToMultiVector) +TYPED_TEST(Coo, MovesEmptyToDense) { - using ValueType = typename TestFixture::value_type; - using IndexType = typename TestFixture::index_type; using Coo = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = typename TestFixture::Dense; auto empty = Coo::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); empty->move_to(res); diff --git a/reference/test/matrix/csr_kernels.cpp b/reference/test/matrix/csr_kernels.cpp index e768fc1cfab..2b732802808 100644 --- a/reference/test/matrix/csr_kernels.cpp +++ b/reference/test/matrix/csr_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,7 @@ class Csr : public ::testing::Test { using Hybrid = gko::matrix::Hybrid; using Vec = gko::matrix::MultiVector; using MixedVec = gko::matrix::MultiVector>; + using Dense = gko::matrix::Dense; using Perm = gko::matrix::Permutation; using ScaledPerm = gko::matrix::ScaledPermutation; @@ -675,7 +677,7 @@ TYPED_TEST(Csr, MixedAppliesLinearCombinationToMultiVectorMatrix3) TYPED_TEST(Csr, AppliesToCsrMatrix) { using T = typename TestFixture::value_type; - this->mtx->apply(this->mtx3_unsorted, this->mtx2); + this->mtx2 = this->mtx->multiply(this->mtx3_unsorted); ASSERT_EQ(this->mtx2->get_size(), gko::dim<2>(2, 3)); ASSERT_EQ(this->mtx2->get_num_stored_elements(), 6); @@ -776,7 +778,8 @@ TYPED_TEST(Csr, AppliesLinearCombinationToCsrMatrix) auto alpha = gko::initialize({-1.0}, this->exec); auto beta = gko::initialize({2.0}, this->exec); - this->mtx->apply(alpha, this->mtx3_unsorted, beta, this->mtx2); + this->mtx2 = + this->mtx->multiply_add(alpha, this->mtx3_unsorted, beta, this->mtx2); ASSERT_EQ(this->mtx2->get_size(), gko::dim<2>(2, 3)); ASSERT_EQ(this->mtx2->get_num_stored_elements(), 6); @@ -912,7 +915,7 @@ TYPED_TEST(Csr, AppliesLinearCombinationToIdentityMatrix) this->exec); auto id = gko::matrix::Identity::create(this->exec, a->get_size()[1]); - a->apply(alpha, id, beta, b); + b = a->scale_add(alpha, beta, b); GKO_ASSERT_MTX_NEAR(b, expect, r::value); GKO_ASSERT_MTX_EQ_SPARSITY(b, expect); @@ -1200,11 +1203,11 @@ TYPED_TEST(Csr, MovesToPrecision) } -TYPED_TEST(Csr, ConvertsToMultiVector) +TYPED_TEST(Csr, ConvertsToDense) { - using MultiVector = typename TestFixture::Vec; - auto dense_mtx = MultiVector::create(this->mtx->get_executor()); - auto dense_other = gko::initialize( + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx->get_executor()); + auto dense_other = gko::initialize( 4, {{1.0, 3.0, 2.0}, {0.0, 5.0, 0.0}}, this->exec); this->mtx->convert_to(dense_mtx); @@ -1213,11 +1216,11 @@ TYPED_TEST(Csr, ConvertsToMultiVector) } -TYPED_TEST(Csr, MovesToMultiVector) +TYPED_TEST(Csr, MovesToDense) { - using MultiVector = typename TestFixture::Vec; - auto dense_mtx = MultiVector::create(this->mtx->get_executor()); - auto dense_other = gko::initialize( + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx->get_executor()); + auto dense_other = gko::initialize( 4, {{1.0, 3.0, 2.0}, {0.0, 5.0, 0.0}}, this->exec); this->mtx->move_to(dense_mtx); @@ -1390,13 +1393,12 @@ TYPED_TEST(Csr, MovesEmptyToPrecision) } -TYPED_TEST(Csr, ConvertsEmptyToMultiVector) +TYPED_TEST(Csr, ConvertsEmptyToDense) { - using ValueType = typename TestFixture::value_type; using Csr = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = typename TestFixture::Dense; auto empty = Csr::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); empty->convert_to(res); @@ -1404,13 +1406,12 @@ TYPED_TEST(Csr, ConvertsEmptyToMultiVector) } -TYPED_TEST(Csr, MovesEmptyToMultiVector) +TYPED_TEST(Csr, MovesEmptyToDense) { - using ValueType = typename TestFixture::value_type; using Csr = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = typename TestFixture::Dense; auto empty = Csr::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); empty->move_to(res); @@ -1696,13 +1697,12 @@ std::unique_ptr> ref_permute( permutation, (mode & permute_mode::inverse) == permute_mode::inverse); if ((mode & permute_mode::rows) == permute_mode::rows) { // compute P * A - permutation_csr->apply(input, result); + result = permutation_csr->multiply(input); } if ((mode & permute_mode::columns) == permute_mode::columns) { // compute A * P^T = (P * A^T)^T - auto tmp = result->transpose(); - auto tmp2 = gko::as(gko::as(tmp.get())->clone()); - permutation_csr->apply(tmp, tmp2); + auto tmp = gko::as(result->transpose()); + auto tmp2 = permutation_csr->multiply(tmp); result = gko::as(tmp2->transpose()); } return result; @@ -1716,15 +1716,13 @@ std::unique_ptr> ref_permute( { using gko::matrix::permute_mode; using Csr = gko::matrix::Csr; - auto result = input->clone(); auto row_permutation_csr = csr_from_permutation(row_permutation, invert); auto col_permutation_csr = csr_from_permutation(col_permutation, invert); - row_permutation_csr->apply(input, result); - auto tmp = result->transpose(); - auto tmp2 = gko::as(gko::as(tmp.get())->clone()); - col_permutation_csr->apply(tmp, tmp2); + auto result = row_permutation_csr->multiply(input); + auto tmp = gko::as(result->transpose()); + auto tmp2 = col_permutation_csr->multiply(tmp); return gko::as(tmp2->transpose()); } diff --git a/reference/test/matrix/dense_kernels.cpp b/reference/test/matrix/dense_kernels.cpp new file mode 100644 index 00000000000..c7a1f80948c --- /dev/null +++ b/reference/test/matrix/dense_kernels.cpp @@ -0,0 +1,1586 @@ +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include "core/matrix/dense_kernels.hpp" + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/test/utils.hpp" +#include "ginkgo/core/matrix/sellp.hpp" + + +namespace { + + +template +class Dense : public ::testing::Test { +protected: + using value_type = T; + using Mtx = gko::matrix::Dense; + using Vec = gko::matrix::MultiVector; + using MixedVec = gko::matrix::MultiVector>; + using ComplexMtx = gko::to_complex; + using RealMtx = gko::remove_complex; + + Dense() : exec(gko::ReferenceExecutor::create()) {} + + void SetUp() override + { + mtx1 = + gko::initialize(4, {{1.0, 2.0, 3.0}, {1.5, 2.5, 3.5}}, exec); + mtx2 = + gko::initialize({I({1.0, -1.0}), I({-2.0, 2.0})}, exec); + mtx3 = + gko::initialize(4, {{1.0, 3.0, 2.0}, {0.0, 5.0, 0.0}}, exec); + mtx4 = gko::initialize( + {{1.0, -1.0, -0.5}, {-2.0, 2.0, 4.5}, {2.1, 3.4, 1.2}}, exec); + mtx5 = gko::initialize({{1.0, 2.0, 0.0}, {0.0, 1.5, 0.0}}, exec); + mtx6 = gko::initialize({{1.0, 2.0, 3.0}, {0.0, 1.5, 0.0}}, exec); + mtx7 = gko::initialize( + {I({1.0, -1.0}), I({-2.0, 2.0}), I({-3.0, 3.0})}, exec); + vec1 = + gko::initialize(4, {{1.0, 2.0, 3.0}, {1.5, 2.5, 3.5}}, exec); + vec2 = + gko::initialize(4, {{1.0, 2.0, 3.0}, {0.5, 1.5, 2.5}}, exec); + } + + std::shared_ptr exec; + std::unique_ptr mtx1; + std::unique_ptr mtx2; + std::unique_ptr mtx3; + std::unique_ptr mtx4; + std::unique_ptr mtx5; + std::unique_ptr mtx6; + std::unique_ptr mtx7; + std::unique_ptr vec1; + std::unique_ptr vec2; + std::default_random_engine rand_engine; + + template + std::unique_ptr gen_mtx(int num_rows, int num_cols) + { + return gko::test::generate_random_matrix( + num_rows, num_cols, + std::uniform_int_distribution(num_cols, num_cols), + std::normal_distribution<>(0.0, 1.0), rand_engine, exec); + } +}; + + +TYPED_TEST_SUITE(Dense, gko::test::ValueTypes, TypenameNameGenerator); + + +TYPED_TEST(Dense, CopyRespectsStride) +{ + using value_type = typename TestFixture::value_type; + auto m = + gko::initialize>({1.0, 2.0}, this->exec); + auto m2 = + gko::matrix::Dense::create(this->exec, gko::dim<2>{2, 1}, 2); + auto original_data = m2->get_values(); + original_data[1] = TypeParam{3.0}; + + m->convert_to(m2); + + EXPECT_EQ(m2->at(0, 0), value_type{1.0}); + EXPECT_EQ(m2->get_stride(), 2); + EXPECT_EQ(m2->at(1, 0), value_type{2.0}); + EXPECT_EQ(m2->get_values(), original_data); + EXPECT_EQ(original_data[1], TypeParam{3.0}); +} + + +TYPED_TEST(Dense, CanBeFilledWithValue) +{ + using value_type = typename TestFixture::value_type; + auto m = + gko::initialize>({1.0, 2.0}, this->exec); + EXPECT_EQ(m->at(0, 0), value_type{1}); + EXPECT_EQ(m->at(0, 1), value_type{2}); + + m->fill(value_type{42}); + + EXPECT_EQ(m->at(0, 0), value_type{42}); + EXPECT_EQ(m->at(0, 1), value_type{42}); +} + + +TYPED_TEST(Dense, CanBeFilledWithValueForStridedMatrices) +{ + using value_type = typename TestFixture::value_type; + using T = value_type; + auto m = gko::initialize>( + 4, {I{1.0, 2.0}, I{3.0, 4.0}, I{5.0, 6.0}}, this->exec); + T in_stride{-1.0}; + m->get_values()[3] = in_stride; + + ASSERT_EQ(m->get_size(), gko::dim<2>(3, 2)); + ASSERT_EQ(m->get_num_stored_elements(), 12); + EXPECT_EQ(m->at(0, 0), value_type{1.0}); + EXPECT_EQ(m->at(0, 1), value_type{2.0}); + EXPECT_EQ(m->at(1, 0), value_type{3.0}); + EXPECT_EQ(m->at(1, 1), value_type{4.0}); + EXPECT_EQ(m->at(2, 0), value_type{5.0}); + EXPECT_EQ(m->at(2, 1), value_type{6.0}); + + m->fill(value_type{42}); + + ASSERT_EQ(m->get_size(), gko::dim<2>(3, 2)); + EXPECT_EQ(m->get_num_stored_elements(), 12); + EXPECT_EQ(m->at(0, 0), value_type{42.0}); + EXPECT_EQ(m->at(0, 1), value_type{42.0}); + EXPECT_EQ(m->at(1, 0), value_type{42.0}); + EXPECT_EQ(m->at(1, 1), value_type{42.0}); + EXPECT_EQ(m->at(2, 0), value_type{42.0}); + EXPECT_EQ(m->at(2, 1), value_type{42.0}); + ASSERT_EQ(m->get_values()[3], in_stride); +} + + +TYPED_TEST(Dense, AppliesToDense) +{ + using T = typename TestFixture::value_type; + T in_stride{-1}; + this->vec2->get_values()[3] = in_stride; + + this->mtx2->apply(this->vec1, this->vec2); + + EXPECT_EQ(this->vec2->at(0, 0), T{-0.5}); + EXPECT_EQ(this->vec2->at(0, 1), T{-0.5}); + EXPECT_EQ(this->vec2->at(0, 2), T{-0.5}); + EXPECT_EQ(this->vec2->at(1, 0), T{1.0}); + EXPECT_EQ(this->vec2->at(1, 1), T{1.0}); + EXPECT_EQ(this->vec2->at(1, 2), T{1.0}); + ASSERT_EQ(this->vec2->get_values()[3], in_stride); +} + + +TYPED_TEST(Dense, AppliesToMixedDense) +{ + using MixedMtx = typename TestFixture::MixedVec; + using MixedT = typename MixedMtx::value_type; + auto mvec1 = MixedMtx::create(this->exec); + auto mvec2 = MixedMtx::create(this->exec); + this->vec1->convert_to(mvec1); + this->vec2->convert_to(mvec2); + + this->mtx2->apply(mvec1, mvec2); + + EXPECT_EQ(mvec2->at(0, 0), MixedT{-0.5}); + EXPECT_EQ(mvec2->at(0, 1), MixedT{-0.5}); + EXPECT_EQ(mvec2->at(0, 2), MixedT{-0.5}); + EXPECT_EQ(mvec2->at(1, 0), MixedT{1.0}); + EXPECT_EQ(mvec2->at(1, 1), MixedT{1.0}); + ASSERT_EQ(mvec2->at(1, 2), MixedT{1.0}); +} + + +TYPED_TEST(Dense, AppliesLinearCombinationToDense) +{ + using Vec = typename TestFixture::Vec; + using T = typename TestFixture::value_type; + auto alpha = gko::initialize({-1.0}, this->exec); + auto beta = gko::initialize({2.0}, this->exec); + T in_stride{-1}; + this->vec2->get_values()[3] = in_stride; + + this->mtx2->apply(alpha, this->vec1, beta, this->vec2); + + EXPECT_EQ(this->vec2->at(0, 0), T{2.5}); + EXPECT_EQ(this->vec2->at(0, 1), T{4.5}); + EXPECT_EQ(this->vec2->at(0, 2), T{6.5}); + EXPECT_EQ(this->vec2->at(1, 0), T{0.0}); + EXPECT_EQ(this->vec2->at(1, 1), T{2.0}); + EXPECT_EQ(this->vec2->at(1, 2), T{4.0}); + ASSERT_EQ(this->vec2->get_values()[3], in_stride); +} + + +TYPED_TEST(Dense, AppliesLinearCombinationToDenseWithZeroBetaNan) +{ + using Vec = typename TestFixture::Vec; + using T = typename TestFixture::value_type; + auto alpha = gko::initialize({-1.0}, this->exec); + auto beta = gko::initialize({0.0}, this->exec); + this->vec2->fill(gko::nan()); + + this->mtx2->apply(alpha, this->vec1, beta, this->vec2); + + EXPECT_EQ(this->vec2->at(0, 0), T{0.5}); + EXPECT_EQ(this->vec2->at(0, 1), T{0.5}); + EXPECT_EQ(this->vec2->at(0, 2), T{0.5}); + EXPECT_EQ(this->vec2->at(1, 0), T{-1.0}); + EXPECT_EQ(this->vec2->at(1, 1), T{-1.0}); + EXPECT_EQ(this->vec2->at(1, 2), T{-1.0}); +} + + +TYPED_TEST(Dense, AppliesLinearCombinationToMixedDense) +{ + using MixedVec = typename TestFixture::MixedVec; + using MixedT = typename MixedVec::value_type; + auto mvec1 = MixedVec::create(this->exec); + auto mvec2 = MixedVec::create(this->exec); + this->vec1->convert_to(mvec1); + this->vec2->convert_to(mvec2); + auto alpha = gko::initialize({-1.0}, this->exec); + auto beta = gko::initialize({2.0}, this->exec); + + this->mtx2->apply(alpha, mvec1, beta, mvec2); + + EXPECT_EQ(mvec2->at(0, 0), MixedT{2.5}); + EXPECT_EQ(mvec2->at(0, 1), MixedT{4.5}); + EXPECT_EQ(mvec2->at(0, 2), MixedT{6.5}); + EXPECT_EQ(mvec2->at(1, 0), MixedT{0.0}); + EXPECT_EQ(mvec2->at(1, 1), MixedT{2.0}); + ASSERT_EQ(mvec2->at(1, 2), MixedT{4.0}); +} + + +GKO_BEGIN_DISABLE_DEPRECATION_WARNINGS + + +TYPED_TEST(Dense, AppliesToDenseDeprecated) +{ + using Mtx = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + auto mtx2 = Mtx::create(this->exec, this->vec2->get_size(), + this->vec2->get_stride()); + this->vec2->as_const_dense_view()->convert_to(mtx2); + T in_stride{-1}; + mtx2->get_values()[3] = in_stride; + + this->mtx2->apply(this->mtx1, mtx2); + + EXPECT_EQ(mtx2->at(0, 0), T{-0.5}); + EXPECT_EQ(mtx2->at(0, 1), T{-0.5}); + EXPECT_EQ(mtx2->at(0, 2), T{-0.5}); + EXPECT_EQ(mtx2->at(1, 0), T{1.0}); + EXPECT_EQ(mtx2->at(1, 1), T{1.0}); + EXPECT_EQ(mtx2->at(1, 2), T{1.0}); + ASSERT_EQ(mtx2->get_values()[3], in_stride); +} + + +TYPED_TEST(Dense, AppliesLinearCombinationToDenseDeprecated) +{ + using Mtx = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + auto mtx2 = Mtx::create(this->exec, this->vec2->get_size(), + this->vec2->get_stride()); + this->vec2->as_const_dense_view()->convert_to(mtx2); + auto alpha = gko::initialize({-1.0}, this->exec); + auto beta = gko::initialize({2.0}, this->exec); + T in_stride{-1}; + mtx2->get_values()[3] = in_stride; + + this->mtx2->apply(alpha, this->mtx1, beta, mtx2); + + EXPECT_EQ(mtx2->at(0, 0), T{2.5}); + EXPECT_EQ(mtx2->at(0, 1), T{4.5}); + EXPECT_EQ(mtx2->at(0, 2), T{6.5}); + EXPECT_EQ(mtx2->at(1, 0), T{0.0}); + EXPECT_EQ(mtx2->at(1, 1), T{2.0}); + EXPECT_EQ(mtx2->at(1, 2), T{4.0}); + ASSERT_EQ(mtx2->get_values()[3], in_stride); +} + + +TYPED_TEST(Dense, ApplyFailsOnWrongInnerDimension) +{ + using Mtx = typename TestFixture::Mtx; + auto res = Mtx::create(this->exec, gko::dim<2>{2}); + + ASSERT_THROW(this->mtx2->apply(this->mtx1, res), gko::DimensionMismatch); +} + + +TYPED_TEST(Dense, ApplyFailsOnWrongNumberOfRows) +{ + using Mtx = typename TestFixture::Mtx; + auto res = Mtx::create(this->exec, gko::dim<2>{3}); + + ASSERT_THROW(this->mtx1->apply(this->mtx2, res), gko::DimensionMismatch); +} + + +TYPED_TEST(Dense, ApplyFailsOnWrongNumberOfCols) +{ + using Mtx = typename TestFixture::Mtx; + auto res = Mtx::create(this->exec, gko::dim<2>{2}, 3); + + ASSERT_THROW(this->mtx1->apply(this->mtx2, res), gko::DimensionMismatch); +} + + +GKO_END_DISABLE_DEPRECATION_WARNINGS + + +TYPED_TEST(Dense, SquareMatrixIsTransposable) +{ + using Mtx = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + auto trans = gko::as(this->mtx4->transpose()); + + GKO_ASSERT_MTX_NEAR( + trans, l({{1.0, -2.0, 2.1}, {-1.0, 2.0, 3.4}, {-0.5, 4.5, 1.2}}), + 0.0); +} + + +TYPED_TEST(Dense, SquareMatrixIsTransposableIntoDense) +{ + using Mtx = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + auto trans = Mtx::create(this->exec, this->mtx4->get_size()); + + this->mtx4->transpose(trans); + + GKO_ASSERT_MTX_NEAR( + trans, l({{1.0, -2.0, 2.1}, {-1.0, 2.0, 3.4}, {-0.5, 4.5, 1.2}}), + 0.0); +} + + +TYPED_TEST(Dense, SquareSubmatrixIsTransposableIntoDense) +{ + using Mtx = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + auto trans = Mtx::create(this->exec, gko::dim<2>{2, 2}, 4); + + this->mtx4->create_subview({0, 2}, {0, 2})->transpose(trans); + + GKO_ASSERT_MTX_NEAR(trans, l({{1.0, -2.0}, {-1.0, 2.0}}), 0.0); + ASSERT_EQ(trans->get_stride(), 4); +} + + +TYPED_TEST(Dense, SquareMatrixIsTransposableIntoDenseFailsForWrongDimensions) +{ + using Mtx = typename TestFixture::Mtx; + + ASSERT_THROW(this->mtx4->transpose(Mtx::create(this->exec)), + gko::DimensionMismatch); +} + + +TYPED_TEST(Dense, NonSquareMatrixIsTransposable) +{ + using Mtx = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + auto trans = gko::as(this->mtx3->transpose()); + + GKO_ASSERT_MTX_NEAR(trans, l({{1.0, 0.0}, {3.0, 5.0}, {2.0, 0.0}}), 0.0); +} + + +TYPED_TEST(Dense, NonSquareMatrixIsTransposableIntoDense) +{ + using Mtx = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + auto trans = + Mtx::create(this->exec, gko::transpose(this->mtx3->get_size())); + + this->mtx3->transpose(trans); + + GKO_ASSERT_MTX_NEAR(trans, l({{1.0, 0.0}, {3.0, 5.0}, {2.0, 0.0}}), 0.0); +} + + +TYPED_TEST(Dense, NonSquareSubmatrixIsTransposableIntoDense) +{ + using Mtx = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + auto trans = Mtx::create(this->exec, gko::dim<2>{2, 1}, 5); + + this->mtx3->create_subview({0, 1}, {0, 2})->transpose(trans); + + GKO_ASSERT_MTX_NEAR(trans, l({1.0, 3.0}), 0.0); + ASSERT_EQ(trans->get_stride(), 5); +} + + +TYPED_TEST(Dense, NonSquareMatrixIsTransposableIntoDenseFailsForWrongDimensions) +{ + using Mtx = typename TestFixture::Mtx; + + ASSERT_THROW(this->mtx3->transpose(Mtx::create(this->exec)), + gko::DimensionMismatch); +} + + +TYPED_TEST(Dense, ExtractsDiagonalFromSquareMatrix) +{ + using T = typename TestFixture::value_type; + + auto diag = this->mtx4->extract_diagonal(); + + ASSERT_EQ(diag->get_size()[0], 3); + ASSERT_EQ(diag->get_size()[1], 3); + ASSERT_EQ(diag->get_values()[0], T{1.}); + ASSERT_EQ(diag->get_values()[1], T{2.}); + ASSERT_EQ(diag->get_values()[2], T{1.2}); +} + + +TYPED_TEST(Dense, ExtractsDiagonalFromTallSkinnyMatrix) +{ + using T = typename TestFixture::value_type; + + auto diag = this->mtx3->extract_diagonal(); + + ASSERT_EQ(diag->get_size()[0], 2); + ASSERT_EQ(diag->get_size()[1], 2); + ASSERT_EQ(diag->get_values()[0], T{1.}); + ASSERT_EQ(diag->get_values()[1], T{5.}); +} + + +TYPED_TEST(Dense, ExtractsDiagonalFromShortFatMatrix) +{ + using T = typename TestFixture::value_type; + + auto diag = this->mtx7->extract_diagonal(); + + ASSERT_EQ(diag->get_size()[0], 2); + ASSERT_EQ(diag->get_size()[1], 2); + ASSERT_EQ(diag->get_values()[0], T{1.}); + ASSERT_EQ(diag->get_values()[1], T{2.}); +} + + +TYPED_TEST(Dense, ExtractsDiagonalFromSquareMatrixIntoDiagonal) +{ + using T = typename TestFixture::value_type; + auto diag = gko::matrix::Diagonal::create(this->exec, 3); + + this->mtx4->extract_diagonal(diag); + + ASSERT_EQ(diag->get_size()[0], 3); + ASSERT_EQ(diag->get_size()[1], 3); + ASSERT_EQ(diag->get_values()[0], T{1.}); + ASSERT_EQ(diag->get_values()[1], T{2.}); + ASSERT_EQ(diag->get_values()[2], T{1.2}); +} + + +TYPED_TEST(Dense, ExtractsDiagonalFromTallSkinnyMatrixIntoDiagonal) +{ + using T = typename TestFixture::value_type; + auto diag = gko::matrix::Diagonal::create(this->exec, 2); + + this->mtx3->extract_diagonal(diag); + + ASSERT_EQ(diag->get_size()[0], 2); + ASSERT_EQ(diag->get_size()[1], 2); + ASSERT_EQ(diag->get_values()[0], T{1.}); + ASSERT_EQ(diag->get_values()[1], T{5.}); +} + + +TYPED_TEST(Dense, ExtractsDiagonalFromShortFatMatrixIntoDiagonal) +{ + using T = typename TestFixture::value_type; + auto diag = gko::matrix::Diagonal::create(this->exec, 2); + + this->mtx7->extract_diagonal(diag); + + ASSERT_EQ(diag->get_size()[0], 2); + ASSERT_EQ(diag->get_size()[1], 2); + ASSERT_EQ(diag->get_values()[0], T{1.}); + ASSERT_EQ(diag->get_values()[1], T{2.}); +} + + +TYPED_TEST(Dense, AddsScaledDiag) +{ + using Vec = typename TestFixture::Vec; + using T = typename TestFixture::value_type; + auto alpha = gko::initialize({2.0}, this->exec); + auto diag = gko::matrix::Diagonal::create( + this->exec, 2, gko::array{this->exec, {3.0, 2.0}}); + + this->mtx2->add_scaled(alpha, diag); + + ASSERT_EQ(this->mtx2->at(0, 0), T{7.0}); + ASSERT_EQ(this->mtx2->at(0, 1), T{-1.0}); + ASSERT_EQ(this->mtx2->at(1, 0), T{-2.0}); + ASSERT_EQ(this->mtx2->at(1, 1), T{6.0}); +} + + +TYPED_TEST(Dense, SubtractsScaledDiag) +{ + using Vec = typename TestFixture::Vec; + using T = typename TestFixture::value_type; + auto alpha = gko::initialize({-2.0}, this->exec); + auto diag = gko::matrix::Diagonal::create( + this->exec, 2, gko::array{this->exec, {3.0, 2.0}}); + + this->mtx2->sub_scaled(alpha, diag); + + ASSERT_EQ(this->mtx2->at(0, 0), T{7.0}); + ASSERT_EQ(this->mtx2->at(0, 1), T{-1.0}); + ASSERT_EQ(this->mtx2->at(1, 0), T{-2.0}); + ASSERT_EQ(this->mtx2->at(1, 1), T{6.0}); +} + + +TYPED_TEST(Dense, ScaleAddIdentityRectangular) +{ + using T = typename TestFixture::value_type; + using Vec = typename TestFixture::Vec; + using Mtx = typename TestFixture::Mtx; + auto alpha = gko::initialize({2.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto b = gko::initialize( + {I{2.0, 0.0}, I{1.0, 2.5}, I{0.0, -4.0}}, this->exec); + + b->add_scaled_identity(alpha, beta); + + GKO_ASSERT_MTX_NEAR(b, l({{0.0, 0.0}, {-1.0, -0.5}, {0.0, 4.0}}), 0.0); +} + + +TYPED_TEST(Dense, AppliesToComplex) +{ + using value_type = typename TestFixture::value_type; + using complex_type = gko::to_complex; + using Vec = gko::matrix::MultiVector; + auto exec = gko::ReferenceExecutor::create(); + auto b = + gko::initialize({{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}}, + {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}, + {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, + exec); + auto x = Vec::create(exec, gko::dim<2>{2, 2}); + + this->mtx1->apply(b, x); + + GKO_ASSERT_MTX_NEAR( + x, + l({{complex_type{14.0, 16.0}, complex_type{20.0, 22.0}}, + {complex_type{17.0, 19.0}, complex_type{24.5, 26.5}}}), + 0.0); +} + + +TYPED_TEST(Dense, AppliesToMixedComplex) +{ + using mixed_value_type = + gko::next_precision; + using mixed_complex_type = gko::to_complex; + using Vec = gko::matrix::Dense; + auto exec = gko::ReferenceExecutor::create(); + auto b = gko::initialize( + {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, + {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, + {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, + exec); + auto x = Vec::create(exec, gko::dim<2>{2, 2}); + + this->mtx1->apply(b, x); + + GKO_ASSERT_MTX_NEAR( + x, + l({{mixed_complex_type{14.0, 16.0}, mixed_complex_type{20.0, 22.0}}, + {mixed_complex_type{17.0, 19.0}, mixed_complex_type{24.5, 26.5}}}), + 0.0); +} + + +TYPED_TEST(Dense, AdvancedAppliesToComplex) +{ + using value_type = typename TestFixture::value_type; + using complex_type = gko::to_complex; + using Vector = gko::matrix::MultiVector; + using VectorComplex = gko::matrix::MultiVector; + auto exec = gko::ReferenceExecutor::create(); + + auto b = gko::initialize( + {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}}, + {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}, + {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, + exec); + auto x = gko::initialize( + {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}}, + {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, + exec); + auto alpha = gko::initialize({-1.0}, this->exec); + auto beta = gko::initialize({2.0}, this->exec); + + this->mtx1->apply(alpha, b, beta, x); + + GKO_ASSERT_MTX_NEAR( + x, + l({{complex_type{-12.0, -16.0}, complex_type{-16.0, -20.0}}, + {complex_type{-13.0, -15.0}, complex_type{-18.5, -20.5}}}), + 0.0); +} + + +TYPED_TEST(Dense, AdvancedAppliesToMixedComplex) +{ + using mixed_value_type = + gko::next_precision; + using mixed_complex_type = gko::to_complex; + using MixedVector = gko::matrix::MultiVector; + using MixedVectorComplex = gko::matrix::MultiVector; + auto exec = gko::ReferenceExecutor::create(); + + auto b = gko::initialize( + {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, + {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, + {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, + exec); + auto x = gko::initialize( + {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, + {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, + exec); + auto alpha = gko::initialize({-1.0}, this->exec); + auto beta = gko::initialize({2.0}, this->exec); + + this->mtx1->apply(alpha, b, beta, x); + + GKO_ASSERT_MTX_NEAR( + x, + l({{mixed_complex_type{-12.0, -16.0}, mixed_complex_type{-16.0, -20.0}}, + {mixed_complex_type{-13.0, -15.0}, + mixed_complex_type{-18.5, -20.5}}}), + 0.0); +} + + +template +class DenseWithIndexType + : public Dense< + typename std::tuple_element<0, decltype(ValueIndexType())>::type> { +public: + using value_type = + typename std::tuple_element<0, decltype(ValueIndexType())>::type; + using index_type = + typename std::tuple_element<1, decltype(ValueIndexType())>::type; +}; + +TYPED_TEST_SUITE(DenseWithIndexType, gko::test::ValueIndexTypes, + PairTypenameNameGenerator); + + +template +void assert_coo_eq_mtx3(const gko::matrix::Coo* coo_mtx) +{ + auto v = coo_mtx->get_const_values(); + auto c = coo_mtx->get_const_col_idxs(); + auto r = coo_mtx->get_const_row_idxs(); + + ASSERT_EQ(coo_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(coo_mtx->get_num_stored_elements(), 4); + EXPECT_EQ(r[0], 0); + EXPECT_EQ(r[1], 0); + EXPECT_EQ(r[2], 0); + EXPECT_EQ(r[3], 1); + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[2], 2); + EXPECT_EQ(c[3], 1); + EXPECT_EQ(v[0], ValueType{1.0}); + EXPECT_EQ(v[1], ValueType{3.0}); + EXPECT_EQ(v[2], ValueType{2.0}); + EXPECT_EQ(v[3], ValueType{5.0}); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToCoo) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Coo = typename gko::matrix::Coo; + auto coo_mtx = Coo::create(this->mtx3->get_executor()); + + this->mtx3->convert_to(coo_mtx); + + assert_coo_eq_mtx3(coo_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, MovesToCoo) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Coo = typename gko::matrix::Coo; + auto coo_mtx = Coo::create(this->mtx4->get_executor()); + + this->mtx3->move_to(coo_mtx); + + assert_coo_eq_mtx3(coo_mtx.get()); +} + + +template +void assert_csr_eq_mtx3(const gko::matrix::Csr* csr_mtx) +{ + auto v = csr_mtx->get_const_values(); + auto c = csr_mtx->get_const_col_idxs(); + auto r = csr_mtx->get_const_row_ptrs(); + ASSERT_EQ(csr_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(csr_mtx->get_num_stored_elements(), 4); + EXPECT_EQ(r[0], 0); + EXPECT_EQ(r[1], 3); + EXPECT_EQ(r[2], 4); + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[2], 2); + EXPECT_EQ(c[3], 1); + EXPECT_EQ(v[0], ValueType{1.0}); + EXPECT_EQ(v[1], ValueType{3.0}); + EXPECT_EQ(v[2], ValueType{2.0}); + EXPECT_EQ(v[3], ValueType{5.0}); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToCsr) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Csr = typename gko::matrix::Csr; + auto csr_s_classical = gko::matrix::csr::spmv_strategy::classical; + auto csr_s_merge = gko::matrix::csr::spmv_strategy::merge_path; + auto csr_mtx_c = Csr::create(this->mtx3->get_executor(), csr_s_classical); + auto csr_mtx_m = Csr::create(this->mtx3->get_executor(), csr_s_merge); + + this->mtx3->convert_to(csr_mtx_c); + this->mtx3->convert_to(csr_mtx_m); + + assert_csr_eq_mtx3(csr_mtx_c.get()); + ASSERT_EQ(csr_mtx_c->get_strategy(), + gko::matrix::csr::spmv_strategy::classical); + GKO_ASSERT_MTX_NEAR(csr_mtx_c, csr_mtx_m, 0.0); + ASSERT_EQ(csr_mtx_m->get_strategy(), + gko::matrix::csr::spmv_strategy::merge_path); +} + + +TYPED_TEST(DenseWithIndexType, MovesToCsr) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Csr = typename gko::matrix::Csr; + auto csr_s_classical = gko::matrix::csr::spmv_strategy::classical; + auto csr_s_merge = gko::matrix::csr::spmv_strategy::merge_path; + auto csr_mtx_c = Csr::create(this->mtx3->get_executor(), csr_s_classical); + auto csr_mtx_m = Csr::create(this->mtx3->get_executor(), csr_s_merge); + auto mtx_clone = this->mtx3->clone(); + + this->mtx3->move_to(csr_mtx_c); + mtx_clone->move_to(csr_mtx_m); + + assert_csr_eq_mtx3(csr_mtx_c.get()); + ASSERT_EQ(csr_mtx_c->get_strategy(), + gko::matrix::csr::spmv_strategy::classical); + GKO_ASSERT_MTX_NEAR(csr_mtx_c, csr_mtx_m, 0.0); + ASSERT_EQ(csr_mtx_m->get_strategy(), + gko::matrix::csr::spmv_strategy::merge_path); +} + + +template +void assert_sparsity_csr_eq_mtx3( + const gko::matrix::SparsityCsr* sparsity_csr_mtx) +{ + auto v = sparsity_csr_mtx->get_const_value(); + auto c = sparsity_csr_mtx->get_const_col_idxs(); + auto r = sparsity_csr_mtx->get_const_row_ptrs(); + + ASSERT_EQ(sparsity_csr_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(sparsity_csr_mtx->get_num_nonzeros(), 4); + EXPECT_EQ(r[0], 0); + EXPECT_EQ(r[1], 3); + EXPECT_EQ(r[2], 4); + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[2], 2); + EXPECT_EQ(c[3], 1); + EXPECT_EQ(v[0], ValueType{1.0}); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToSparsityCsr) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using SparsityCsr = + typename gko::matrix::SparsityCsr; + auto sparsity_csr_mtx = SparsityCsr::create(this->mtx3->get_executor()); + + this->mtx3->convert_to(sparsity_csr_mtx); + + assert_sparsity_csr_eq_mtx3(sparsity_csr_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, MovesToSparsityCsr) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using SparsityCsr = + typename gko::matrix::SparsityCsr; + auto sparsity_csr_mtx = SparsityCsr::create(this->mtx3->get_executor()); + + this->mtx3->move_to(sparsity_csr_mtx); + + assert_sparsity_csr_eq_mtx3(sparsity_csr_mtx.get()); +} + + +template +void assert_ell_eq_mtx5(const gko::matrix::Ell* ell_mtx) +{ + auto v = ell_mtx->get_const_values(); + auto c = ell_mtx->get_const_col_idxs(); + + ASSERT_EQ(ell_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(ell_mtx->get_num_stored_elements_per_row(), 2); + ASSERT_EQ(ell_mtx->get_num_stored_elements(), 4); + ASSERT_EQ(ell_mtx->get_stride(), 2); + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[2], 1); + EXPECT_EQ(c[3], gko::invalid_index()); + EXPECT_EQ(v[0], ValueType{1.0}); + EXPECT_EQ(v[1], ValueType{1.5}); + EXPECT_EQ(v[2], ValueType{2.0}); + EXPECT_EQ(v[3], ValueType{0.0}); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToEll) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Ell = typename gko::matrix::Ell; + auto ell_mtx = Ell::create(this->mtx5->get_executor()); + + this->mtx5->convert_to(ell_mtx); + + assert_ell_eq_mtx5(ell_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, MovesToEll) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Ell = typename gko::matrix::Ell; + auto ell_mtx = Ell::create(this->mtx5->get_executor()); + + this->mtx5->move_to(ell_mtx); + + assert_ell_eq_mtx5(ell_mtx.get()); +} + + +template +void assert_strided_ell_eq_mtx5( + const gko::matrix::Ell* ell_mtx) +{ + constexpr auto invalid_index = gko::invalid_index(); + auto v = ell_mtx->get_const_values(); + auto c = ell_mtx->get_const_col_idxs(); + + ASSERT_EQ(ell_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(ell_mtx->get_num_stored_elements_per_row(), 2); + ASSERT_EQ(ell_mtx->get_num_stored_elements(), 6); + ASSERT_EQ(ell_mtx->get_stride(), 3); + // only check the actual matrix entries. + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[3], 1); + EXPECT_EQ(c[4], invalid_index); + EXPECT_EQ(v[0], ValueType{1.0}); + EXPECT_EQ(v[1], ValueType{1.5}); + EXPECT_EQ(v[3], ValueType{2.0}); + EXPECT_EQ(v[4], ValueType{0.0}); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToEllWithStride) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Ell = typename gko::matrix::Ell; + auto ell_mtx = + Ell::create(this->mtx5->get_executor(), gko::dim<2>{2, 3}, 2, 3); + + this->mtx5->convert_to(ell_mtx); + + assert_strided_ell_eq_mtx5(ell_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, MovesToEllWithStride) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Ell = typename gko::matrix::Ell; + auto ell_mtx = + Ell::create(this->mtx5->get_executor(), gko::dim<2>{2, 3}, 2, 3); + + this->mtx5->move_to(ell_mtx); + + assert_strided_ell_eq_mtx5(ell_mtx.get()); +} + + +template +void assert_hybrid_auto_eq_mtx3( + const gko::matrix::Hybrid* hybrid_mtx) +{ + auto v = hybrid_mtx->get_const_coo_values(); + auto c = hybrid_mtx->get_const_coo_col_idxs(); + auto r = hybrid_mtx->get_const_coo_row_idxs(); + auto n = hybrid_mtx->get_ell_num_stored_elements_per_row(); + auto p = hybrid_mtx->get_ell_stride(); + + ASSERT_EQ(hybrid_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(hybrid_mtx->get_ell_num_stored_elements(), 0); + ASSERT_EQ(hybrid_mtx->get_coo_num_stored_elements(), 4); + EXPECT_EQ(n, 0); + EXPECT_EQ(p, 2); + EXPECT_EQ(r[0], 0); + EXPECT_EQ(r[1], 0); + EXPECT_EQ(r[2], 0); + EXPECT_EQ(r[3], 1); + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[2], 2); + EXPECT_EQ(c[3], 1); + EXPECT_EQ(v[0], ValueType{1.0}); + EXPECT_EQ(v[1], ValueType{3.0}); + EXPECT_EQ(v[2], ValueType{2.0}); + EXPECT_EQ(v[3], ValueType{5.0}); +} + + +TYPED_TEST(DenseWithIndexType, MovesToHybridAutomatically) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Hybrid = typename gko::matrix::Hybrid; + auto hybrid_mtx = Hybrid::create(this->mtx3->get_executor()); + + this->mtx3->move_to(hybrid_mtx); + + assert_hybrid_auto_eq_mtx3(hybrid_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToHybridAutomatically) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Hybrid = typename gko::matrix::Hybrid; + auto hybrid_mtx = Hybrid::create(this->mtx3->get_executor()); + + this->mtx3->convert_to(hybrid_mtx); + + assert_hybrid_auto_eq_mtx3(hybrid_mtx.get()); +} + + +template +void assert_hybrid_strided_eq_mtx3( + const gko::matrix::Hybrid* hybrid_mtx) +{ + auto v = hybrid_mtx->get_const_coo_values(); + auto c = hybrid_mtx->get_const_coo_col_idxs(); + auto r = hybrid_mtx->get_const_coo_row_idxs(); + auto n = hybrid_mtx->get_ell_num_stored_elements_per_row(); + auto p = hybrid_mtx->get_ell_stride(); + + ASSERT_EQ(hybrid_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(hybrid_mtx->get_ell_num_stored_elements(), 0); + ASSERT_EQ(hybrid_mtx->get_coo_num_stored_elements(), 4); + EXPECT_EQ(n, 0); + EXPECT_EQ(p, 3); + EXPECT_EQ(r[0], 0); + EXPECT_EQ(r[1], 0); + EXPECT_EQ(r[2], 0); + EXPECT_EQ(r[3], 1); + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[2], 2); + EXPECT_EQ(c[3], 1); + EXPECT_EQ(v[0], ValueType{1.0}); + EXPECT_EQ(v[1], ValueType{3.0}); + EXPECT_EQ(v[2], ValueType{2.0}); + EXPECT_EQ(v[3], ValueType{5.0}); +} + + +TYPED_TEST(DenseWithIndexType, MovesToHybridWithStrideAutomatically) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Hybrid = typename gko::matrix::Hybrid; + auto hybrid_mtx = + Hybrid::create(this->mtx3->get_executor(), gko::dim<2>{2, 3}, 0, 3); + + this->mtx3->move_to(hybrid_mtx); + + assert_hybrid_strided_eq_mtx3(hybrid_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToHybridWithStrideAutomatically) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Hybrid = typename gko::matrix::Hybrid; + auto hybrid_mtx = + Hybrid::create(this->mtx3->get_executor(), gko::dim<2>{2, 3}, 0, 3); + + this->mtx3->convert_to(hybrid_mtx); + + assert_hybrid_strided_eq_mtx3(hybrid_mtx.get()); +} + + +template +void assert_hybrid_limited_eq_mtx3( + const gko::matrix::Hybrid* hybrid_mtx) +{ + constexpr auto invalid_index = gko::invalid_index(); + auto v = hybrid_mtx->get_const_ell_values(); + auto c = hybrid_mtx->get_const_ell_col_idxs(); + auto n = hybrid_mtx->get_ell_num_stored_elements_per_row(); + auto p = hybrid_mtx->get_ell_stride(); + + ASSERT_EQ(hybrid_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(hybrid_mtx->get_ell_num_stored_elements(), 6); + ASSERT_EQ(hybrid_mtx->get_coo_num_stored_elements(), 1); + EXPECT_EQ(n, 2); + EXPECT_EQ(p, 3); + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[2], invalid_index); + EXPECT_EQ(c[3], 1); + EXPECT_EQ(c[4], invalid_index); + EXPECT_EQ(c[5], invalid_index); + EXPECT_EQ(v[0], ValueType{1.0}); + EXPECT_EQ(v[1], ValueType{5.0}); + EXPECT_EQ(v[2], ValueType{0.0}); + EXPECT_EQ(v[3], ValueType{3.0}); + EXPECT_EQ(v[4], ValueType{0.0}); + EXPECT_EQ(v[5], ValueType{0.0}); + EXPECT_EQ(hybrid_mtx->get_const_coo_values()[0], ValueType{2.0}); + EXPECT_EQ(hybrid_mtx->get_const_coo_row_idxs()[0], 0); + EXPECT_EQ(hybrid_mtx->get_const_coo_col_idxs()[0], 2); +} + + +TYPED_TEST(DenseWithIndexType, MovesToHybridWithStrideAndCooLengthByColumns2) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Hybrid = typename gko::matrix::Hybrid; + auto hybrid_mtx = + Hybrid::create(this->mtx3->get_executor(), gko::dim<2>{2, 3}, 2, 3, 3, + std::make_shared(2)); + + this->mtx3->move_to(hybrid_mtx); + + assert_hybrid_limited_eq_mtx3(hybrid_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToHybridWithStrideAndCooLengthByColumns2) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Hybrid = typename gko::matrix::Hybrid; + auto hybrid_mtx = + Hybrid::create(this->mtx3->get_executor(), gko::dim<2>{2, 3}, 2, 3, 3, + std::make_shared(2)); + + this->mtx3->convert_to(hybrid_mtx); + + assert_hybrid_limited_eq_mtx3(hybrid_mtx.get()); +} + + +template +void assert_hybrid_percent_eq_mtx3( + const gko::matrix::Hybrid* hybrid_mtx) +{ + auto v = hybrid_mtx->get_const_ell_values(); + auto c = hybrid_mtx->get_const_ell_col_idxs(); + auto n = hybrid_mtx->get_ell_num_stored_elements_per_row(); + auto p = hybrid_mtx->get_ell_stride(); + auto coo_v = hybrid_mtx->get_const_coo_values(); + auto coo_c = hybrid_mtx->get_const_coo_col_idxs(); + auto coo_r = hybrid_mtx->get_const_coo_row_idxs(); + + ASSERT_EQ(hybrid_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(hybrid_mtx->get_ell_num_stored_elements(), 3); + EXPECT_EQ(n, 1); + EXPECT_EQ(p, 3); + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[2], gko::invalid_index()); + EXPECT_EQ(v[0], ValueType{1.0}); + EXPECT_EQ(v[1], ValueType{5.0}); + EXPECT_EQ(v[2], ValueType{0.0}); + ASSERT_EQ(hybrid_mtx->get_coo_num_stored_elements(), 2); + EXPECT_EQ(coo_v[0], ValueType{3.0}); + EXPECT_EQ(coo_v[1], ValueType{2.0}); + EXPECT_EQ(coo_c[0], 1); + EXPECT_EQ(coo_c[1], 2); + EXPECT_EQ(coo_r[0], 0); + EXPECT_EQ(coo_r[1], 0); +} + + +TYPED_TEST(DenseWithIndexType, MovesToHybridWithStrideByPercent40) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Hybrid = typename gko::matrix::Hybrid; + auto hybrid_mtx = + Hybrid::create(this->mtx3->get_executor(), gko::dim<2>{2, 3}, 1, 3, + std::make_shared(0.4)); + + this->mtx3->move_to(hybrid_mtx); + + assert_hybrid_percent_eq_mtx3(hybrid_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToHybridWithStrideByPercent40) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Hybrid = typename gko::matrix::Hybrid; + auto hybrid_mtx = + Hybrid::create(this->mtx3->get_executor(), gko::dim<2>{2, 3}, 1, 3, + std::make_shared(0.4)); + + this->mtx3->convert_to(hybrid_mtx); + + assert_hybrid_percent_eq_mtx3(hybrid_mtx.get()); +} + + +template +void assert_sellp_eq_mtx6( + const gko::matrix::Sellp* sellp_mtx) +{ + constexpr auto invalid_index = gko::invalid_index(); + auto v = sellp_mtx->get_const_values(); + auto c = sellp_mtx->get_const_col_idxs(); + auto s = sellp_mtx->get_const_slice_sets(); + auto l = sellp_mtx->get_const_slice_lengths(); + + ASSERT_EQ(sellp_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(sellp_mtx->get_total_cols(), 3); + ASSERT_EQ(sellp_mtx->get_num_stored_elements(), + 3 * gko::matrix::default_slice_size); + ASSERT_EQ(sellp_mtx->get_slice_size(), gko::matrix::default_slice_size); + ASSERT_EQ(sellp_mtx->get_stride_factor(), + gko::matrix::default_stride_factor); + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[gko::matrix::default_slice_size], 1); + EXPECT_EQ(c[gko::matrix::default_slice_size + 1], invalid_index); + EXPECT_EQ(c[2 * gko::matrix::default_slice_size], 2); + EXPECT_EQ(c[2 * gko::matrix::default_slice_size + 1], invalid_index); + EXPECT_EQ(v[0], ValueType{1.0}); + EXPECT_EQ(v[1], ValueType{1.5}); + EXPECT_EQ(v[gko::matrix::default_slice_size], ValueType{2.0}); + EXPECT_EQ(v[gko::matrix::default_slice_size + 1], ValueType{0.0}); + EXPECT_EQ(v[2 * gko::matrix::default_slice_size], ValueType{3.0}); + EXPECT_EQ(v[2 * gko::matrix::default_slice_size + 1], ValueType{0.0}); + EXPECT_EQ(s[0], 0); + EXPECT_EQ(s[1], 3); + EXPECT_EQ(l[0], 3); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToSellp) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Sellp = typename gko::matrix::Sellp; + auto sellp_mtx = Sellp::create(this->mtx6->get_executor()); + + this->mtx6->convert_to(sellp_mtx); + + assert_sellp_eq_mtx6(sellp_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, MovesToSellp) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Sellp = typename gko::matrix::Sellp; + auto sellp_mtx = Sellp::create(this->mtx6->get_executor()); + + this->mtx6->move_to(sellp_mtx); + + assert_sellp_eq_mtx6(sellp_mtx.get()); +} + + +template +void assert_sellp_strided_eq_mtx6( + const gko::matrix::Sellp* sellp_mtx) +{ + constexpr auto invalid_index = gko::invalid_index(); + auto v = sellp_mtx->get_const_values(); + auto c = sellp_mtx->get_const_col_idxs(); + auto s = sellp_mtx->get_const_slice_sets(); + auto l = sellp_mtx->get_const_slice_lengths(); + + ASSERT_EQ(sellp_mtx->get_size(), gko::dim<2>(2, 3)); + ASSERT_EQ(sellp_mtx->get_total_cols(), 4); + ASSERT_EQ(sellp_mtx->get_num_stored_elements(), 8); + ASSERT_EQ(sellp_mtx->get_slice_size(), 2); + ASSERT_EQ(sellp_mtx->get_stride_factor(), 2); + EXPECT_EQ(c[0], 0); + EXPECT_EQ(c[1], 1); + EXPECT_EQ(c[2], 1); + EXPECT_EQ(c[3], invalid_index); + EXPECT_EQ(c[4], 2); + EXPECT_EQ(c[5], invalid_index); + EXPECT_EQ(c[6], invalid_index); + EXPECT_EQ(c[7], invalid_index); + EXPECT_EQ(v[0], ValueType{1.0}); + EXPECT_EQ(v[1], ValueType{1.5}); + EXPECT_EQ(v[2], ValueType{2.0}); + EXPECT_EQ(v[3], ValueType{0.0}); + EXPECT_EQ(v[4], ValueType{3.0}); + EXPECT_EQ(v[5], ValueType{0.0}); + EXPECT_EQ(v[6], ValueType{0.0}); + EXPECT_EQ(v[7], ValueType{0.0}); + EXPECT_EQ(s[0], 0); + EXPECT_EQ(s[1], 4); + EXPECT_EQ(l[0], 4); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToSellpWithSliceSizeAndStrideFactor) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Sellp = typename gko::matrix::Sellp; + auto sellp_mtx = + Sellp::create(this->mtx6->get_executor(), gko::dim<2>{}, 2, 2, 0); + + this->mtx6->convert_to(sellp_mtx); + + assert_sellp_strided_eq_mtx6(sellp_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, MovesToSellpWithSliceSizeAndStrideFactor) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Sellp = typename gko::matrix::Sellp; + auto sellp_mtx = + Sellp::create(this->mtx6->get_executor(), gko::dim<2>{}, 2, 2, 0); + + this->mtx6->move_to(sellp_mtx); + + assert_sellp_strided_eq_mtx6(sellp_mtx.get()); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsToAndFromSellpWithMoreThanOneSlice) +{ + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Mtx = typename TestFixture::Mtx; + using Sellp = typename gko::matrix::Sellp; + auto x = this->template gen_mtx(65, 25); + + auto sellp_mtx = Sellp::create(this->exec); + auto dense_mtx = Mtx::create(this->exec); + x->convert_to(sellp_mtx); + sellp_mtx->convert_to(dense_mtx); + + GKO_ASSERT_MTX_NEAR(dense_mtx, x, 0.0); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsEmptyToCoo) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Coo = typename gko::matrix::Coo; + auto empty = MultiVector::create(this->exec); + auto res = Coo::create(this->exec); + + empty->convert_to(res); + + ASSERT_EQ(res->get_num_stored_elements(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, MovesEmptyToCoo) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Coo = typename gko::matrix::Coo; + auto empty = MultiVector::create(this->exec); + auto res = Coo::create(this->exec); + + empty->move_to(res); + + ASSERT_EQ(res->get_num_stored_elements(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsEmptyMatrixToCsr) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Csr = typename gko::matrix::Csr; + auto empty = MultiVector::create(this->exec); + auto res = Csr::create(this->exec); + + empty->convert_to(res); + + ASSERT_EQ(res->get_num_stored_elements(), 0); + ASSERT_EQ(*res->get_const_row_ptrs(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, MovesEmptyMatrixToCsr) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Csr = typename gko::matrix::Csr; + auto empty = MultiVector::create(this->exec); + auto res = Csr::create(this->exec); + + empty->move_to(res); + + ASSERT_EQ(res->get_num_stored_elements(), 0); + ASSERT_EQ(*res->get_const_row_ptrs(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsEmptyToSparsityCsr) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using SparsityCsr = + typename gko::matrix::SparsityCsr; + auto empty = MultiVector::create(this->exec); + auto res = SparsityCsr::create(this->exec); + + empty->convert_to(res); + + ASSERT_EQ(res->get_num_nonzeros(), 0); + ASSERT_EQ(*res->get_const_row_ptrs(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, MovesEmptyToSparsityCsr) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using SparsityCsr = + typename gko::matrix::SparsityCsr; + auto empty = MultiVector::create(this->exec); + auto res = SparsityCsr::create(this->exec); + + empty->move_to(res); + + ASSERT_EQ(res->get_num_nonzeros(), 0); + ASSERT_EQ(*res->get_const_row_ptrs(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsEmptyToEll) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Ell = typename gko::matrix::Ell; + auto empty = MultiVector::create(this->exec); + auto res = Ell::create(this->exec); + + empty->convert_to(res); + + ASSERT_EQ(res->get_num_stored_elements(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, MovesEmptyToEll) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Ell = typename gko::matrix::Ell; + auto empty = MultiVector::create(this->exec); + auto res = Ell::create(this->exec); + + empty->move_to(res); + + ASSERT_EQ(res->get_num_stored_elements(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsEmptyToHybrid) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Hybrid = typename gko::matrix::Hybrid; + auto empty = MultiVector::create(this->exec); + auto res = Hybrid::create(this->exec); + + empty->convert_to(res); + + ASSERT_EQ(res->get_num_stored_elements(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, MovesEmptyToHybrid) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Hybrid = typename gko::matrix::Hybrid; + auto empty = MultiVector::create(this->exec); + auto res = Hybrid::create(this->exec); + + empty->move_to(res); + + ASSERT_EQ(res->get_num_stored_elements(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, ConvertsEmptyToSellp) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Sellp = typename gko::matrix::Sellp; + auto empty = MultiVector::create(this->exec); + auto res = Sellp::create(this->exec); + + empty->convert_to(res); + + ASSERT_EQ(res->get_num_stored_elements(), 0); + ASSERT_EQ(*res->get_const_slice_sets(), 0); + ASSERT_FALSE(res->get_size()); +} + + +TYPED_TEST(DenseWithIndexType, MovesEmptyToSellp) +{ + using MultiVector = typename TestFixture::Mtx; + using value_type = typename TestFixture::value_type; + using index_type = typename TestFixture::index_type; + using Sellp = typename gko::matrix::Sellp; + auto empty = MultiVector::create(this->exec); + auto res = Sellp::create(this->exec); + + empty->move_to(res); + + ASSERT_EQ(res->get_num_stored_elements(), 0); + ASSERT_EQ(*res->get_const_slice_sets(), 0); + ASSERT_FALSE(res->get_size()); +} + + +template +class MultiVectorComplex : public ::testing::Test { +protected: + using value_type = T; + using Mtx = gko::matrix::MultiVector; + using RealMtx = gko::matrix::MultiVector>; +}; + + +TYPED_TEST_SUITE(MultiVectorComplex, gko::test::ComplexValueTypes, + TypenameNameGenerator); + + +TYPED_TEST(MultiVectorComplex, NonSquareMatrixIsConjugateTransposable) +{ + using MultiVector = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + auto exec = gko::ReferenceExecutor::create(); + auto mtx = gko::initialize({{T{1.0, 2.0}, T{-1.0, 2.1}}, + {T{-2.0, 1.5}, T{4.5, 0.0}}, + {T{1.0, 0.0}, T{0.0, 1.0}}}, + exec); + + auto trans = gko::as(mtx->conj_transpose()); + + GKO_ASSERT_MTX_NEAR(trans, + l({{T{1.0, -2.0}, T{-2.0, -1.5}, T{1.0, 0.0}}, + {T{-1.0, -2.1}, T{4.5, 0.0}, T{0.0, -1.0}}}), + 0.0); +} + + +TYPED_TEST(MultiVectorComplex, + NonSquareMatrixIsConjugateTransposableIntoMultiVector) +{ + using MultiVector = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + auto exec = gko::ReferenceExecutor::create(); + auto mtx = gko::initialize({{T{1.0, 2.0}, T{-1.0, 2.1}}, + {T{-2.0, 1.5}, T{4.5, 0.0}}, + {T{1.0, 0.0}, T{0.0, 1.0}}}, + exec); + auto trans = MultiVector::create(exec, gko::transpose(mtx->get_size())); + + mtx->conj_transpose(trans); + + GKO_ASSERT_MTX_NEAR(trans, + l({{T{1.0, -2.0}, T{-2.0, -1.5}, T{1.0, 0.0}}, + {T{-1.0, -2.1}, T{4.5, 0.0}, T{0.0, -1.0}}}), + 0.0); +} + + +} // namespace diff --git a/reference/test/matrix/diagonal_kernels.cpp b/reference/test/matrix/diagonal_kernels.cpp index 742adec28f4..37446551596 100644 --- a/reference/test/matrix/diagonal_kernels.cpp +++ b/reference/test/matrix/diagonal_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -45,11 +46,11 @@ class Diagonal : public ::testing::Test { 4, {{2.0, 3.0, 4.0}, {3.0, 4.5, 6.0}}, exec)) { csr1 = Csr::create(exec); - csr1->copy_from(dense1); + csr1->copy_from(dense1->as_const_dense_view().get()); csr2 = Csr::create(exec); - csr2->copy_from(dense2); + csr2->copy_from(dense2->as_const_dense_view().get()); csr3 = Csr::create(exec); - csr3->copy_from(dense3); + csr3->copy_from(dense3->as_const_dense_view().get()); this->create_diag1(diag1.get()); this->create_diag2(diag2.get()); } diff --git a/reference/test/matrix/ell_kernels.cpp b/reference/test/matrix/ell_kernels.cpp index ebf082f0611..80919ff1bdc 100644 --- a/reference/test/matrix/ell_kernels.cpp +++ b/reference/test/matrix/ell_kernels.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ class Ell : public ::testing::Test { typename std::tuple_element<1, decltype(ValueIndexType())>::type; using Mtx = gko::matrix::Ell; using Csr = gko::matrix::Csr; + using Dense = gko::matrix::Dense; using Vec = gko::matrix::MultiVector; using MixedVec = gko::matrix::MultiVector>; @@ -499,10 +501,10 @@ TYPED_TEST(Ell, MovesToPrecision) } -TYPED_TEST(Ell, ConvertsToMultiVector) +TYPED_TEST(Ell, ConvertsToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx1->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx1->get_executor()); this->mtx1->convert_to(dense_mtx); @@ -514,10 +516,10 @@ TYPED_TEST(Ell, ConvertsToMultiVector) } -TYPED_TEST(Ell, MovesToMultiVector) +TYPED_TEST(Ell, MovesToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx1->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx1->get_executor()); this->mtx1->move_to(dense_mtx); @@ -633,12 +635,12 @@ TYPED_TEST(Ell, ApplyWithStrideFailsOnWrongNumberOfCols) } -TYPED_TEST(Ell, ConvertsWithStrideToMultiVector) +TYPED_TEST(Ell, ConvertsWithStrideToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx2->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx2->get_executor()); // clang-format off - auto dense_other = gko::initialize( + auto dense_other = gko::initialize( 4, {{1.0, 3.0, 2.0}, {0.0, 5.0, 0.0}}, this->exec); // clang-format on @@ -653,10 +655,10 @@ TYPED_TEST(Ell, ConvertsWithStrideToMultiVector) } -TYPED_TEST(Ell, MovesWithStrideToMultiVector) +TYPED_TEST(Ell, MovesWithStrideToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx2->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx2->get_executor()); this->mtx2->move_to(dense_mtx); @@ -788,14 +790,14 @@ TYPED_TEST(Ell, MovesEmptyToPrecision) } -TYPED_TEST(Ell, ConvertsEmptyToMultiVector) +TYPED_TEST(Ell, ConvertsEmptyToDense) { using ValueType = typename TestFixture::value_type; using IndexType = typename TestFixture::index_type; using Ell = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto empty = Ell::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); empty->convert_to(res); @@ -803,14 +805,14 @@ TYPED_TEST(Ell, ConvertsEmptyToMultiVector) } -TYPED_TEST(Ell, MovesEmptyToMultiVector) +TYPED_TEST(Ell, MovesEmptyToDense) { using ValueType = typename TestFixture::value_type; using IndexType = typename TestFixture::index_type; using Ell = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto empty = Ell::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); empty->move_to(res); diff --git a/reference/test/matrix/fbcsr_kernels.cpp b/reference/test/matrix/fbcsr_kernels.cpp index 92f42df6708..0ff1045e149 100644 --- a/reference/test/matrix/fbcsr_kernels.cpp +++ b/reference/test/matrix/fbcsr_kernels.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -37,6 +38,7 @@ class Fbcsr : public ::testing::Test { typename std::tuple_element<1, decltype(ValueIndexType())>::type; using Mtx = gko::matrix::Fbcsr; using Csr = gko::matrix::Csr; + using Dense = gko::matrix::Dense; using MultiVector = gko::matrix::MultiVector; using SparCsr = gko::matrix::SparsityCsr; using Diag = gko::matrix::Diagonal; @@ -312,27 +314,27 @@ TYPED_TEST(Fbcsr, MovesToPrecision) } -TYPED_TEST(Fbcsr, ConvertsToMultiVector) +TYPED_TEST(Fbcsr, ConvertsToDense) { - using MultiVector = typename TestFixture::MultiVector; - auto dense_mtx = MultiVector::create(this->mtx->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx->get_executor()); this->mtx->convert_to(dense_mtx); - auto refdenmtx = MultiVector::create(this->mtx->get_executor()); + auto refdenmtx = Dense::create(this->mtx->get_executor()); this->refcsrmtx->convert_to(refdenmtx); GKO_ASSERT_MTX_NEAR(dense_mtx, refdenmtx, 0.0); } -TYPED_TEST(Fbcsr, MovesToMultiVector) +TYPED_TEST(Fbcsr, MovesToDense) { - using MultiVector = typename TestFixture::MultiVector; - auto dense_mtx = MultiVector::create(this->mtx->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx->get_executor()); this->mtx->move_to(dense_mtx); - auto refdenmtx = MultiVector::create(this->mtx->get_executor()); + auto refdenmtx = Dense::create(this->mtx->get_executor()); this->refcsrmtx->convert_to(refdenmtx); GKO_ASSERT_MTX_NEAR(dense_mtx, refdenmtx, 0.0); } @@ -426,13 +428,13 @@ TYPED_TEST(Fbcsr, MovesEmptyToPrecision) } -TYPED_TEST(Fbcsr, ConvertsEmptyToMultiVector) +TYPED_TEST(Fbcsr, ConvertsEmptyToDense) { using ValueType = typename TestFixture::value_type; using Fbcsr = typename TestFixture::Mtx; - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; auto empty = Fbcsr::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); empty->convert_to(res); @@ -440,13 +442,13 @@ TYPED_TEST(Fbcsr, ConvertsEmptyToMultiVector) } -TYPED_TEST(Fbcsr, MovesEmptyToMultiVector) +TYPED_TEST(Fbcsr, MovesEmptyToDense) { using ValueType = typename TestFixture::value_type; using Fbcsr = typename TestFixture::Mtx; - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; auto empty = Fbcsr::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); empty->move_to(res); diff --git a/reference/test/matrix/fft_kernels.cpp b/reference/test/matrix/fft_kernels.cpp index a30767a7f83..db9a3793ddf 100644 --- a/reference/test/matrix/fft_kernels.cpp +++ b/reference/test/matrix/fft_kernels.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -25,6 +26,7 @@ class Fft : public ::testing::Test { protected: using value_type = T; using Vec = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Mtx = gko::matrix::Fft; using Mtx2 = gko::matrix::Fft2; using Mtx3 = gko::matrix::Fft3; @@ -60,15 +62,15 @@ class Fft : public ::testing::Test { fft(Mtx::create(exec, n)), fft2(Mtx2::create(exec, n1 * n2, n3)), fft3(Mtx3::create(exec, n1, n2, n3)), - dense_fft(Vec::create(exec, gko::dim<2>{n, n})), - dense_fft2(Vec::create(exec, gko::dim<2>{n, n})), - dense_fft3(Vec::create(exec, gko::dim<2>{n, n})), + dense_fft(Dense::create(exec, gko::dim<2>{n, n})), + dense_fft2(Dense::create(exec, gko::dim<2>{n, n})), + dense_fft3(Dense::create(exec, gko::dim<2>{n, n})), ifft(Mtx::create(exec, n, true)), ifft2(Mtx2::create(exec, n1 * n2, n3, true)), ifft3(Mtx3::create(exec, n1, n2, n3, true)), - dense_ifft(Vec::create(exec, gko::dim<2>{n, n})), - dense_ifft2(Vec::create(exec, gko::dim<2>{n, n})), - dense_ifft3(Vec::create(exec, gko::dim<2>{n, n})) + dense_ifft(Dense::create(exec, gko::dim<2>{n, n})), + dense_ifft2(Dense::create(exec, gko::dim<2>{n, n})), + dense_ifft3(Dense::create(exec, gko::dim<2>{n, n})) { std::uniform_int_distribution nz_dist(nrhs - 2, nrhs); std::uniform_real_distribution> @@ -143,15 +145,15 @@ class Fft : public ::testing::Test { std::unique_ptr fft; std::unique_ptr fft2; std::unique_ptr fft3; - std::unique_ptr dense_fft; - std::unique_ptr dense_fft2; - std::unique_ptr dense_fft3; + std::unique_ptr dense_fft; + std::unique_ptr dense_fft2; + std::unique_ptr dense_fft3; std::unique_ptr ifft; std::unique_ptr ifft2; std::unique_ptr ifft3; - std::unique_ptr dense_ifft; - std::unique_ptr dense_ifft2; - std::unique_ptr dense_ifft3; + std::unique_ptr dense_ifft; + std::unique_ptr dense_ifft2; + std::unique_ptr dense_ifft3; }; TYPED_TEST_SUITE(Fft, gko::test::ComplexValueTypesBase, TypenameNameGenerator); @@ -326,9 +328,9 @@ TYPED_TEST(Fft, AppliesStrided1DToMultiVector) { using T = typename TestFixture::value_type; auto in_view = - this->amplitude->create_submatrix({0, this->n}, {0, this->subcols}); + this->amplitude->create_subview({0, this->n}, {0, this->subcols}); auto ref_view = - this->frequency1->create_submatrix({0, this->n}, {0, this->subcols}); + this->frequency1->create_subview({0, this->n}, {0, this->subcols}); auto out = TestFixture::Vec::create(this->exec, in_view->get_size(), this->stride); @@ -354,9 +356,9 @@ TYPED_TEST(Fft, AppliesStridedInverse1DToMultiVector) { using T = typename TestFixture::value_type; auto in_view = - this->frequency1->create_submatrix({0, this->n}, {0, this->subcols}); + this->frequency1->create_subview({0, this->n}, {0, this->subcols}); auto ref_view = - this->amplitude->create_submatrix({0, this->n}, {0, this->subcols}); + this->amplitude->create_subview({0, this->n}, {0, this->subcols}); auto out = TestFixture::Vec::create(this->exec, in_view->get_size(), this->stride); @@ -382,9 +384,9 @@ TYPED_TEST(Fft, AppliesStrided2DToMultiVector) { using T = typename TestFixture::value_type; auto in_view = - this->amplitude->create_submatrix({0, this->n}, {0, this->subcols}); + this->amplitude->create_subview({0, this->n}, {0, this->subcols}); auto ref_view = - this->frequency2->create_submatrix({0, this->n}, {0, this->subcols}); + this->frequency2->create_subview({0, this->n}, {0, this->subcols}); auto out = TestFixture::Vec::create(this->exec, in_view->get_size(), this->stride); @@ -410,9 +412,9 @@ TYPED_TEST(Fft, AppliesStridedInverse2DToMultiVector) { using T = typename TestFixture::value_type; auto in_view = - this->frequency2->create_submatrix({0, this->n}, {0, this->subcols}); + this->frequency2->create_subview({0, this->n}, {0, this->subcols}); auto ref_view = - this->amplitude->create_submatrix({0, this->n}, {0, this->subcols}); + this->amplitude->create_subview({0, this->n}, {0, this->subcols}); auto out = TestFixture::Vec::create(this->exec, in_view->get_size(), this->stride); @@ -438,9 +440,9 @@ TYPED_TEST(Fft, AppliesStrided3DToMultiVector) { using T = typename TestFixture::value_type; auto in_view = - this->amplitude->create_submatrix({0, this->n}, {0, this->subcols}); + this->amplitude->create_subview({0, this->n}, {0, this->subcols}); auto ref_view = - this->frequency3->create_submatrix({0, this->n}, {0, this->subcols}); + this->frequency3->create_subview({0, this->n}, {0, this->subcols}); auto out = TestFixture::Vec::create(this->exec, in_view->get_size(), this->stride); @@ -466,9 +468,9 @@ TYPED_TEST(Fft, AppliesStridedInverse3DToMultiVector) { using T = typename TestFixture::value_type; auto in_view = - this->frequency3->create_submatrix({0, this->n}, {0, this->subcols}); + this->frequency3->create_subview({0, this->n}, {0, this->subcols}); auto ref_view = - this->amplitude->create_submatrix({0, this->n}, {0, this->subcols}); + this->amplitude->create_subview({0, this->n}, {0, this->subcols}); auto out = TestFixture::Vec::create(this->exec, in_view->get_size(), this->stride); diff --git a/reference/test/matrix/hybrid_kernels.cpp b/reference/test/matrix/hybrid_kernels.cpp index c2f83228750..8421ba478d6 100644 --- a/reference/test/matrix/hybrid_kernels.cpp +++ b/reference/test/matrix/hybrid_kernels.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ class Hybrid : public ::testing::Test { using Mtx = gko::matrix::Hybrid; using Vec = gko::matrix::MultiVector; using Csr = gko::matrix::Csr; + using Dense = gko::matrix::Dense; using MixedVec = gko::matrix::MultiVector>; Hybrid() @@ -274,10 +276,10 @@ TYPED_TEST(Hybrid, MovesToPrecision) } -TYPED_TEST(Hybrid, ConvertsToMultiVector) +TYPED_TEST(Hybrid, ConvertsToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx1->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx1->get_executor()); this->mtx1->convert_to(dense_mtx); @@ -289,10 +291,10 @@ TYPED_TEST(Hybrid, ConvertsToMultiVector) } -TYPED_TEST(Hybrid, MovesToMultiVector) +TYPED_TEST(Hybrid, MovesToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx1->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx1->get_executor()); this->mtx1->move_to(dense_mtx); @@ -401,14 +403,14 @@ TYPED_TEST(Hybrid, MovesEmptyToPrecision) } -TYPED_TEST(Hybrid, ConvertsEmptyToMultiVector) +TYPED_TEST(Hybrid, ConvertsEmptyToDense) { using ValueType = typename TestFixture::value_type; using IndexType = typename TestFixture::index_type; using Hybrid = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto other = Hybrid::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); other->convert_to(res); @@ -416,14 +418,14 @@ TYPED_TEST(Hybrid, ConvertsEmptyToMultiVector) } -TYPED_TEST(Hybrid, MovesEmptyToMultiVector) +TYPED_TEST(Hybrid, MovesEmptyToDense) { using ValueType = typename TestFixture::value_type; using IndexType = typename TestFixture::index_type; using Hybrid = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto other = Hybrid::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); other->move_to(res); @@ -569,12 +571,12 @@ TYPED_TEST(Hybrid, ApplyWithStrideFailsOnWrongNumberOfCols) } -TYPED_TEST(Hybrid, ConvertsWithStrideToMultiVector) +TYPED_TEST(Hybrid, ConvertsWithStrideToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx2->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx2->get_executor()); // clang-format off - auto dense_other = gko::initialize( + auto dense_other = gko::initialize( 4, {{1.0, 3.0, 2.0}, {0.0, 5.0, 0.0}}, this->exec); // clang-format on @@ -589,10 +591,10 @@ TYPED_TEST(Hybrid, ConvertsWithStrideToMultiVector) } -TYPED_TEST(Hybrid, MovesWithStrideToMultiVector) +TYPED_TEST(Hybrid, MovesWithStrideToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx2->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx2->get_executor()); this->mtx2->move_to(dense_mtx); diff --git a/reference/test/matrix/identity.cpp b/reference/test/matrix/identity.cpp index 25dcd98a6e2..adddd3c04c7 100644 --- a/reference/test/matrix/identity.cpp +++ b/reference/test/matrix/identity.cpp @@ -4,6 +4,7 @@ #include +#include #include #include diff --git a/reference/test/matrix/multivector_kernels.cpp b/reference/test/matrix/multivector_kernels.cpp index dc64d07dabe..82e81b48097 100644 --- a/reference/test/matrix/multivector_kernels.cpp +++ b/reference/test/matrix/multivector_kernels.cpp @@ -14,16 +14,10 @@ #include #include #include -#include -#include -#include -#include -#include +#include #include #include #include -#include -#include #include "core/test/utils.hpp" @@ -171,133 +165,6 @@ TYPED_TEST(MultiVector, CanBeFilledWithValueForStridedMatrices) } -TYPED_TEST(MultiVector, AppliesToMultiVector) -{ - using T = typename TestFixture::value_type; - T in_stride{-1}; - this->mtx3->get_values()[3] = in_stride; - - this->mtx2->apply(this->mtx1, this->mtx3); - - EXPECT_EQ(this->mtx3->at(0, 0), T{-0.5}); - EXPECT_EQ(this->mtx3->at(0, 1), T{-0.5}); - EXPECT_EQ(this->mtx3->at(0, 2), T{-0.5}); - EXPECT_EQ(this->mtx3->at(1, 0), T{1.0}); - EXPECT_EQ(this->mtx3->at(1, 1), T{1.0}); - EXPECT_EQ(this->mtx3->at(1, 2), T{1.0}); - ASSERT_EQ(this->mtx3->get_values()[3], in_stride); -} - - -TYPED_TEST(MultiVector, AppliesToMixedMultiVector) -{ - using MixedMtx = typename TestFixture::MixedMtx; - using MixedT = typename MixedMtx::value_type; - auto mmtx1 = MixedMtx::create(this->exec); - auto mmtx3 = MixedMtx::create(this->exec); - this->mtx1->convert_to(mmtx1); - this->mtx3->convert_to(mmtx3); - - this->mtx2->apply(mmtx1, mmtx3); - - EXPECT_EQ(mmtx3->at(0, 0), MixedT{-0.5}); - EXPECT_EQ(mmtx3->at(0, 1), MixedT{-0.5}); - EXPECT_EQ(mmtx3->at(0, 2), MixedT{-0.5}); - EXPECT_EQ(mmtx3->at(1, 0), MixedT{1.0}); - EXPECT_EQ(mmtx3->at(1, 1), MixedT{1.0}); - ASSERT_EQ(mmtx3->at(1, 2), MixedT{1.0}); -} - - -TYPED_TEST(MultiVector, AppliesLinearCombinationToMultiVector) -{ - using Mtx = typename TestFixture::Mtx; - using T = typename TestFixture::value_type; - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({2.0}, this->exec); - T in_stride{-1}; - this->mtx3->get_values()[3] = in_stride; - - this->mtx2->apply(alpha, this->mtx1, beta, this->mtx3); - - EXPECT_EQ(this->mtx3->at(0, 0), T{2.5}); - EXPECT_EQ(this->mtx3->at(0, 1), T{4.5}); - EXPECT_EQ(this->mtx3->at(0, 2), T{6.5}); - EXPECT_EQ(this->mtx3->at(1, 0), T{0.0}); - EXPECT_EQ(this->mtx3->at(1, 1), T{2.0}); - EXPECT_EQ(this->mtx3->at(1, 2), T{4.0}); - ASSERT_EQ(this->mtx3->get_values()[3], in_stride); -} - - -TYPED_TEST(MultiVector, AppliesLinearCombinationToMultiVectorWithZeroBetaNan) -{ - using Mtx = typename TestFixture::Mtx; - using T = typename TestFixture::value_type; - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({0.0}, this->exec); - this->mtx3->fill(gko::nan()); - - this->mtx2->apply(alpha, this->mtx1, beta, this->mtx3); - - EXPECT_EQ(this->mtx3->at(0, 0), T{0.5}); - EXPECT_EQ(this->mtx3->at(0, 1), T{0.5}); - EXPECT_EQ(this->mtx3->at(0, 2), T{0.5}); - EXPECT_EQ(this->mtx3->at(1, 0), T{-1.0}); - EXPECT_EQ(this->mtx3->at(1, 1), T{-1.0}); - EXPECT_EQ(this->mtx3->at(1, 2), T{-1.0}); -} - - -TYPED_TEST(MultiVector, AppliesLinearCombinationToMixedMultiVector) -{ - using MixedMtx = typename TestFixture::MixedMtx; - using MixedT = typename MixedMtx::value_type; - auto mmtx1 = MixedMtx::create(this->exec); - auto mmtx3 = MixedMtx::create(this->exec); - this->mtx1->convert_to(mmtx1); - this->mtx3->convert_to(mmtx3); - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({2.0}, this->exec); - - this->mtx2->apply(alpha, mmtx1, beta, mmtx3); - - EXPECT_EQ(mmtx3->at(0, 0), MixedT{2.5}); - EXPECT_EQ(mmtx3->at(0, 1), MixedT{4.5}); - EXPECT_EQ(mmtx3->at(0, 2), MixedT{6.5}); - EXPECT_EQ(mmtx3->at(1, 0), MixedT{0.0}); - EXPECT_EQ(mmtx3->at(1, 1), MixedT{2.0}); - ASSERT_EQ(mmtx3->at(1, 2), MixedT{4.0}); -} - - -TYPED_TEST(MultiVector, ApplyFailsOnWrongInnerDimension) -{ - using Mtx = typename TestFixture::Mtx; - auto res = Mtx::create(this->exec, gko::dim<2>{2}); - - ASSERT_THROW(this->mtx2->apply(this->mtx1, res), gko::DimensionMismatch); -} - - -TYPED_TEST(MultiVector, ApplyFailsOnWrongNumberOfRows) -{ - using Mtx = typename TestFixture::Mtx; - auto res = Mtx::create(this->exec, gko::dim<2>{3}); - - ASSERT_THROW(this->mtx1->apply(this->mtx2, res), gko::DimensionMismatch); -} - - -TYPED_TEST(MultiVector, ApplyFailsOnWrongNumberOfCols) -{ - using Mtx = typename TestFixture::Mtx; - auto res = Mtx::create(this->exec, gko::dim<2>{2}, 3); - - ASSERT_THROW(this->mtx1->apply(this->mtx2, res), gko::DimensionMismatch); -} - - TYPED_TEST(MultiVector, ScalesData) { using Mtx = typename TestFixture::Mtx; @@ -943,89 +810,6 @@ TYPED_TEST(MultiVector, } -TYPED_TEST(MultiVector, ExtractsDiagonalFromSquareMatrix) -{ - using T = typename TestFixture::value_type; - - auto diag = this->mtx5->extract_diagonal(); - - ASSERT_EQ(diag->get_size()[0], 3); - ASSERT_EQ(diag->get_size()[1], 3); - ASSERT_EQ(diag->get_values()[0], T{1.}); - ASSERT_EQ(diag->get_values()[1], T{2.}); - ASSERT_EQ(diag->get_values()[2], T{1.2}); -} - - -TYPED_TEST(MultiVector, ExtractsDiagonalFromTallSkinnyMatrix) -{ - using T = typename TestFixture::value_type; - - auto diag = this->mtx4->extract_diagonal(); - - ASSERT_EQ(diag->get_size()[0], 2); - ASSERT_EQ(diag->get_size()[1], 2); - ASSERT_EQ(diag->get_values()[0], T{1.}); - ASSERT_EQ(diag->get_values()[1], T{5.}); -} - - -TYPED_TEST(MultiVector, ExtractsDiagonalFromShortFatMatrix) -{ - using T = typename TestFixture::value_type; - - auto diag = this->mtx8->extract_diagonal(); - - ASSERT_EQ(diag->get_size()[0], 2); - ASSERT_EQ(diag->get_size()[1], 2); - ASSERT_EQ(diag->get_values()[0], T{1.}); - ASSERT_EQ(diag->get_values()[1], T{2.}); -} - - -TYPED_TEST(MultiVector, ExtractsDiagonalFromSquareMatrixIntoDiagonal) -{ - using T = typename TestFixture::value_type; - auto diag = gko::matrix::Diagonal::create(this->exec, 3); - - this->mtx5->extract_diagonal(diag); - - ASSERT_EQ(diag->get_size()[0], 3); - ASSERT_EQ(diag->get_size()[1], 3); - ASSERT_EQ(diag->get_values()[0], T{1.}); - ASSERT_EQ(diag->get_values()[1], T{2.}); - ASSERT_EQ(diag->get_values()[2], T{1.2}); -} - - -TYPED_TEST(MultiVector, ExtractsDiagonalFromTallSkinnyMatrixIntoDiagonal) -{ - using T = typename TestFixture::value_type; - auto diag = gko::matrix::Diagonal::create(this->exec, 2); - - this->mtx4->extract_diagonal(diag); - - ASSERT_EQ(diag->get_size()[0], 2); - ASSERT_EQ(diag->get_size()[1], 2); - ASSERT_EQ(diag->get_values()[0], T{1.}); - ASSERT_EQ(diag->get_values()[1], T{5.}); -} - - -TYPED_TEST(MultiVector, ExtractsDiagonalFromShortFatMatrixIntoDiagonal) -{ - using T = typename TestFixture::value_type; - auto diag = gko::matrix::Diagonal::create(this->exec, 2); - - this->mtx8->extract_diagonal(diag); - - ASSERT_EQ(diag->get_size()[0], 2); - ASSERT_EQ(diag->get_size()[1], 2); - ASSERT_EQ(diag->get_values()[0], T{1.}); - ASSERT_EQ(diag->get_values()[1], T{2.}); -} - - TYPED_TEST(MultiVector, InplaceAbsolute) { using T = typename TestFixture::value_type; @@ -1105,116 +889,6 @@ TYPED_TEST(MultiVector, OutplaceSubmatrixAbsoluteIntoMultiVector) } -TYPED_TEST(MultiVector, AppliesToComplex) -{ - using value_type = typename TestFixture::value_type; - using complex_type = gko::to_complex; - using Vec = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - auto b = - gko::initialize({{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}}, - {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}, - {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, - exec); - auto x = Vec::create(exec, gko::dim<2>{2, 2}); - - this->mtx1->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{complex_type{14.0, 16.0}, complex_type{20.0, 22.0}}, - {complex_type{17.0, 19.0}, complex_type{24.5, 26.5}}}), - 0.0); -} - - -TYPED_TEST(MultiVector, AppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using Vec = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, - exec); - auto x = Vec::create(exec, gko::dim<2>{2, 2}); - - this->mtx1->apply(b, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{14.0, 16.0}, mixed_complex_type{20.0, 22.0}}, - {mixed_complex_type{17.0, 19.0}, mixed_complex_type{24.5, 26.5}}}), - 0.0); -} - - -TYPED_TEST(MultiVector, AdvancedAppliesToComplex) -{ - using value_type = typename TestFixture::value_type; - using complex_type = gko::to_complex; - using MultiVector = gko::matrix::MultiVector; - using MultiVectorComplex = gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - auto b = gko::initialize( - {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}}, - {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}, - {complex_type{3.0, 4.0}, complex_type{4.0, 5.0}}}, - exec); - auto x = gko::initialize( - {{complex_type{1.0, 0.0}, complex_type{2.0, 1.0}}, - {complex_type{2.0, 2.0}, complex_type{3.0, 3.0}}}, - exec); - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({2.0}, this->exec); - - this->mtx1->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{complex_type{-12.0, -16.0}, complex_type{-16.0, -20.0}}, - {complex_type{-13.0, -15.0}, complex_type{-18.5, -20.5}}}), - 0.0); -} - - -TYPED_TEST(MultiVector, AdvancedAppliesToMixedComplex) -{ - using mixed_value_type = - gko::next_precision; - using mixed_complex_type = gko::to_complex; - using MixedMultiVector = gko::matrix::MultiVector; - using MixedMultiVectorComplex = - gko::matrix::MultiVector; - auto exec = gko::ReferenceExecutor::create(); - - auto b = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}, - {mixed_complex_type{3.0, 4.0}, mixed_complex_type{4.0, 5.0}}}, - exec); - auto x = gko::initialize( - {{mixed_complex_type{1.0, 0.0}, mixed_complex_type{2.0, 1.0}}, - {mixed_complex_type{2.0, 2.0}, mixed_complex_type{3.0, 3.0}}}, - exec); - auto alpha = gko::initialize({-1.0}, this->exec); - auto beta = gko::initialize({2.0}, this->exec); - - this->mtx1->apply(alpha, b, beta, x); - - GKO_ASSERT_MTX_NEAR( - x, - l({{mixed_complex_type{-12.0, -16.0}, mixed_complex_type{-16.0, -20.0}}, - {mixed_complex_type{-13.0, -15.0}, - mixed_complex_type{-18.5, -20.5}}}), - 0.0); -} - - TYPED_TEST(MultiVector, MakeComplex) { using T = typename TestFixture::value_type; @@ -1365,21 +1039,6 @@ TYPED_TEST(MultiVector, MakeTemporaryConversionConstDoesntConvertBack) } -TYPED_TEST(MultiVector, ScaleAddIdentityRectangular) -{ - using T = typename TestFixture::value_type; - using Vec = typename TestFixture::Mtx; - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( - {I{2.0, 0.0}, I{1.0, 2.5}, I{0.0, -4.0}}, this->exec); - - b->add_scaled_identity(alpha, beta); - - GKO_ASSERT_MTX_NEAR(b, l({{0.0, 0.0}, {-1.0, -0.5}, {0.0, 4.0}}), 0.0); -} - - template class MultiVectorWithIndexType : public MultiVector< @@ -1429,657 +1088,22 @@ TYPED_TEST_SUITE(MultiVectorWithIndexType, gko::test::ValueIndexTypes, PairTypenameNameGenerator); -template -void assert_coo_eq_mtx4(const gko::matrix::Coo* coo_mtx) -{ - auto v = coo_mtx->get_const_values(); - auto c = coo_mtx->get_const_col_idxs(); - auto r = coo_mtx->get_const_row_idxs(); - - ASSERT_EQ(coo_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(coo_mtx->get_num_stored_elements(), 4); - EXPECT_EQ(r[0], 0); - EXPECT_EQ(r[1], 0); - EXPECT_EQ(r[2], 0); - EXPECT_EQ(r[3], 1); - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[2], 2); - EXPECT_EQ(c[3], 1); - EXPECT_EQ(v[0], ValueType{1.0}); - EXPECT_EQ(v[1], ValueType{3.0}); - EXPECT_EQ(v[2], ValueType{2.0}); - EXPECT_EQ(v[3], ValueType{5.0}); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsToCoo) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Coo = typename gko::matrix::Coo; - auto coo_mtx = Coo::create(this->mtx4->get_executor()); - - this->mtx4->convert_to(coo_mtx); - - assert_coo_eq_mtx4(coo_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesToCoo) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Coo = typename gko::matrix::Coo; - auto coo_mtx = Coo::create(this->mtx4->get_executor()); - - this->mtx4->move_to(coo_mtx); - - assert_coo_eq_mtx4(coo_mtx.get()); -} - - -template -void assert_csr_eq_mtx4(const gko::matrix::Csr* csr_mtx) -{ - auto v = csr_mtx->get_const_values(); - auto c = csr_mtx->get_const_col_idxs(); - auto r = csr_mtx->get_const_row_ptrs(); - ASSERT_EQ(csr_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(csr_mtx->get_num_stored_elements(), 4); - EXPECT_EQ(r[0], 0); - EXPECT_EQ(r[1], 3); - EXPECT_EQ(r[2], 4); - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[2], 2); - EXPECT_EQ(c[3], 1); - EXPECT_EQ(v[0], ValueType{1.0}); - EXPECT_EQ(v[1], ValueType{3.0}); - EXPECT_EQ(v[2], ValueType{2.0}); - EXPECT_EQ(v[3], ValueType{5.0}); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsToCsr) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Csr = typename gko::matrix::Csr; - auto csr_s_classical = gko::matrix::csr::spmv_strategy::classical; - auto csr_s_merge = gko::matrix::csr::spmv_strategy::merge_path; - auto csr_mtx_c = Csr::create(this->mtx4->get_executor(), csr_s_classical); - auto csr_mtx_m = Csr::create(this->mtx4->get_executor(), csr_s_merge); - - this->mtx4->convert_to(csr_mtx_c); - this->mtx4->convert_to(csr_mtx_m); - - assert_csr_eq_mtx4(csr_mtx_c.get()); - ASSERT_EQ(csr_mtx_c->get_strategy(), - gko::matrix::csr::spmv_strategy::classical); - GKO_ASSERT_MTX_NEAR(csr_mtx_c, csr_mtx_m, 0.0); - ASSERT_EQ(csr_mtx_m->get_strategy(), - gko::matrix::csr::spmv_strategy::merge_path); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesToCsr) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Csr = typename gko::matrix::Csr; - auto csr_s_classical = gko::matrix::csr::spmv_strategy::classical; - auto csr_s_merge = gko::matrix::csr::spmv_strategy::merge_path; - auto csr_mtx_c = Csr::create(this->mtx4->get_executor(), csr_s_classical); - auto csr_mtx_m = Csr::create(this->mtx4->get_executor(), csr_s_merge); - auto mtx_clone = this->mtx4->clone(); - - this->mtx4->move_to(csr_mtx_c); - mtx_clone->move_to(csr_mtx_m); - - assert_csr_eq_mtx4(csr_mtx_c.get()); - ASSERT_EQ(csr_mtx_c->get_strategy(), - gko::matrix::csr::spmv_strategy::classical); - GKO_ASSERT_MTX_NEAR(csr_mtx_c, csr_mtx_m, 0.0); - ASSERT_EQ(csr_mtx_m->get_strategy(), - gko::matrix::csr::spmv_strategy::merge_path); -} - - -template -void assert_sparsity_csr_eq_mtx4( - const gko::matrix::SparsityCsr* sparsity_csr_mtx) -{ - auto v = sparsity_csr_mtx->get_const_value(); - auto c = sparsity_csr_mtx->get_const_col_idxs(); - auto r = sparsity_csr_mtx->get_const_row_ptrs(); - - ASSERT_EQ(sparsity_csr_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(sparsity_csr_mtx->get_num_nonzeros(), 4); - EXPECT_EQ(r[0], 0); - EXPECT_EQ(r[1], 3); - EXPECT_EQ(r[2], 4); - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[2], 2); - EXPECT_EQ(c[3], 1); - EXPECT_EQ(v[0], ValueType{1.0}); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsToSparsityCsr) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using SparsityCsr = - typename gko::matrix::SparsityCsr; - auto sparsity_csr_mtx = SparsityCsr::create(this->mtx4->get_executor()); - - this->mtx4->convert_to(sparsity_csr_mtx); - - assert_sparsity_csr_eq_mtx4(sparsity_csr_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesToSparsityCsr) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using SparsityCsr = - typename gko::matrix::SparsityCsr; - auto sparsity_csr_mtx = SparsityCsr::create(this->mtx4->get_executor()); - - this->mtx4->move_to(sparsity_csr_mtx); - - assert_sparsity_csr_eq_mtx4(sparsity_csr_mtx.get()); -} - - -template -void assert_ell_eq_mtx6(const gko::matrix::Ell* ell_mtx) -{ - auto v = ell_mtx->get_const_values(); - auto c = ell_mtx->get_const_col_idxs(); - - ASSERT_EQ(ell_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(ell_mtx->get_num_stored_elements_per_row(), 2); - ASSERT_EQ(ell_mtx->get_num_stored_elements(), 4); - ASSERT_EQ(ell_mtx->get_stride(), 2); - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[2], 1); - EXPECT_EQ(c[3], gko::invalid_index()); - EXPECT_EQ(v[0], ValueType{1.0}); - EXPECT_EQ(v[1], ValueType{1.5}); - EXPECT_EQ(v[2], ValueType{2.0}); - EXPECT_EQ(v[3], ValueType{0.0}); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsToEll) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Ell = typename gko::matrix::Ell; - auto ell_mtx = Ell::create(this->mtx6->get_executor()); - - this->mtx6->convert_to(ell_mtx); - - assert_ell_eq_mtx6(ell_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesToEll) +TYPED_TEST(MultiVector, ConvertsEmptyToPrecision) { - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Ell = typename gko::matrix::Ell; - auto ell_mtx = Ell::create(this->mtx6->get_executor()); - - this->mtx6->move_to(ell_mtx); - - assert_ell_eq_mtx6(ell_mtx.get()); -} - + using MultiVector = typename TestFixture::Mtx; + using T = typename TestFixture::value_type; + using OtherT = typename gko::next_precision; + using OtherMultiVector = typename gko::matrix::MultiVector; + auto empty = OtherMultiVector::create(this->exec); + auto res = MultiVector::create(this->exec); -template -void assert_strided_ell_eq_mtx6( - const gko::matrix::Ell* ell_mtx) -{ - constexpr auto invalid_index = gko::invalid_index(); - auto v = ell_mtx->get_const_values(); - auto c = ell_mtx->get_const_col_idxs(); + empty->convert_to(res); - ASSERT_EQ(ell_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(ell_mtx->get_num_stored_elements_per_row(), 2); - ASSERT_EQ(ell_mtx->get_num_stored_elements(), 6); - ASSERT_EQ(ell_mtx->get_stride(), 3); - // only check the actual matrix entries. - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[3], 1); - EXPECT_EQ(c[4], invalid_index); - EXPECT_EQ(v[0], ValueType{1.0}); - EXPECT_EQ(v[1], ValueType{1.5}); - EXPECT_EQ(v[3], ValueType{2.0}); - EXPECT_EQ(v[4], ValueType{0.0}); + ASSERT_FALSE(res->get_size()); } -TYPED_TEST(MultiVectorWithIndexType, ConvertsToEllWithStride) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Ell = typename gko::matrix::Ell; - auto ell_mtx = - Ell::create(this->mtx6->get_executor(), gko::dim<2>{2, 3}, 2, 3); - - this->mtx6->convert_to(ell_mtx); - - assert_strided_ell_eq_mtx6(ell_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesToEllWithStride) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Ell = typename gko::matrix::Ell; - auto ell_mtx = - Ell::create(this->mtx6->get_executor(), gko::dim<2>{2, 3}, 2, 3); - - this->mtx6->move_to(ell_mtx); - - assert_strided_ell_eq_mtx6(ell_mtx.get()); -} - - -template -void assert_hybrid_auto_eq_mtx4( - const gko::matrix::Hybrid* hybrid_mtx) -{ - auto v = hybrid_mtx->get_const_coo_values(); - auto c = hybrid_mtx->get_const_coo_col_idxs(); - auto r = hybrid_mtx->get_const_coo_row_idxs(); - auto n = hybrid_mtx->get_ell_num_stored_elements_per_row(); - auto p = hybrid_mtx->get_ell_stride(); - - ASSERT_EQ(hybrid_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(hybrid_mtx->get_ell_num_stored_elements(), 0); - ASSERT_EQ(hybrid_mtx->get_coo_num_stored_elements(), 4); - EXPECT_EQ(n, 0); - EXPECT_EQ(p, 2); - EXPECT_EQ(r[0], 0); - EXPECT_EQ(r[1], 0); - EXPECT_EQ(r[2], 0); - EXPECT_EQ(r[3], 1); - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[2], 2); - EXPECT_EQ(c[3], 1); - EXPECT_EQ(v[0], ValueType{1.0}); - EXPECT_EQ(v[1], ValueType{3.0}); - EXPECT_EQ(v[2], ValueType{2.0}); - EXPECT_EQ(v[3], ValueType{5.0}); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesToHybridAutomatically) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Hybrid = typename gko::matrix::Hybrid; - auto hybrid_mtx = Hybrid::create(this->mtx4->get_executor()); - - this->mtx4->move_to(hybrid_mtx); - - assert_hybrid_auto_eq_mtx4(hybrid_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsToHybridAutomatically) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Hybrid = typename gko::matrix::Hybrid; - auto hybrid_mtx = Hybrid::create(this->mtx4->get_executor()); - - this->mtx4->convert_to(hybrid_mtx); - - assert_hybrid_auto_eq_mtx4(hybrid_mtx.get()); -} - - -template -void assert_hybrid_strided_eq_mtx4( - const gko::matrix::Hybrid* hybrid_mtx) -{ - auto v = hybrid_mtx->get_const_coo_values(); - auto c = hybrid_mtx->get_const_coo_col_idxs(); - auto r = hybrid_mtx->get_const_coo_row_idxs(); - auto n = hybrid_mtx->get_ell_num_stored_elements_per_row(); - auto p = hybrid_mtx->get_ell_stride(); - - ASSERT_EQ(hybrid_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(hybrid_mtx->get_ell_num_stored_elements(), 0); - ASSERT_EQ(hybrid_mtx->get_coo_num_stored_elements(), 4); - EXPECT_EQ(n, 0); - EXPECT_EQ(p, 3); - EXPECT_EQ(r[0], 0); - EXPECT_EQ(r[1], 0); - EXPECT_EQ(r[2], 0); - EXPECT_EQ(r[3], 1); - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[2], 2); - EXPECT_EQ(c[3], 1); - EXPECT_EQ(v[0], ValueType{1.0}); - EXPECT_EQ(v[1], ValueType{3.0}); - EXPECT_EQ(v[2], ValueType{2.0}); - EXPECT_EQ(v[3], ValueType{5.0}); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesToHybridWithStrideAutomatically) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Hybrid = typename gko::matrix::Hybrid; - auto hybrid_mtx = - Hybrid::create(this->mtx4->get_executor(), gko::dim<2>{2, 3}, 0, 3); - - this->mtx4->move_to(hybrid_mtx); - - assert_hybrid_strided_eq_mtx4(hybrid_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsToHybridWithStrideAutomatically) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Hybrid = typename gko::matrix::Hybrid; - auto hybrid_mtx = - Hybrid::create(this->mtx4->get_executor(), gko::dim<2>{2, 3}, 0, 3); - - this->mtx4->convert_to(hybrid_mtx); - - assert_hybrid_strided_eq_mtx4(hybrid_mtx.get()); -} - - -template -void assert_hybrid_limited_eq_mtx4( - const gko::matrix::Hybrid* hybrid_mtx) -{ - constexpr auto invalid_index = gko::invalid_index(); - auto v = hybrid_mtx->get_const_ell_values(); - auto c = hybrid_mtx->get_const_ell_col_idxs(); - auto n = hybrid_mtx->get_ell_num_stored_elements_per_row(); - auto p = hybrid_mtx->get_ell_stride(); - - ASSERT_EQ(hybrid_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(hybrid_mtx->get_ell_num_stored_elements(), 6); - ASSERT_EQ(hybrid_mtx->get_coo_num_stored_elements(), 1); - EXPECT_EQ(n, 2); - EXPECT_EQ(p, 3); - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[2], invalid_index); - EXPECT_EQ(c[3], 1); - EXPECT_EQ(c[4], invalid_index); - EXPECT_EQ(c[5], invalid_index); - EXPECT_EQ(v[0], ValueType{1.0}); - EXPECT_EQ(v[1], ValueType{5.0}); - EXPECT_EQ(v[2], ValueType{0.0}); - EXPECT_EQ(v[3], ValueType{3.0}); - EXPECT_EQ(v[4], ValueType{0.0}); - EXPECT_EQ(v[5], ValueType{0.0}); - EXPECT_EQ(hybrid_mtx->get_const_coo_values()[0], ValueType{2.0}); - EXPECT_EQ(hybrid_mtx->get_const_coo_row_idxs()[0], 0); - EXPECT_EQ(hybrid_mtx->get_const_coo_col_idxs()[0], 2); -} - - -TYPED_TEST(MultiVectorWithIndexType, - MovesToHybridWithStrideAndCooLengthByColumns2) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Hybrid = typename gko::matrix::Hybrid; - auto hybrid_mtx = - Hybrid::create(this->mtx4->get_executor(), gko::dim<2>{2, 3}, 2, 3, 3, - std::make_shared(2)); - - this->mtx4->move_to(hybrid_mtx); - - assert_hybrid_limited_eq_mtx4(hybrid_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, - ConvertsToHybridWithStrideAndCooLengthByColumns2) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Hybrid = typename gko::matrix::Hybrid; - auto hybrid_mtx = - Hybrid::create(this->mtx4->get_executor(), gko::dim<2>{2, 3}, 2, 3, 3, - std::make_shared(2)); - - this->mtx4->convert_to(hybrid_mtx); - - assert_hybrid_limited_eq_mtx4(hybrid_mtx.get()); -} - - -template -void assert_hybrid_percent_eq_mtx4( - const gko::matrix::Hybrid* hybrid_mtx) -{ - auto v = hybrid_mtx->get_const_ell_values(); - auto c = hybrid_mtx->get_const_ell_col_idxs(); - auto n = hybrid_mtx->get_ell_num_stored_elements_per_row(); - auto p = hybrid_mtx->get_ell_stride(); - auto coo_v = hybrid_mtx->get_const_coo_values(); - auto coo_c = hybrid_mtx->get_const_coo_col_idxs(); - auto coo_r = hybrid_mtx->get_const_coo_row_idxs(); - - ASSERT_EQ(hybrid_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(hybrid_mtx->get_ell_num_stored_elements(), 3); - EXPECT_EQ(n, 1); - EXPECT_EQ(p, 3); - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[2], gko::invalid_index()); - EXPECT_EQ(v[0], ValueType{1.0}); - EXPECT_EQ(v[1], ValueType{5.0}); - EXPECT_EQ(v[2], ValueType{0.0}); - ASSERT_EQ(hybrid_mtx->get_coo_num_stored_elements(), 2); - EXPECT_EQ(coo_v[0], ValueType{3.0}); - EXPECT_EQ(coo_v[1], ValueType{2.0}); - EXPECT_EQ(coo_c[0], 1); - EXPECT_EQ(coo_c[1], 2); - EXPECT_EQ(coo_r[0], 0); - EXPECT_EQ(coo_r[1], 0); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesToHybridWithStrideByPercent40) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Hybrid = typename gko::matrix::Hybrid; - auto hybrid_mtx = - Hybrid::create(this->mtx4->get_executor(), gko::dim<2>{2, 3}, 1, 3, - std::make_shared(0.4)); - - this->mtx4->move_to(hybrid_mtx); - - assert_hybrid_percent_eq_mtx4(hybrid_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsToHybridWithStrideByPercent40) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Hybrid = typename gko::matrix::Hybrid; - auto hybrid_mtx = - Hybrid::create(this->mtx4->get_executor(), gko::dim<2>{2, 3}, 1, 3, - std::make_shared(0.4)); - - this->mtx4->convert_to(hybrid_mtx); - - assert_hybrid_percent_eq_mtx4(hybrid_mtx.get()); -} - - -template -void assert_sellp_eq_mtx7( - const gko::matrix::Sellp* sellp_mtx) -{ - constexpr auto invalid_index = gko::invalid_index(); - auto v = sellp_mtx->get_const_values(); - auto c = sellp_mtx->get_const_col_idxs(); - auto s = sellp_mtx->get_const_slice_sets(); - auto l = sellp_mtx->get_const_slice_lengths(); - - ASSERT_EQ(sellp_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(sellp_mtx->get_total_cols(), 3); - ASSERT_EQ(sellp_mtx->get_num_stored_elements(), - 3 * gko::matrix::default_slice_size); - ASSERT_EQ(sellp_mtx->get_slice_size(), gko::matrix::default_slice_size); - ASSERT_EQ(sellp_mtx->get_stride_factor(), - gko::matrix::default_stride_factor); - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[gko::matrix::default_slice_size], 1); - EXPECT_EQ(c[gko::matrix::default_slice_size + 1], invalid_index); - EXPECT_EQ(c[2 * gko::matrix::default_slice_size], 2); - EXPECT_EQ(c[2 * gko::matrix::default_slice_size + 1], invalid_index); - EXPECT_EQ(v[0], ValueType{1.0}); - EXPECT_EQ(v[1], ValueType{1.5}); - EXPECT_EQ(v[gko::matrix::default_slice_size], ValueType{2.0}); - EXPECT_EQ(v[gko::matrix::default_slice_size + 1], ValueType{0.0}); - EXPECT_EQ(v[2 * gko::matrix::default_slice_size], ValueType{3.0}); - EXPECT_EQ(v[2 * gko::matrix::default_slice_size + 1], ValueType{0.0}); - EXPECT_EQ(s[0], 0); - EXPECT_EQ(s[1], 3); - EXPECT_EQ(l[0], 3); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsToSellp) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Sellp = typename gko::matrix::Sellp; - auto sellp_mtx = Sellp::create(this->mtx7->get_executor()); - - this->mtx7->convert_to(sellp_mtx); - - assert_sellp_eq_mtx7(sellp_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesToSellp) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Sellp = typename gko::matrix::Sellp; - auto sellp_mtx = Sellp::create(this->mtx7->get_executor()); - - this->mtx7->move_to(sellp_mtx); - - assert_sellp_eq_mtx7(sellp_mtx.get()); -} - - -template -void assert_sellp_strided_eq_mtx7( - const gko::matrix::Sellp* sellp_mtx) -{ - constexpr auto invalid_index = gko::invalid_index(); - auto v = sellp_mtx->get_const_values(); - auto c = sellp_mtx->get_const_col_idxs(); - auto s = sellp_mtx->get_const_slice_sets(); - auto l = sellp_mtx->get_const_slice_lengths(); - - ASSERT_EQ(sellp_mtx->get_size(), gko::dim<2>(2, 3)); - ASSERT_EQ(sellp_mtx->get_total_cols(), 4); - ASSERT_EQ(sellp_mtx->get_num_stored_elements(), 8); - ASSERT_EQ(sellp_mtx->get_slice_size(), 2); - ASSERT_EQ(sellp_mtx->get_stride_factor(), 2); - EXPECT_EQ(c[0], 0); - EXPECT_EQ(c[1], 1); - EXPECT_EQ(c[2], 1); - EXPECT_EQ(c[3], invalid_index); - EXPECT_EQ(c[4], 2); - EXPECT_EQ(c[5], invalid_index); - EXPECT_EQ(c[6], invalid_index); - EXPECT_EQ(c[7], invalid_index); - EXPECT_EQ(v[0], ValueType{1.0}); - EXPECT_EQ(v[1], ValueType{1.5}); - EXPECT_EQ(v[2], ValueType{2.0}); - EXPECT_EQ(v[3], ValueType{0.0}); - EXPECT_EQ(v[4], ValueType{3.0}); - EXPECT_EQ(v[5], ValueType{0.0}); - EXPECT_EQ(v[6], ValueType{0.0}); - EXPECT_EQ(v[7], ValueType{0.0}); - EXPECT_EQ(s[0], 0); - EXPECT_EQ(s[1], 4); - EXPECT_EQ(l[0], 4); -} - - -TYPED_TEST(MultiVectorWithIndexType, - ConvertsToSellpWithSliceSizeAndStrideFactor) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Sellp = typename gko::matrix::Sellp; - auto sellp_mtx = - Sellp::create(this->mtx7->get_executor(), gko::dim<2>{}, 2, 2, 0); - - this->mtx7->convert_to(sellp_mtx); - - assert_sellp_strided_eq_mtx7(sellp_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesToSellpWithSliceSizeAndStrideFactor) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Sellp = typename gko::matrix::Sellp; - auto sellp_mtx = - Sellp::create(this->mtx7->get_executor(), gko::dim<2>{}, 2, 2, 0); - - this->mtx7->move_to(sellp_mtx); - - assert_sellp_strided_eq_mtx7(sellp_mtx.get()); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsToAndFromSellpWithMoreThanOneSlice) -{ - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Mtx = typename TestFixture::Mtx; - using Sellp = typename gko::matrix::Sellp; - auto x = this->template gen_mtx(65, 25); - - auto sellp_mtx = Sellp::create(this->exec); - auto multivector_mtx = Mtx::create(this->exec); - x->convert_to(sellp_mtx); - sellp_mtx->convert_to(multivector_mtx); - - GKO_ASSERT_MTX_NEAR(multivector_mtx, x, 0.0); -} - - -TYPED_TEST(MultiVector, ConvertsEmptyToPrecision) +TYPED_TEST(MultiVector, MovesEmptyToPrecision) { using MultiVector = typename TestFixture::Mtx; using T = typename TestFixture::value_type; @@ -2088,223 +1112,8 @@ TYPED_TEST(MultiVector, ConvertsEmptyToPrecision) auto empty = OtherMultiVector::create(this->exec); auto res = MultiVector::create(this->exec); - empty->convert_to(res); - - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVector, MovesEmptyToPrecision) -{ - using MultiVector = typename TestFixture::Mtx; - using T = typename TestFixture::value_type; - using OtherT = typename gko::next_precision; - using OtherMultiVector = typename gko::matrix::MultiVector; - auto empty = OtherMultiVector::create(this->exec); - auto res = MultiVector::create(this->exec); - - empty->move_to(res); - - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsEmptyToCoo) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Coo = typename gko::matrix::Coo; - auto empty = MultiVector::create(this->exec); - auto res = Coo::create(this->exec); - - empty->convert_to(res); - - ASSERT_EQ(res->get_num_stored_elements(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesEmptyToCoo) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Coo = typename gko::matrix::Coo; - auto empty = MultiVector::create(this->exec); - auto res = Coo::create(this->exec); - - empty->move_to(res); - - ASSERT_EQ(res->get_num_stored_elements(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsEmptyMatrixToCsr) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Csr = typename gko::matrix::Csr; - auto empty = MultiVector::create(this->exec); - auto res = Csr::create(this->exec); - - empty->convert_to(res); - - ASSERT_EQ(res->get_num_stored_elements(), 0); - ASSERT_EQ(*res->get_const_row_ptrs(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesEmptyMatrixToCsr) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Csr = typename gko::matrix::Csr; - auto empty = MultiVector::create(this->exec); - auto res = Csr::create(this->exec); - - empty->move_to(res); - - ASSERT_EQ(res->get_num_stored_elements(), 0); - ASSERT_EQ(*res->get_const_row_ptrs(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsEmptyToSparsityCsr) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using SparsityCsr = - typename gko::matrix::SparsityCsr; - auto empty = MultiVector::create(this->exec); - auto res = SparsityCsr::create(this->exec); - - empty->convert_to(res); - - ASSERT_EQ(res->get_num_nonzeros(), 0); - ASSERT_EQ(*res->get_const_row_ptrs(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesEmptyToSparsityCsr) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using SparsityCsr = - typename gko::matrix::SparsityCsr; - auto empty = MultiVector::create(this->exec); - auto res = SparsityCsr::create(this->exec); - - empty->move_to(res); - - ASSERT_EQ(res->get_num_nonzeros(), 0); - ASSERT_EQ(*res->get_const_row_ptrs(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsEmptyToEll) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Ell = typename gko::matrix::Ell; - auto empty = MultiVector::create(this->exec); - auto res = Ell::create(this->exec); - - empty->convert_to(res); - - ASSERT_EQ(res->get_num_stored_elements(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesEmptyToEll) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Ell = typename gko::matrix::Ell; - auto empty = MultiVector::create(this->exec); - auto res = Ell::create(this->exec); - - empty->move_to(res); - - ASSERT_EQ(res->get_num_stored_elements(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsEmptyToHybrid) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Hybrid = typename gko::matrix::Hybrid; - auto empty = MultiVector::create(this->exec); - auto res = Hybrid::create(this->exec); - - empty->convert_to(res); - - ASSERT_EQ(res->get_num_stored_elements(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesEmptyToHybrid) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Hybrid = typename gko::matrix::Hybrid; - auto empty = MultiVector::create(this->exec); - auto res = Hybrid::create(this->exec); - - empty->move_to(res); - - ASSERT_EQ(res->get_num_stored_elements(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, ConvertsEmptyToSellp) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Sellp = typename gko::matrix::Sellp; - auto empty = MultiVector::create(this->exec); - auto res = Sellp::create(this->exec); - - empty->convert_to(res); - - ASSERT_EQ(res->get_num_stored_elements(), 0); - ASSERT_EQ(*res->get_const_slice_sets(), 0); - ASSERT_FALSE(res->get_size()); -} - - -TYPED_TEST(MultiVectorWithIndexType, MovesEmptyToSellp) -{ - using MultiVector = typename TestFixture::Mtx; - using value_type = typename TestFixture::value_type; - using index_type = typename TestFixture::index_type; - using Sellp = typename gko::matrix::Sellp; - auto empty = MultiVector::create(this->exec); - auto res = Sellp::create(this->exec); - empty->move_to(res); - ASSERT_EQ(res->get_num_stored_elements(), 0); - ASSERT_EQ(*res->get_const_slice_sets(), 0); ASSERT_FALSE(res->get_size()); } diff --git a/reference/test/matrix/permutation.cpp b/reference/test/matrix/permutation.cpp index 25261ae279b..467e5b5b4dc 100644 --- a/reference/test/matrix/permutation.cpp +++ b/reference/test/matrix/permutation.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -29,11 +30,11 @@ class Permutation : public ::testing::Test { Permutation() : exec(gko::ReferenceExecutor::create()) {} - std::unique_ptr> ref_combine( + std::unique_ptr> ref_combine( const gko::matrix::Permutation* first, const gko::matrix::Permutation* second) { - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; const auto exec = first->get_executor(); gko::matrix_data first_perm_data; gko::matrix_data second_perm_data; @@ -44,7 +45,8 @@ class Permutation : public ::testing::Test { first_mtx->read(first_perm_data); second_mtx->read(second_perm_data); auto combined_mtx = first_mtx->clone(); - second_mtx->apply(first_mtx, combined_mtx); + second_mtx->apply(first_mtx->as_const_multivector_view(), + combined_mtx->as_multivector_view()); return combined_mtx; } @@ -196,16 +198,4 @@ TYPED_TEST(Permutation, AdvancedAppliesRowPermutationToMultiVector) } -TYPED_TEST(Permutation, ApplyFailsWithNonMultiVectorMatrix) -{ - using index_type = typename TestFixture::index_type; - using T = typename TestFixture::value_type; - auto mtx = gko::matrix::Csr::create(this->exec); - auto mtx2 = mtx->clone(); - auto perm = gko::matrix::Permutation::create(this->exec); - - ASSERT_THROW(perm->apply(mtx, mtx2), gko::NotSupported); -} - - } // namespace diff --git a/reference/test/matrix/scaled_permutation.cpp b/reference/test/matrix/scaled_permutation.cpp index 866c1ea35be..34ff8d9942d 100644 --- a/reference/test/matrix/scaled_permutation.cpp +++ b/reference/test/matrix/scaled_permutation.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -25,6 +26,7 @@ class ScaledPermutation : public ::testing::Test { using index_type = typename std::tuple_element<1, decltype(ValueIndexType())>::type; using Vec = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Mtx = gko::matrix::ScaledPermutation; ScaledPermutation() : exec(gko::ReferenceExecutor::create()) @@ -37,19 +39,20 @@ class ScaledPermutation : public ::testing::Test { gko::array{this->exec, {1, 0}}); } - std::unique_ptr ref_combine(const Mtx* first, const Mtx* second) + std::unique_ptr ref_combine(const Mtx* first, const Mtx* second) { const auto exec = first->get_executor(); gko::matrix_data first_perm_data; gko::matrix_data second_perm_data; first->write(first_perm_data); second->write(second_perm_data); - const auto first_mtx = Vec::create(exec); - const auto second_mtx = Vec::create(exec); + const auto first_mtx = Dense::create(exec); + const auto second_mtx = Dense::create(exec); first_mtx->read(first_perm_data); second_mtx->read(second_perm_data); auto combined_mtx = first_mtx->clone(); - second_mtx->apply(first_mtx, combined_mtx); + second_mtx->apply(first_mtx->as_const_multivector_view(), + combined_mtx->as_multivector_view()); return combined_mtx; } diff --git a/reference/test/matrix/sellp_kernels.cpp b/reference/test/matrix/sellp_kernels.cpp index 280500f8f14..187b344d4ab 100644 --- a/reference/test/matrix/sellp_kernels.cpp +++ b/reference/test/matrix/sellp_kernels.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ class Sellp : public ::testing::Test { typename std::tuple_element<1, decltype(ValueIndexType())>::type; using Mtx = gko::matrix::Sellp; using Csr = gko::matrix::Csr; + using Dense = gko::matrix::Dense; using Vec = gko::matrix::MultiVector; Sellp() @@ -246,10 +248,10 @@ TYPED_TEST(Sellp, MovesToPrecision) } -TYPED_TEST(Sellp, ConvertsToMultiVector) +TYPED_TEST(Sellp, ConvertsToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx1->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx1->get_executor()); this->mtx1->convert_to(dense_mtx); @@ -261,10 +263,10 @@ TYPED_TEST(Sellp, ConvertsToMultiVector) } -TYPED_TEST(Sellp, MovesToMultiVector) +TYPED_TEST(Sellp, MovesToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx1->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx1->get_executor()); this->mtx1->move_to(dense_mtx); @@ -363,14 +365,14 @@ TYPED_TEST(Sellp, MovesEmptyToPrecision) } -TYPED_TEST(Sellp, ConvertsEmptyToMultiVector) +TYPED_TEST(Sellp, ConvertsEmptyToDense) { using ValueType = typename TestFixture::value_type; using IndexType = typename TestFixture::index_type; using Sellp = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto empty = Sellp::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); empty->convert_to(res); @@ -378,14 +380,14 @@ TYPED_TEST(Sellp, ConvertsEmptyToMultiVector) } -TYPED_TEST(Sellp, MovesEmptyToMultiVector) +TYPED_TEST(Sellp, MovesEmptyToDense) { using ValueType = typename TestFixture::value_type; using IndexType = typename TestFixture::index_type; using Sellp = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; auto empty = Sellp::create(this->exec); - auto res = MultiVector::create(this->exec); + auto res = Dense::create(this->exec); empty->move_to(res); @@ -535,12 +537,12 @@ TYPED_TEST(Sellp, ApplyWithSliceSizeAndStrideFactorFailsOnWrongNumberOfCols) } -TYPED_TEST(Sellp, ConvertsWithSliceSizeAndStrideFactorToMultiVector) +TYPED_TEST(Sellp, ConvertsWithSliceSizeAndStrideFactorToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx2->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx2->get_executor()); // clang-format off - auto dense_other = gko::initialize( + auto dense_other = gko::initialize( 4, {{1.0, 3.0, 2.0}, {0.0, 5.0, 0.0}}, this->exec); // clang-format on @@ -555,10 +557,10 @@ TYPED_TEST(Sellp, ConvertsWithSliceSizeAndStrideFactorToMultiVector) } -TYPED_TEST(Sellp, MovesWithSliceSizeAndStrideFactorToMultiVector) +TYPED_TEST(Sellp, MovesWithSliceSizeAndStrideFactorToDense) { - using Vec = typename TestFixture::Vec; - auto dense_mtx = Vec::create(this->mtx2->get_executor()); + using Dense = typename TestFixture::Dense; + auto dense_mtx = Dense::create(this->mtx2->get_executor()); this->mtx2->move_to(dense_mtx); diff --git a/reference/test/matrix/sparsity_csr.cpp b/reference/test/matrix/sparsity_csr.cpp index 00a29e00203..8f2f3d8e914 100644 --- a/reference/test/matrix/sparsity_csr.cpp +++ b/reference/test/matrix/sparsity_csr.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -26,7 +27,7 @@ class SparsityCsr : public ::testing::Test { typename std::tuple_element<1, decltype(ValueIndexType())>::type; using Mtx = gko::matrix::SparsityCsr; using Csr = gko::matrix::Csr; - using MultiVectorMtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; SparsityCsr() : exec(gko::ReferenceExecutor::create()), @@ -54,11 +55,11 @@ TYPED_TEST_SUITE(SparsityCsr, gko::test::ValueIndexTypes, TYPED_TEST(SparsityCsr, CanBeCreatedFromExistingCsrMatrix) { using Csr = typename TestFixture::Csr; - using MultiVectorMtx = typename TestFixture::MultiVectorMtx; + using Dense = typename TestFixture::Dense; using Mtx = typename TestFixture::Mtx; auto csr_mtx = gko::initialize( {{2.0, 3.0, 0.0}, {0.0, 1.0, 1.0}, {0.0, 0.0, -3.0}}, this->exec); - auto comp_mtx = gko::initialize( + auto comp_mtx = gko::initialize( {{1.0, 1.0, 0.0}, {0.0, 1.0, 1.0}, {0.0, 0.0, 1.0}}, this->exec); auto mtx = Mtx::create(this->exec, std::move(csr_mtx)); @@ -67,13 +68,13 @@ TYPED_TEST(SparsityCsr, CanBeCreatedFromExistingCsrMatrix) } -TYPED_TEST(SparsityCsr, CanBeCreatedFromExistingMultiVectorMatrix) +TYPED_TEST(SparsityCsr, CanBeCreatedFromExistingDense) { - using MultiVectorMtx = typename TestFixture::MultiVectorMtx; + using Dense = typename TestFixture::Dense; using Mtx = typename TestFixture::Mtx; - auto dense_mtx = gko::initialize( + auto dense_mtx = gko::initialize( {{2.0, 3.0, 0.0}, {0.0, 1.0, 1.0}, {0.0, 0.0, -3.0}}, this->exec); - auto comp_mtx = gko::initialize( + auto comp_mtx = gko::initialize( {{1.0, 1.0, 0.0}, {0.0, 1.0, 1.0}, {0.0, 0.0, 1.0}}, this->exec); auto mtx = Mtx::create(this->exec, std::move(dense_mtx)); diff --git a/reference/test/matrix/sparsity_csr_kernels.cpp b/reference/test/matrix/sparsity_csr_kernels.cpp index 3509875645e..87abe594509 100644 --- a/reference/test/matrix/sparsity_csr_kernels.cpp +++ b/reference/test/matrix/sparsity_csr_kernels.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include diff --git a/reference/test/multigrid/fixed_coarsening_kernels.cpp b/reference/test/multigrid/fixed_coarsening_kernels.cpp index 888eb941376..503ba4d1edf 100644 --- a/reference/test/multigrid/fixed_coarsening_kernels.cpp +++ b/reference/test/multigrid/fixed_coarsening_kernels.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/reference/test/multigrid/pgm_kernels.cpp b/reference/test/multigrid/pgm_kernels.cpp index b55376f9414..c7811b91fac 100644 --- a/reference/test/multigrid/pgm_kernels.cpp +++ b/reference/test/multigrid/pgm_kernels.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/reference/test/preconditioner/batch_jacobi_kernels.cpp b/reference/test/preconditioner/batch_jacobi_kernels.cpp index afc59c0f783..e41cae16697 100644 --- a/reference/test/preconditioner/batch_jacobi_kernels.cpp +++ b/reference/test/preconditioner/batch_jacobi_kernels.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -12,6 +12,7 @@ #include #include #include +#include #include #include diff --git a/reference/test/preconditioner/ic.cpp b/reference/test/preconditioner/ic.cpp index d81f7f32cf1..d30acc4de74 100644 --- a/reference/test/preconditioner/ic.cpp +++ b/reference/test/preconditioner/ic.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/reference/test/preconditioner/ilu.cpp b/reference/test/preconditioner/ilu.cpp index c713b2a2cf8..b9fe38c8d21 100644 --- a/reference/test/preconditioner/ilu.cpp +++ b/reference/test/preconditioner/ilu.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ class Ilu : public ::testing::Test { protected: using value_type = T; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using l_solver_type = gko::solver::Bicgstab; using u_solver_type = gko::solver::Bicgstab; using ilu_prec_type = gko::preconditioner::Ilu; @@ -37,11 +39,11 @@ class Ilu : public ::testing::Test { Ilu() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize({{2., 1., 1.}, {2., 5., 2.}, {2., 5., 5.}}, - exec)), - l_factor(gko::initialize( + mtx(gko::initialize({{2., 1., 1.}, {2., 5., 2.}, {2., 5., 5.}}, + exec)), + l_factor(gko::initialize( {{1., 0., 0.}, {1., 1., 0.}, {1., 1., 1.}}, exec)), - u_factor(gko::initialize( + u_factor(gko::initialize( {{2., 1., 1.}, {0., 4., 1.}, {0., 0., 3.}}, exec)), l_u_composition(Composition::create(l_factor, u_factor)), l_factory(l_solver_type::build() @@ -71,9 +73,9 @@ class Ilu : public ::testing::Test { {} std::shared_ptr exec; - std::shared_ptr mtx; - std::shared_ptr l_factor; - std::shared_ptr u_factor; + std::shared_ptr mtx; + std::shared_ptr l_factor; + std::shared_ptr u_factor; std::shared_ptr l_u_composition; std::shared_ptr l_factory; std::shared_ptr u_factory; @@ -129,7 +131,7 @@ TYPED_TEST(Ilu, ThrowOnWrongCompositionInput2) TYPED_TEST(Ilu, SetsCorrectMatrices) { - using Mtx = typename TestFixture::Mtx; + using Dense = typename TestFixture::Dense; using lower_trs = typename TestFixture::l_solver_type; using upper_trs = typename TestFixture::u_solver_type; auto ilu = this->ilu_pre_factory->generate(this->l_u_composition); @@ -140,11 +142,11 @@ TYPED_TEST(Ilu, SetsCorrectMatrices) // These convert steps are required since `get_system_matrix` usually // just returns `LinOp`, which `GKO_ASSERT_MTX_NEAR` can not use properly - std::unique_ptr converted_l_factor{Mtx::create(this->exec)}; - std::unique_ptr converted_u_factor{Mtx::create(this->exec)}; - gko::as>(internal_l_factor.get()) + std::unique_ptr converted_l_factor{Dense::create(this->exec)}; + std::unique_ptr converted_u_factor{Dense::create(this->exec)}; + gko::as>(internal_l_factor.get()) ->convert_to(converted_l_factor); - gko::as>(internal_u_factor.get()) + gko::as>(internal_u_factor.get()) ->convert_to(converted_u_factor); GKO_ASSERT_MTX_NEAR(converted_l_factor, this->l_factor, 0); GKO_ASSERT_MTX_NEAR(converted_u_factor, this->u_factor, 0); @@ -154,23 +156,23 @@ TYPED_TEST(Ilu, SetsCorrectMatrices) TYPED_TEST(Ilu, CanBeTransposed) { using Ilu = typename TestFixture::ilu_prec_type; - using Mtx = typename TestFixture::Mtx; + using Dense = typename TestFixture::Dense; using lower_trs = typename TestFixture::l_solver_type; using upper_trs = typename TestFixture::u_solver_type; auto ilu = this->ilu_pre_factory->generate(this->l_u_composition); - auto l_ref = gko::as( + auto l_ref = gko::as( gko::as(ilu->get_l_solver())->get_system_matrix()); - auto u_ref = gko::as( + auto u_ref = gko::as( gko::as(ilu->get_u_solver())->get_system_matrix()); auto transp = gko::as(ilu->transpose()); - auto l_transp = gko::as( - gko::as( + auto l_transp = gko::as( + gko::as( gko::as(transp->get_u_solver())->get_system_matrix()) ->transpose()); - auto u_transp = gko::as( - gko::as( + auto u_transp = gko::as( + gko::as( gko::as(transp->get_l_solver())->get_system_matrix()) ->transpose()); GKO_ASSERT_MTX_EQ_SPARSITY(l_ref, l_transp); @@ -183,23 +185,23 @@ TYPED_TEST(Ilu, CanBeTransposed) TYPED_TEST(Ilu, CanBeConjTransposed) { using Ilu = typename TestFixture::ilu_prec_type; - using Mtx = typename TestFixture::Mtx; + using Dense = typename TestFixture::Dense; using lower_trs = typename TestFixture::l_solver_type; using upper_trs = typename TestFixture::u_solver_type; auto ilu = this->ilu_pre_factory->generate(this->l_u_composition); - auto l_ref = gko::as( + auto l_ref = gko::as( gko::as(ilu->get_l_solver())->get_system_matrix()); - auto u_ref = gko::as( + auto u_ref = gko::as( gko::as(ilu->get_u_solver())->get_system_matrix()); auto transp = gko::as(ilu->conj_transpose()); - auto l_transp = gko::as( - gko::as( + auto l_transp = gko::as( + gko::as( gko::as(transp->get_u_solver())->get_system_matrix()) ->conj_transpose()); - auto u_transp = gko::as( - gko::as( + auto u_transp = gko::as( + gko::as( gko::as(transp->get_l_solver())->get_system_matrix()) ->conj_transpose()); GKO_ASSERT_MTX_EQ_SPARSITY(l_ref, l_transp); @@ -513,16 +515,17 @@ TYPED_TEST(Ilu, SolvesDifferentNumberOfRhs) class DefaultIlu : public ::testing::Test { protected: using Mtx = gko::matrix::MultiVector<>; + using Dense = gko::matrix::Dense<>; using default_ilu_prec_type = gko::preconditioner::Ilu<>; DefaultIlu() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize({{2., 1., 1.}, {2., 5., 2.}, {2., 5., 5.}}, - exec)) + mtx(gko::initialize({{2., 1., 1.}, {2., 5., 2.}, {2., 5., 5.}}, + exec)) {} std::shared_ptr exec; - std::shared_ptr mtx; + std::shared_ptr mtx; }; diff --git a/reference/test/preconditioner/isai_kernels.cpp b/reference/test/preconditioner/isai_kernels.cpp index adce05d2793..6a64bd19bce 100644 --- a/reference/test/preconditioner/isai_kernels.cpp +++ b/reference/test/preconditioner/isai_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -44,6 +45,7 @@ class Isai : public ::testing::Test { using SpdIsai = gko::preconditioner::SpdIsai; using Mtx = gko::matrix::Csr; using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Csr = gko::matrix::Csr; Isai() @@ -58,24 +60,24 @@ class Isai : public ::testing::Test { .with_reduction_factor( gko::remove_complex{1e-6})) .on(exec)), - a_dense{gko::initialize( - {{2, 1, 2}, {1, -2, 3}, {-1, 1, 1}}, exec)}, - a_dense_inv{gko::initialize({{0.3125, -0.0625, -0.4375}, - {0.25, -0.25, 0.25}, - {0.0625, 0.1875, 0.3125}}, - exec)}, - l_dense{gko::initialize( + a_dense{gko::initialize({{2, 1, 2}, {1, -2, 3}, {-1, 1, 1}}, + exec)}, + a_dense_inv{gko::initialize({{0.3125, -0.0625, -0.4375}, + {0.25, -0.25, 0.25}, + {0.0625, 0.1875, 0.3125}}, + exec)}, + l_dense{gko::initialize( {{2., 0., 0.}, {1., -2., 0.}, {-1., 1., -1.}}, exec)}, - l_dense_inv{gko::initialize( + l_dense_inv{gko::initialize( {{.5, 0., 0.}, {.25, -.5, 0.}, {-.25, -.5, -1.}}, exec)}, - u_dense{gko::initialize( + u_dense{gko::initialize( {{4., 1., -1.}, {0., -2., 4.}, {0., 0., 8.}}, exec)}, - u_dense_inv{gko::initialize( + u_dense_inv{gko::initialize( {{.25, .125, -0.03125}, {0., -.5, .25}, {0., 0., .125}}, exec)}, - spd_dense{gko::initialize( + spd_dense{gko::initialize( {{.0625, -.0625, .25}, {-.0625, .3125, -.5}, {.25, -.5, 2.25}}, exec)}, - spd_dense_inv{gko::initialize( + spd_dense_inv{gko::initialize( {{4., 0., 0.}, {2., 2., 0.}, {-3., 1., 1.}}, exec)}, a_csr{Csr::create(exec)}, a_csr_inv{Csr::create(exec)}, @@ -210,22 +212,22 @@ class Isai : public ::testing::Test { spd_dense_inv->convert_to(spd_csr_inv); l_csr_longrow = read("isai_l.mtx"); l_csr_longrow_e = read("isai_l_excess.mtx"); - l_csr_longrow_e_rhs = read("isai_l_excess_rhs.mtx"); + l_csr_longrow_e_rhs = read("isai_l_excess_rhs.mtx"); l_csr_longrow_inv_partial = read("isai_l_inv_partial.mtx"); l_csr_longrow_inv = read("isai_l_inv.mtx"); u_csr_longrow = read("isai_u.mtx"); u_csr_longrow_e = read("isai_u_excess.mtx"); - u_csr_longrow_e_rhs = read("isai_u_excess_rhs.mtx"); + u_csr_longrow_e_rhs = read("isai_u_excess_rhs.mtx"); u_csr_longrow_inv_partial = read("isai_u_inv_partial.mtx"); u_csr_longrow_inv = read("isai_u_inv.mtx"); a_csr_longrow = read("isai_a.mtx"); a_csr_longrow_e = read("isai_a_excess.mtx"); - a_csr_longrow_e_rhs = read("isai_a_excess_rhs.mtx"); + a_csr_longrow_e_rhs = read("isai_a_excess_rhs.mtx"); a_csr_longrow_inv_partial = read("isai_a_inv_partial.mtx"); a_csr_longrow_inv = read("isai_a_inv.mtx"); spd_csr_longrow = read("isai_spd.mtx"); spd_csr_longrow_e = read("isai_spd_excess.mtx"); - spd_csr_longrow_e_rhs = read("isai_spd_excess_rhs.mtx"); + spd_csr_longrow_e_rhs = read("isai_spd_excess_rhs.mtx"); spd_csr_longrow_inv_partial = read("isai_spd_inv_partial.mtx"); spd_csr_longrow_inv = read("isai_spd_inv.mtx"); } @@ -267,28 +269,28 @@ class Isai : public ::testing::Test { std::unique_ptr upper_isai_factory; std::unique_ptr general_isai_factory; std::unique_ptr spd_isai_factory; - std::shared_ptr a_dense; - std::shared_ptr a_dense_inv; - std::shared_ptr l_dense; - std::shared_ptr l_dense_inv; - std::shared_ptr u_dense; - std::shared_ptr u_dense_inv; - std::shared_ptr spd_dense; - std::shared_ptr spd_dense_inv; + std::shared_ptr a_dense; + std::shared_ptr a_dense_inv; + std::shared_ptr l_dense; + std::shared_ptr l_dense_inv; + std::shared_ptr u_dense; + std::shared_ptr u_dense_inv; + std::shared_ptr spd_dense; + std::shared_ptr spd_dense_inv; std::shared_ptr a_csr; std::shared_ptr a_csr_inv; std::shared_ptr l_csr; std::shared_ptr l_csr_inv; std::shared_ptr l_csr_longrow; std::shared_ptr l_csr_longrow_e; - std::shared_ptr l_csr_longrow_e_rhs; + std::shared_ptr l_csr_longrow_e_rhs; std::shared_ptr l_csr_longrow_inv_partial; std::shared_ptr l_csr_longrow_inv; std::shared_ptr u_csr; std::shared_ptr u_csr_inv; std::shared_ptr u_csr_longrow; std::shared_ptr u_csr_longrow_e; - std::shared_ptr u_csr_longrow_e_rhs; + std::shared_ptr u_csr_longrow_e_rhs; std::shared_ptr u_csr_longrow_inv_partial; std::shared_ptr u_csr_longrow_inv; std::shared_ptr l_sparse; @@ -307,14 +309,14 @@ class Isai : public ::testing::Test { std::shared_ptr a_sparse_inv; std::shared_ptr a_csr_longrow; std::shared_ptr a_csr_longrow_e; - std::shared_ptr a_csr_longrow_e_rhs; + std::shared_ptr a_csr_longrow_e_rhs; std::shared_ptr a_csr_longrow_inv_partial; std::shared_ptr a_csr_longrow_inv; std::shared_ptr spd_csr; std::shared_ptr spd_csr_inv; std::shared_ptr spd_csr_longrow; std::shared_ptr spd_csr_longrow_e; - std::shared_ptr spd_csr_longrow_e_rhs; + std::shared_ptr spd_csr_longrow_e_rhs; std::shared_ptr spd_csr_longrow_inv_partial; std::shared_ptr spd_csr_longrow_inv; std::shared_ptr spd_sparse; @@ -458,7 +460,7 @@ TYPED_TEST(Isai, KernelGenerateALongrow) TYPED_TEST(Isai, KernelGenerateExcessALongrow) { using Csr = typename TestFixture::Csr; - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; using index_type = typename TestFixture::index_type; auto num_rows = this->a_csr_longrow->get_size()[0]; @@ -472,7 +474,7 @@ TYPED_TEST(Isai, KernelGenerateExcessALongrow) std::fill_n(a2.get_data() + 15, 21, 355); std::fill_n(a2.get_data() + 36, 65, 509); auto result = Csr::create(this->exec, gko::dim<2>(122, 122), 509); - auto result_rhs = MultiVector::create(this->exec, gko::dim<2>(122, 1)); + auto result_rhs = Dense::create(this->exec, gko::dim<2>(122, 1)); gko::kernels::reference::isai::generate_excess_system( this->exec, this->a_csr_longrow.get(), this->a_csr_longrow.get(), @@ -659,7 +661,7 @@ TYPED_TEST(Isai, KernelGenerateLLongrow) TYPED_TEST(Isai, KernelGenerateExcessLLongrow) { using Csr = typename TestFixture::Csr; - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; using index_type = typename TestFixture::index_type; auto num_rows = this->l_csr_longrow->get_size()[0]; @@ -675,7 +677,7 @@ TYPED_TEST(Isai, KernelGenerateExcessLLongrow) a2.get_data()[34] = 124; a2.get_data()[35] = 248; auto result = Csr::create(this->exec, gko::dim<2>(66, 66), 248); - auto result_rhs = MultiVector::create(this->exec, gko::dim<2>(66, 1)); + auto result_rhs = Dense::create(this->exec, gko::dim<2>(66, 1)); gko::kernels::reference::isai::generate_excess_system( this->exec, this->l_csr_longrow.get(), this->l_csr_longrow.get(), @@ -860,7 +862,7 @@ TYPED_TEST(Isai, KernelGenerateULongrow) TYPED_TEST(Isai, KernelGenerateExcessULongrow) { using Csr = typename TestFixture::Csr; - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; using index_type = typename TestFixture::index_type; auto num_rows = this->u_csr_longrow->get_size()[0]; @@ -872,7 +874,7 @@ TYPED_TEST(Isai, KernelGenerateExcessULongrow) auto a2 = zeros; std::fill_n(a2.get_data() + 3, 33, 153); auto result = Csr::create(this->exec, gko::dim<2>(33, 33), 153); - auto result_rhs = MultiVector::create(this->exec, gko::dim<2>(33, 1)); + auto result_rhs = Dense::create(this->exec, gko::dim<2>(33, 1)); gko::kernels::reference::isai::generate_excess_system( this->exec, this->u_csr_longrow.get(), this->u_csr_longrow.get(), @@ -968,7 +970,7 @@ TYPED_TEST(Isai, KernelGenerateSpdLongrow) TYPED_TEST(Isai, KernelGenerateExcessSpdLongrow) { using Csr = typename TestFixture::Csr; - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; using index_type = typename TestFixture::index_type; auto num_rows = this->spd_csr_longrow->get_size()[0]; @@ -980,7 +982,7 @@ TYPED_TEST(Isai, KernelGenerateExcessSpdLongrow) auto a2 = zeros; std::fill_n(a2.get_data() + 36, 65, 338); auto result = Csr::create(this->exec, gko::dim<2>(36, 36), 338); - auto result_rhs = MultiVector::create(this->exec, gko::dim<2>(36, 1)); + auto result_rhs = Dense::create(this->exec, gko::dim<2>(36, 1)); gko::kernels::reference::isai::generate_excess_system( this->exec, this->spd_csr_longrow.get(), @@ -997,7 +999,7 @@ TYPED_TEST(Isai, KernelGenerateExcessSpdLongrow) TYPED_TEST(Isai, KernelScatterExcessSolution) { using Csr = typename TestFixture::Csr; - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; using index_type = typename TestFixture::index_type; gko::array ptrs{ @@ -1013,7 +1015,7 @@ TYPED_TEST(Isai, KernelScatterExcessSolution) {1, 11, 12, 4, 13, 14, 15, 16, 17, 10}}, gko::array{this->exec, {0, 0, 1, 0, 0, 1, 2, 0, 1, 0}}, gko::array{this->exec, {0, 1, 3, 4, 7, 9, 10}}); - auto sol = MultiVector::create( + auto sol = Dense::create( this->exec, gko::dim<2>(7, 1), gko::array{this->exec, {11, 12, 13, 14, 15, 16, 17}}, 1); @@ -1467,14 +1469,15 @@ TYPED_TEST(Isai, UseWithIluPreconditioner) using UpperIsai = typename TestFixture::UpperIsai; const auto vec = gko::initialize({128, -64, 32}, this->exec); auto result = MultiVector::create(this->exec, vec->get_size()); - auto mtx = gko::share(MultiVector::create_with_config_of(this->l_dense)); - this->l_dense->apply(this->u_dense, mtx); + auto mtx = gko::share(MultiVector::create_with_config_of( + this->l_dense->as_multivector_view())); + this->l_dense->apply(this->u_dense->as_const_multivector_view(), mtx); auto ilu_factory = gko::preconditioner::Ilu::build() .with_l_solver(LowerIsai::build()) .with_u_solver(UpperIsai::build()) .on(this->exec); - auto ilu = ilu_factory->generate(mtx); + auto ilu = ilu_factory->generate(mtx->as_const_dense_view()); ilu->apply(vec, result); @@ -1626,11 +1629,11 @@ TYPED_TEST(Isai, IsExactInverseOnFullSparsitySet) { using Isai = typename TestFixture::GeneralIsai; using Csr = typename TestFixture::Csr; - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; auto mtx = gko::share(gko::test::generate_tridiag_matrix( 12, gko::to_std_array(-1, 2, -1), this->exec)); - auto inv_mtx = gko::test::generate_tridiag_inverse_matrix( + auto inv_mtx = gko::test::generate_tridiag_inverse_matrix( 12, gko::to_std_array(-1, 2, -1), this->exec); auto isai = Isai::build() @@ -1647,11 +1650,11 @@ TYPED_TEST(Isai, IsExactInverseOnFullSparsitySetLarge) { using Isai = typename TestFixture::GeneralIsai; using Csr = typename TestFixture::Csr; - using MultiVector = typename TestFixture::MultiVector; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; auto mtx = gko::share(gko::test::generate_tridiag_matrix( 33, gko::to_std_array(-1, 2, -1), this->exec)); - auto inv_mtx = gko::test::generate_tridiag_inverse_matrix( + auto inv_mtx = gko::test::generate_tridiag_inverse_matrix( 33, gko::to_std_array(-1, 2, -1), this->exec); auto isai = Isai::build() diff --git a/reference/test/preconditioner/jacobi.cpp b/reference/test/preconditioner/jacobi.cpp index 751fe268188..d112071efc6 100644 --- a/reference/test/preconditioner/jacobi.cpp +++ b/reference/test/preconditioner/jacobi.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -27,6 +28,7 @@ class Jacobi : public ::testing::Test { typename std::tuple_element<1, decltype(ValueIndexType())>::type; using Bj = gko::preconditioner::Jacobi; using Mtx = gko::matrix::Csr; + using Dense = gko::matrix::Dense; using Vec = gko::matrix::MultiVector; Jacobi() @@ -157,7 +159,7 @@ TYPED_TEST(Jacobi, GeneratesCorrectStorageScheme) } -TYPED_TEST(Jacobi, ScalarJacobiConvertsToMultiVector) +TYPED_TEST(Jacobi, ScalarJacobiConvertsToDense) { using value_type = typename TestFixture::value_type; using index_type = typename TestFixture::index_type; @@ -168,7 +170,7 @@ TYPED_TEST(Jacobi, ScalarJacobiConvertsToMultiVector) csr->copy_from(this->mtx); auto scalar_j = this->scalar_j_factory->generate(csr); - auto dense_j = gko::matrix::MultiVector::create(this->exec); + auto dense_j = gko::matrix::Dense::create(this->exec); scalar_j->convert_to(dense_j); auto j_val = scalar_j->get_blocks(); @@ -195,7 +197,7 @@ TYPED_TEST(Jacobi, ScalarJacobiCanBeTransposed) csr->copy_from(this->mtx); auto scalar_j = this->scalar_j_factory->generate(csr); - auto dense_j = gko::matrix::MultiVector::create(this->exec); + auto dense_j = gko::matrix::Dense::create(this->exec); auto t_j = scalar_j->transpose(); auto trans_j = gko::as(t_j.get())->get_blocks(); auto scal_j = scalar_j->get_blocks(); @@ -378,10 +380,10 @@ TYPED_TEST(Jacobi, ScalarJacobiGeneratesOnDifferentPrecision) TYPED_TEST(Jacobi, ScalarJacobiHandleZero) { using value_type = typename TestFixture::value_type; + using Mtx = typename TestFixture::Dense; using Vec = typename TestFixture::Vec; - using Bj = typename TestFixture::Bj; auto mtx = gko::share( - gko::initialize({{0, 0, 0}, {0, 2, 0}, {0, 0, 0}}, this->exec)); + gko::initialize({{0, 0, 0}, {0, 2, 0}, {0, 0, 0}}, this->exec)); auto b = gko::initialize({1, 2, 3}, this->exec); auto x = Vec::create(this->exec, gko::dim<2>(3, 1)); diff --git a/reference/test/preconditioner/jacobi_kernels.cpp b/reference/test/preconditioner/jacobi_kernels.cpp index 92a1f83cbe5..82836ea690c 100644 --- a/reference/test/preconditioner/jacobi_kernels.cpp +++ b/reference/test/preconditioner/jacobi_kernels.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -29,6 +30,7 @@ class Jacobi : public ::testing::Test { typename std::tuple_element<1, decltype(ValueIndexType())>::type; using Bj = gko::preconditioner::Jacobi; using Mtx = gko::matrix::Csr; + using Dense = gko::matrix::Dense; using Vec = gko::matrix::MultiVector; using mdata = gko::matrix_data; @@ -1070,11 +1072,11 @@ TYPED_TEST(Jacobi, } -TYPED_TEST(Jacobi, ConvertsToMultiVector) +TYPED_TEST(Jacobi, ConvertsToDense) { - using Vec = typename TestFixture::Vec; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; - auto dense = Vec::create(this->exec); + auto dense = Dense::create(this->exec); this->bj_factory->generate(this->mtx)->move_to(dense); @@ -1090,12 +1092,12 @@ TYPED_TEST(Jacobi, ConvertsToMultiVector) } -TYPED_TEST(Jacobi, ConvertsToMultiVectorWithAdaptivePrecision) +TYPED_TEST(Jacobi, ConvertsToDenseWithAdaptivePrecision) { - using Vec = typename TestFixture::Vec; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; auto half_tol = std::sqrt(r::value); - auto dense = Vec::create(this->exec); + auto dense = Dense::create(this->exec); this->adaptive_bj_factory->generate(this->mtx)->move_to(dense); @@ -1111,11 +1113,11 @@ TYPED_TEST(Jacobi, ConvertsToMultiVectorWithAdaptivePrecision) } -TYPED_TEST(Jacobi, ConvertsEmptyToMultiVector) +TYPED_TEST(Jacobi, ConvertsEmptyToDense) { - using Vec = typename TestFixture::Vec; - auto empty = gko::share(Vec::create(this->exec)); - auto res = Vec::create(this->exec); + using Dense = typename TestFixture::Dense; + auto empty = gko::share(Dense::create(this->exec)); + auto res = Dense::create(this->exec); TestFixture::Bj::build().on(this->exec)->generate(empty)->move_to(res); @@ -1186,14 +1188,14 @@ TYPED_TEST(Jacobi, BlockL1) } -TYPED_TEST(Jacobi, L1BlockJaocbiConvertsToMultiVector) +TYPED_TEST(Jacobi, L1BlockJaocbiConvertsToDense) { using Bj = typename TestFixture::Bj; using Mtx = typename TestFixture::Mtx; - using Vec = typename TestFixture::Vec; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; using index_type = typename TestFixture::index_type; - auto dense = Vec::create(this->exec); + auto dense = Dense::create(this->exec); auto bj_factory = Bj::build() .with_max_block_size(3u) .with_block_pointers(this->block_pointers) @@ -1231,15 +1233,15 @@ TYPED_TEST(Jacobi, L1BlockJaocbiConvertsToMultiVector) } -TYPED_TEST(Jacobi, L1BlockJaocbiConvertsToMultiVectorWithAdaptivePrecision) +TYPED_TEST(Jacobi, L1BlockJaocbiConvertsToDenseWithAdaptivePrecision) { using Bj = typename TestFixture::Bj; using Mtx = typename TestFixture::Mtx; - using Vec = typename TestFixture::Vec; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; using index_type = typename TestFixture::index_type; auto half_tol = std::sqrt(r::value); - auto dense = Vec::create(this->exec); + auto dense = Dense::create(this->exec); auto bj_factory = Bj::build() .with_max_block_size(17u) // make sure group size is 1 diff --git a/reference/test/preconditioner/sor_kernels.cpp b/reference/test/preconditioner/sor_kernels.cpp index 18f7e77b596..e334066edba 100644 --- a/reference/test/preconditioner/sor_kernels.cpp +++ b/reference/test/preconditioner/sor_kernels.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include diff --git a/reference/test/reorder/mc64.cpp b/reference/test/reorder/mc64.cpp index 825fe057a4e..4c8df21affc 100644 --- a/reference/test/reorder/mc64.cpp +++ b/reference/test/reorder/mc64.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include diff --git a/reference/test/reorder/mc64_kernels.cpp b/reference/test/reorder/mc64_kernels.cpp index ad0dc606499..7f5404ca85b 100644 --- a/reference/test/reorder/mc64_kernels.cpp +++ b/reference/test/reorder/mc64_kernels.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -15,6 +15,7 @@ #include #include #include +#include #include #include diff --git a/reference/test/reorder/nested_dissection.cpp b/reference/test/reorder/nested_dissection.cpp index 1c3f6ccf0e9..94130732a24 100644 --- a/reference/test/reorder/nested_dissection.cpp +++ b/reference/test/reorder/nested_dissection.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -30,7 +31,7 @@ class NestedDissection : public ::testing::Test { using index_type = IndexType; using reorder_type = gko::experimental::reorder::NestedDissection; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; NestedDissection() : exec(gko::ReferenceExecutor::create()), nd_factory(reorder_type::build().on(exec)), @@ -90,8 +91,10 @@ TYPED_TEST(NestedDissection, ComputesSensiblePermutation) auto perm_array = gko::make_array_view(this->exec, perm->get_size()[0], perm->get_permutation()); - auto permuted = gko::as( - this->star_mtx->permute(&perm_array)); + auto permuted = this->star_mtx->as_multivector_view() + ->permute(&perm_array) + ->as_dense_view() + ->clone(); GKO_ASSERT_MTX_NEAR(permuted, I>({{1.0, 0.0, 0.0, 1.0}, {0.0, 1.0, 0.0, 1.0}, diff --git a/reference/test/reorder/rcm.cpp b/reference/test/reorder/rcm.cpp index 3746a7f4c9e..688a7987bf7 100644 --- a/reference/test/reorder/rcm.cpp +++ b/reference/test/reorder/rcm.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -77,7 +78,7 @@ TYPED_TEST(Rcm, CanBeCreatedWithStartingStrategy) reorder_type::build() .with_strategy(gko::reorder::starting_strategy::minimum_degree) .on(this->exec) - ->generate(gko::initialize>( + ->generate(gko::initialize>( 3, {{1.0, 0.0, 0.0}, {0.0, 1.0, 0.0}, {0.0, 0.0, 1.0}}, this->exec)); diff --git a/reference/test/reorder/rcm_kernels.cpp b/reference/test/reorder/rcm_kernels.cpp index f9d44f2dfd6..f3fa750c21d 100644 --- a/reference/test/reorder/rcm_kernels.cpp +++ b/reference/test/reorder/rcm_kernels.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -11,6 +11,7 @@ #include #include #include +#include #include #include diff --git a/reference/test/reorder/scaled_reordered.cpp b/reference/test/reorder/scaled_reordered.cpp index 6e1c718ef4e..fa0f51ec614 100644 --- a/reference/test/reorder/scaled_reordered.cpp +++ b/reference/test/reorder/scaled_reordered.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/reference/test/solver/batch_bicgstab_kernels.cpp b/reference/test/solver/batch_bicgstab_kernels.cpp index c5a4fe04fac..44253af0cb3 100644 --- a/reference/test/solver/batch_bicgstab_kernels.cpp +++ b/reference/test/solver/batch_bicgstab_kernels.cpp @@ -31,7 +31,7 @@ class BatchBicgstab : public ::testing::Test { using value_type = T; using real_type = gko::remove_complex; using solver_type = gko::batch::solver::Bicgstab; - using Mtx = gko::batch::matrix::MultiVector; + using Mtx = gko::batch::matrix::Dense; using EllMtx = gko::batch::matrix::Ell; using CsrMtx = gko::batch::matrix::Csr; using MVec = gko::batch::MultiVector; diff --git a/reference/test/solver/batch_cg_kernels.cpp b/reference/test/solver/batch_cg_kernels.cpp index c0fea43be1f..bad3b45e61e 100644 --- a/reference/test/solver/batch_cg_kernels.cpp +++ b/reference/test/solver/batch_cg_kernels.cpp @@ -31,7 +31,7 @@ class BatchCg : public ::testing::Test { using value_type = T; using real_type = gko::remove_complex; using solver_type = gko::batch::solver::Cg; - using Mtx = gko::batch::matrix::MultiVector; + using Mtx = gko::batch::matrix::Dense; using EllMtx = gko::batch::matrix::Ell; using CsrMtx = gko::batch::matrix::Csr; using MVec = gko::batch::MultiVector; diff --git a/reference/test/solver/bicg_kernels.cpp b/reference/test/solver/bicg_kernels.cpp index bb36fb01248..77aec032d0b 100644 --- a/reference/test/solver/bicg_kernels.cpp +++ b/reference/test/solver/bicg_kernels.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -26,12 +27,13 @@ class Bicg : public ::testing::Test { protected: using value_type = T; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Bicg; Bicg() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{2, -1.0, 0.0}, {-1.0, 2, -1.0}, {0.0, -1.0, 2}}, exec)), - mtx_big(gko::initialize( + mtx_big(gko::initialize( {{8828.0, 2673.0, 4150.0, -3139.5, 3829.5, 5856.0}, {2673.0, 10765.5, 1805.0, 73.0, 1966.0, 3919.5}, {4150.0, 1805.0, 6472.5, 2656.0, 2409.5, 3836.5}, @@ -39,7 +41,7 @@ class Bicg : public ::testing::Test { {3829.5, 1966.0, 2409.5, 665.0, 4240.5, 4373.5}, {5856.0, 3919.5, 3836.5, -132.0, 4373.5, 5678.0}}, exec)), - mtx_non_symmetric(gko::initialize( + mtx_non_symmetric(gko::initialize( {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)), stopped{}, non_stopped{}, @@ -92,9 +94,9 @@ class Bicg : public ::testing::Test { } std::shared_ptr exec; - std::shared_ptr mtx; - std::shared_ptr mtx_big; - std::shared_ptr mtx_non_symmetric; + std::shared_ptr mtx; + std::shared_ptr mtx_big; + std::shared_ptr mtx_non_symmetric; std::unique_ptr small_one; std::unique_ptr small_zero; std::unique_ptr small_prev_rho; diff --git a/reference/test/solver/bicgstab_kernels.cpp b/reference/test/solver/bicgstab_kernels.cpp index 10a56f96a9c..62ebc95275e 100644 --- a/reference/test/solver/bicgstab_kernels.cpp +++ b/reference/test/solver/bicgstab_kernels.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -25,12 +26,13 @@ template class Bicgstab : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Bicgstab; Bicgstab() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{1.0, -3.0, 0.0}, {-4.0, 1.0, -3.0}, {2.0, -1.0, 2.0}}, exec)), stopped{}, finalized{}, @@ -65,16 +67,16 @@ class Bicgstab : public ::testing::Test { { auto small_size = gko::dim<2>{2, 2}; auto small_scalar_size = gko::dim<2>{1, small_size[1]}; - small_b = Mtx::create(exec, small_size, small_size[1] + 1); - small_x = Mtx::create(exec, small_size, small_size[1] + 2); - small_one = Mtx::create(exec, small_size); - small_zero = Mtx::create(exec, small_size); - small_prev_rho = Mtx::create(exec, small_scalar_size); - small_rho = Mtx::create(exec, small_scalar_size); - small_alpha = Mtx::create(exec, small_scalar_size); - small_beta = Mtx::create(exec, small_scalar_size); - small_gamma = Mtx::create(exec, small_scalar_size); - small_omega = Mtx::create(exec, small_scalar_size); + small_b = Vec::create(exec, small_size, small_size[1] + 1); + small_x = Vec::create(exec, small_size, small_size[1] + 2); + small_one = Vec::create(exec, small_size); + small_zero = Vec::create(exec, small_size); + small_prev_rho = Vec::create(exec, small_scalar_size); + small_rho = Vec::create(exec, small_scalar_size); + small_alpha = Vec::create(exec, small_scalar_size); + small_beta = Vec::create(exec, small_scalar_size); + small_gamma = Vec::create(exec, small_scalar_size); + small_omega = Vec::create(exec, small_scalar_size); small_zero->fill(0); small_one->fill(1); small_r = small_zero->clone(); @@ -93,25 +95,25 @@ class Bicgstab : public ::testing::Test { } std::shared_ptr exec; - std::shared_ptr mtx; - std::unique_ptr small_one; - std::unique_ptr small_zero; - std::unique_ptr small_prev_rho; - std::unique_ptr small_rho; - std::unique_ptr small_alpha; - std::unique_ptr small_beta; - std::unique_ptr small_gamma; - std::unique_ptr small_omega; - std::unique_ptr small_x; - std::unique_ptr small_b; - std::unique_ptr small_r; - std::unique_ptr small_rr; - std::unique_ptr small_v; - std::unique_ptr small_s; - std::unique_ptr small_t; - std::unique_ptr small_z; - std::unique_ptr small_y; - std::unique_ptr small_p; + std::shared_ptr mtx; + std::unique_ptr small_one; + std::unique_ptr small_zero; + std::unique_ptr small_prev_rho; + std::unique_ptr small_rho; + std::unique_ptr small_alpha; + std::unique_ptr small_beta; + std::unique_ptr small_gamma; + std::unique_ptr small_omega; + std::unique_ptr small_x; + std::unique_ptr small_b; + std::unique_ptr small_r; + std::unique_ptr small_rr; + std::unique_ptr small_v; + std::unique_ptr small_s; + std::unique_ptr small_t; + std::unique_ptr small_z; + std::unique_ptr small_y; + std::unique_ptr small_p; gko::array small_stop; gko::stopping_status stopped; gko::stopping_status finalized; @@ -409,11 +411,11 @@ TYPED_TEST(Bicgstab, KernelFinalize) TYPED_TEST(Bicgstab, SolvesMultiVectorSystem) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto solver = this->bicgstab_factory->generate(this->mtx); - auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); + auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); solver->apply(b, x); @@ -424,10 +426,10 @@ TYPED_TEST(Bicgstab, SolvesMultiVectorSystem) TYPED_TEST(Bicgstab, SolvesMultiVectorSystemMixed) { using value_type = gko::next_precision; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; auto solver = this->bicgstab_factory->generate(this->mtx); - auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); + auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); solver->apply(b, x); @@ -438,13 +440,13 @@ TYPED_TEST(Bicgstab, SolvesMultiVectorSystemMixed) TYPED_TEST(Bicgstab, SolvesMultiVectorSystemComplex) { - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; + using Vec = gko::to_complex; + using value_type = typename Vec::value_type; auto solver = this->bicgstab_factory->generate(this->mtx); - auto b = gko::initialize( + auto b = gko::initialize( {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, this->exec); - auto x = gko::initialize( + auto x = gko::initialize( {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, this->exec); @@ -481,14 +483,14 @@ TYPED_TEST(Bicgstab, SolvesMultiVectorSystemMixedComplex) TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystems) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; using T = value_type; auto half_tol = std::sqrt(r::value); auto solver = this->bicgstab_factory->generate(this->mtx); - auto b = gko::initialize( + auto b = gko::initialize( {I{-1.0, -5.0}, I{3.0, 1.0}, I{1.0, -2.0}}, this->exec); - auto x = gko::initialize( + auto x = gko::initialize( {I{0.0, 0.0}, I{0.0, 0.0}, I{0.0, 0.0}}, this->exec); solver->apply(b, x); @@ -500,14 +502,14 @@ TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystems) TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystemsWithImplicitResNormCrit) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; using T = value_type; auto half_tol = std::sqrt(r::value); auto solver = this->bicgstab_factory2->generate(this->mtx); - auto b = gko::initialize( + auto b = gko::initialize( {I{-1.0, -5.0}, I{3.0, 1.0}, I{1.0, -2.0}}, this->exec); - auto x = gko::initialize( + auto x = gko::initialize( {I{0.0, 0.0}, I{0.0, 0.0}, I{0.0, 0.0}}, this->exec); solver->apply(b, x); @@ -519,13 +521,13 @@ TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystemsWithImplicitResNormCrit) TYPED_TEST(Bicgstab, SolvesMultiVectorSystemUsingAdvancedApply) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto solver = this->bicgstab_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); - auto x = gko::initialize({0.5, 1.0, 2.0}, this->exec); + auto alpha = gko::initialize({2.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); + auto x = gko::initialize({0.5, 1.0, 2.0}, this->exec); solver->apply(alpha, b, beta, x); @@ -536,12 +538,12 @@ TYPED_TEST(Bicgstab, SolvesMultiVectorSystemUsingAdvancedApply) TYPED_TEST(Bicgstab, SolvesMultiVectorSystemUsingAdvancedApplyMixed) { using value_type = gko::next_precision; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; auto solver = this->bicgstab_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); - auto x = gko::initialize({0.5, 1.0, 2.0}, this->exec); + auto alpha = gko::initialize({2.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); + auto x = gko::initialize({0.5, 1.0, 2.0}, this->exec); solver->apply(alpha, b, beta, x); @@ -552,16 +554,16 @@ TYPED_TEST(Bicgstab, SolvesMultiVectorSystemUsingAdvancedApplyMixed) TYPED_TEST(Bicgstab, SolvesMultiVectorSystemUsingAdvancedApplyComplex) { - using Scalar = typename TestFixture::Mtx; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; + using Scalar = typename TestFixture::Vec; + using Vec = gko::to_complex; + using value_type = typename Vec::value_type; auto solver = this->bicgstab_factory->generate(this->mtx); auto alpha = gko::initialize({2.0}, this->exec); auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( + auto b = gko::initialize( {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, this->exec); - auto x = gko::initialize( + auto x = gko::initialize( {value_type{0.5, -0.5}, value_type{1.0, 0.5}, value_type{2.0, -1.0}}, this->exec); @@ -601,16 +603,16 @@ TYPED_TEST(Bicgstab, SolvesMultiVectorSystemUsingAdvancedApplyMixedComplex) TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystemsUsingAdvancedApply) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; using T = value_type; auto half_tol = std::sqrt(r::value); auto solver = this->bicgstab_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( + auto alpha = gko::initialize({2.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto b = gko::initialize( {I{-1.0, -5.0}, I{3.0, 1.0}, I{1.0, -2.0}}, this->exec); - auto x = gko::initialize( + auto x = gko::initialize( {I{0.5, 1.0}, I{1.0, 2.0}, I{2.0, 3.0}}, this->exec); solver->apply(alpha, b, beta, x); @@ -623,7 +625,8 @@ TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystemsUsingAdvancedApply) // The following test-data was generated and validated with MATLAB TYPED_TEST(Bicgstab, SolvesBigMultiVectorSystemForDivergenceCheck1) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; // beta encounters huge value out of the half-precision range in the first // part of the second iteration @@ -631,18 +634,18 @@ TYPED_TEST(Bicgstab, SolvesBigMultiVectorSystemForDivergenceCheck1) // rounding error for bfloat16 SKIP_IF_BFLOAT16(value_type); auto half_tol = std::sqrt(r::value); - std::shared_ptr locmtx = - gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, - {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, - {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, - {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, - {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, - {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, - this->exec); + std::shared_ptr locmtx = + gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, + {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, + {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, + {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, + {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, + {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, + this->exec); auto solver = this->bicgstab_factory_precision->generate(locmtx); auto b = - gko::initialize({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + gko::initialize({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); solver->apply(b, x); @@ -656,7 +659,8 @@ TYPED_TEST(Bicgstab, SolvesBigMultiVectorSystemForDivergenceCheck1) TYPED_TEST(Bicgstab, SolvesBigMultiVectorSystemForDivergenceCheck2) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; // beta encounters huge value out of the half-precision range in the first // part of second iteration @@ -664,18 +668,18 @@ TYPED_TEST(Bicgstab, SolvesBigMultiVectorSystemForDivergenceCheck2) // rounding error for bfloat16 SKIP_IF_BFLOAT16(value_type); auto half_tol = std::sqrt(r::value); - std::shared_ptr locmtx = - gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, - {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, - {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, - {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, - {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, - {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, - this->exec); + std::shared_ptr locmtx = + gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, + {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, + {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, + {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, + {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, + {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, + this->exec); auto solver = this->bicgstab_factory_precision->generate(locmtx); auto b = - gko::initialize({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + gko::initialize({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); solver->apply(b, x); @@ -689,31 +693,32 @@ TYPED_TEST(Bicgstab, SolvesBigMultiVectorSystemForDivergenceCheck2) TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystemsDivergenceCheck) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; using T = value_type; // beta encounters huge value out of the half-precision range in the first // part of second iteration SKIP_IF_HALF(value_type); - std::shared_ptr locmtx = - gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, - {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, - {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, - {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, - {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, - {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, - this->exec); + std::shared_ptr locmtx = + gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, + {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, + {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, + {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, + {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, + {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, + this->exec); auto solver = this->bicgstab_factory_precision->generate(locmtx); auto b1 = - gko::initialize({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec); - auto x1 = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + gko::initialize({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec); + auto x1 = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); auto b2 = - gko::initialize({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec); - auto x2 = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); - auto bc = gko::initialize({I{0., 0.}, I{0., 0.}, I{0., 0.}, + gko::initialize({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec); + auto x2 = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + auto bc = gko::initialize({I{0., 0.}, I{0., 0.}, I{0., 0.}, I{0., 0.}, I{0., 0.}, I{0., 0.}}, this->exec); - auto xc = gko::initialize({I{0., 0.}, I{0., 0.}, I{0., 0.}, + auto xc = gko::initialize({I{0., 0.}, I{0., 0.}, I{0., 0.}, I{0., 0.}, I{0., 0.}, I{0., 0.}}, this->exec); for (size_t i = 0; i < xc->get_size()[0]; ++i) { @@ -727,22 +732,22 @@ TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystemsDivergenceCheck) solver->apply(b2, x2); solver->apply(bc, xc); auto testMtx = - gko::initialize({I{0., 0.}, I{0., 0.}, I{0., 0.}, - I{0., 0.}, I{0., 0.}, I{0., 0.}}, - this->exec); + gko::initialize({I{0., 0.}, I{0., 0.}, I{0., 0.}, + I{0., 0.}, I{0., 0.}, I{0., 0.}}, + this->exec); for (size_t i = 0; i < testMtx->get_size()[0]; ++i) { testMtx->at(i, 0) = x1->at(i); testMtx->at(i, 1) = x2->at(i); } - auto alpha = gko::initialize({1.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto residual1 = gko::initialize({0.}, this->exec); + auto alpha = gko::initialize({1.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto residual1 = gko::initialize({0.}, this->exec); residual1->copy_from(b1); - auto residual2 = gko::initialize({0.}, this->exec); + auto residual2 = gko::initialize({0.}, this->exec); residual2->copy_from(b2); - auto residualC = gko::initialize({0.}, this->exec); + auto residualC = gko::initialize({0.}, this->exec); residualC->copy_from(bc); locmtx->apply(alpha, x1, beta, residual1); @@ -769,12 +774,12 @@ TYPED_TEST(Bicgstab, SolvesMultipleMultiVectorSystemsDivergenceCheck) TYPED_TEST(Bicgstab, SolvesTransposedMultiVectorSystem) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto half_tol = std::sqrt(r::value); auto solver = this->bicgstab_factory->generate(this->mtx->transpose()); - auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); + auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); solver->transpose()->apply(b, x); @@ -784,12 +789,12 @@ TYPED_TEST(Bicgstab, SolvesTransposedMultiVectorSystem) TYPED_TEST(Bicgstab, SolvesConjTransposedMultiVectorSystem) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto half_tol = std::sqrt(r::value); auto solver = this->bicgstab_factory->generate(this->mtx->conj_transpose()); - auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); + auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); solver->conj_transpose()->apply(b, x); diff --git a/reference/test/solver/cb_gmres_kernels.cpp b/reference/test/solver/cb_gmres_kernels.cpp index 30d1c8b2887..f1009d0518a 100644 --- a/reference/test/solver/cb_gmres_kernels.cpp +++ b/reference/test/solver/cb_gmres_kernels.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -31,22 +32,23 @@ class CbGmres : public ::testing::Test { using storage_helper_type = typename std::tuple_element<1, decltype(ValueEnumType())>::type; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using gmres_type = gko::solver::CbGmres; CbGmres() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)), - mtx2(gko::initialize( + mtx2(gko::initialize( {{1.0, 2.0, 3.0}, {4.0, 2.0, 1.0}, {0.0, 1.0, 2.0}}, exec)), mtx_medium( - gko::initialize({{-86.40, 153.30, -108.90, 8.60, -61.60}, - {7.70, -77.00, 3.30, -149.20, 74.80}, - {-121.40, 37.10, 55.30, -74.20, -19.20}, - {-111.40, -22.60, 110.10, -106.20, 88.90}, - {-0.70, 111.70, 154.40, 235.00, -76.50}}, - exec)), - mtx_big(gko::initialize( + gko::initialize({{-86.40, 153.30, -108.90, 8.60, -61.60}, + {7.70, -77.00, 3.30, -149.20, 74.80}, + {-121.40, 37.10, 55.30, -74.20, -19.20}, + {-111.40, -22.60, 110.10, -106.20, 88.90}, + {-0.70, 111.70, 154.40, 235.00, -76.50}}, + exec)), + mtx_big(gko::initialize( {{2295.7, -764.8, 1166.5, 428.9, 291.7, -774.5}, {2752.6, -1127.7, 1212.8, -299.1, 987.7, 786.8}, {138.3, 78.2, 485.5, -899.9, 392.9, 1408.9}, @@ -107,10 +109,10 @@ class CbGmres : public ::testing::Test { } std::shared_ptr exec; - std::shared_ptr mtx; - std::shared_ptr mtx2; - std::shared_ptr mtx_medium; - std::shared_ptr mtx_big; + std::shared_ptr mtx; + std::shared_ptr mtx2; + std::shared_ptr mtx_medium; + std::shared_ptr mtx_big; gko::solver::cb_gmres::storage_precision storage_prec; std::unique_ptr cb_gmres_factory; std::unique_ptr cb_gmres_factory_big; diff --git a/reference/test/solver/cg_kernels.cpp b/reference/test/solver/cg_kernels.cpp index 2cd5366a0b3..2364c5ae8b8 100644 --- a/reference/test/solver/cg_kernels.cpp +++ b/reference/test/solver/cg_kernels.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -25,11 +26,13 @@ template class Cg : public ::testing::Test { protected: using value_type = T; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Cg; + Cg() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{2, -1.0, 0.0}, {-1.0, 2, -1.0}, {0.0, -1.0, 2}}, exec)), stopped{}, non_stopped{}, @@ -41,7 +44,7 @@ class Cg : public ::testing::Test { gko::stop::ResidualNorm::build() .with_reduction_factor(r::value)) .on(exec)), - mtx_big(gko::initialize( + mtx_big(gko::initialize( {{8828.0, 2673.0, 4150.0, -3139.5, 3829.5, 5856.0}, {2673.0, 10765.5, 1805.0, 73.0, 1966.0, 3919.5}, {4150.0, 1805.0, 6472.5, 2656.0, 2409.5, 3836.5}, @@ -66,13 +69,13 @@ class Cg : public ::testing::Test { { auto small_size = gko::dim<2>{2, 2}; auto small_scalar_size = gko::dim<2>{1, small_size[1]}; - small_b = Mtx::create(exec, small_size, small_size[1] + 1); - small_x = Mtx::create(exec, small_size, small_size[1] + 2); - small_one = Mtx::create(exec, small_size); - small_zero = Mtx::create(exec, small_size); - small_prev_rho = Mtx::create(exec, small_scalar_size); - small_rho = Mtx::create(exec, small_scalar_size); - small_beta = Mtx::create(exec, small_scalar_size); + small_b = Vec::create(exec, small_size, small_size[1] + 1); + small_x = Vec::create(exec, small_size, small_size[1] + 2); + small_one = Vec::create(exec, small_size); + small_zero = Vec::create(exec, small_size); + small_prev_rho = Vec::create(exec, small_scalar_size); + small_rho = Vec::create(exec, small_scalar_size); + small_beta = Vec::create(exec, small_scalar_size); small_zero->fill(0); small_one->fill(1); small_r = small_zero->clone(); @@ -86,19 +89,19 @@ class Cg : public ::testing::Test { } std::shared_ptr exec; - std::shared_ptr mtx; - std::shared_ptr mtx_big; - std::unique_ptr small_one; - std::unique_ptr small_zero; - std::unique_ptr small_prev_rho; - std::unique_ptr small_beta; - std::unique_ptr small_rho; - std::unique_ptr small_x; - std::unique_ptr small_b; - std::unique_ptr small_r; - std::unique_ptr small_z; - std::unique_ptr small_p; - std::unique_ptr small_q; + std::shared_ptr mtx; + std::shared_ptr mtx_big; + std::unique_ptr small_one; + std::unique_ptr small_zero; + std::unique_ptr small_prev_rho; + std::unique_ptr small_beta; + std::unique_ptr small_rho; + std::unique_ptr small_x; + std::unique_ptr small_b; + std::unique_ptr small_r; + std::unique_ptr small_z; + std::unique_ptr small_p; + std::unique_ptr small_q; gko::array small_stop; gko::stopping_status stopped; gko::stopping_status non_stopped; @@ -226,11 +229,11 @@ TYPED_TEST(Cg, KernelStep2DivByZero) TYPED_TEST(Cg, SolvesStencilSystem) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto solver = this->cg_factory->generate(this->mtx); - auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); + auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); solver->apply(b, x); @@ -241,10 +244,10 @@ TYPED_TEST(Cg, SolvesStencilSystem) TYPED_TEST(Cg, SolvesStencilSystemMixed) { using value_type = gko::next_precision; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; auto solver = this->cg_factory->generate(this->mtx); - auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); + auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0}, this->exec); solver->apply(b, x); @@ -255,13 +258,13 @@ TYPED_TEST(Cg, SolvesStencilSystemMixed) TYPED_TEST(Cg, SolvesStencilSystemComplex) { - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; + using Vec = gko::to_complex; + using value_type = typename Vec::value_type; auto solver = this->cg_factory->generate(this->mtx); - auto b = gko::initialize( + auto b = gko::initialize( {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, this->exec); - auto x = gko::initialize( + auto x = gko::initialize( {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, this->exec); @@ -298,13 +301,13 @@ TYPED_TEST(Cg, SolvesStencilSystemMixedComplex) TYPED_TEST(Cg, SolvesMultipleStencilSystems) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; using T = value_type; auto solver = this->cg_factory->generate(this->mtx); - auto b = gko::initialize( + auto b = gko::initialize( {I{-1.0, 1.0}, I{3.0, 0.0}, I{1.0, 1.0}}, this->exec); - auto x = gko::initialize( + auto x = gko::initialize( {I{0.0, 0.0}, I{0.0, 0.0}, I{0.0, 0.0}}, this->exec); solver->apply(b, x); @@ -316,13 +319,13 @@ TYPED_TEST(Cg, SolvesMultipleStencilSystems) TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApply) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; auto solver = this->cg_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); - auto x = gko::initialize({0.5, 1.0, 2.0}, this->exec); + auto alpha = gko::initialize({2.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); + auto x = gko::initialize({0.5, 1.0, 2.0}, this->exec); solver->apply(alpha, b, beta, x); @@ -333,12 +336,12 @@ TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApply) TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApplyMixed) { using value_type = gko::next_precision; - using Mtx = gko::matrix::MultiVector; + using Vec = gko::matrix::MultiVector; auto solver = this->cg_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); - auto x = gko::initialize({0.5, 1.0, 2.0}, this->exec); + auto alpha = gko::initialize({2.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto b = gko::initialize({-1.0, 3.0, 1.0}, this->exec); + auto x = gko::initialize({0.5, 1.0, 2.0}, this->exec); solver->apply(alpha, b, beta, x); @@ -349,16 +352,16 @@ TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApplyMixed) TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApplyComplex) { - using Scalar = typename TestFixture::Mtx; - using Mtx = gko::to_complex; - using value_type = typename Mtx::value_type; + using Scalar = typename TestFixture::Vec; + using Vec = gko::to_complex; + using value_type = typename Vec::value_type; auto solver = this->cg_factory->generate(this->mtx); auto alpha = gko::initialize({2.0}, this->exec); auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( + auto b = gko::initialize( {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, this->exec); - auto x = gko::initialize( + auto x = gko::initialize( {value_type{0.5, -1.0}, value_type{1.0, -2.0}, value_type{2.0, -4.0}}, this->exec); @@ -398,15 +401,15 @@ TYPED_TEST(Cg, SolvesStencilSystemUsingAdvancedApplyMixedComplex) TYPED_TEST(Cg, SolvesMultipleStencilSystemsUsingAdvancedApply) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; using T = value_type; auto solver = this->cg_factory->generate(this->mtx); - auto alpha = gko::initialize({2.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); - auto b = gko::initialize( + auto alpha = gko::initialize({2.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); + auto b = gko::initialize( {I{-1.0, 1.0}, I{3.0, 0.0}, I{1.0, 1.0}}, this->exec); - auto x = gko::initialize( + auto x = gko::initialize( {I{0.5, 1.0}, I{1.0, 2.0}, I{2.0, 3.0}}, this->exec); solver->apply(alpha, b, beta, x); @@ -418,15 +421,15 @@ TYPED_TEST(Cg, SolvesMultipleStencilSystemsUsingAdvancedApply) TYPED_TEST(Cg, SolvesBigMultiVectorSystem1) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; // the system is already out of half precision range SKIP_IF_HALF(value_type); auto solver = this->cg_factory_big->generate(this->mtx_big); - auto b = gko::initialize( + auto b = gko::initialize( {1300083.0, 1018120.5, 906410.0, -42679.5, 846779.5, 1176858.5}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); solver->apply(b, x); @@ -437,15 +440,15 @@ TYPED_TEST(Cg, SolvesBigMultiVectorSystem1) TYPED_TEST(Cg, SolvesBigMultiVectorSystem2) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; // the system is already out of half precision range SKIP_IF_HALF(value_type); auto solver = this->cg_factory_big->generate(this->mtx_big); - auto b = gko::initialize( + auto b = gko::initialize( {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); solver->apply(b, x); @@ -456,15 +459,15 @@ TYPED_TEST(Cg, SolvesBigMultiVectorSystem2) TYPED_TEST(Cg, SolvesBigMultiVectorSystem3) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; // the system is already out of half precision range SKIP_IF_HALF(value_type); auto solver = this->cg_factory_big2->generate(this->mtx_big); - auto b = gko::initialize( + auto b = gko::initialize( {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); solver->apply(b, x); @@ -475,25 +478,25 @@ TYPED_TEST(Cg, SolvesBigMultiVectorSystem3) TYPED_TEST(Cg, SolvesMultipleMultiVectorSystemForDivergenceCheck) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; // the system is already out of half precision range SKIP_IF_HALF(value_type); auto solver = this->cg_factory_big->generate(this->mtx_big); - auto b1 = gko::initialize( + auto b1 = gko::initialize( {1300083.0, 1018120.5, 906410.0, -42679.5, 846779.5, 1176858.5}, this->exec); - auto b2 = gko::initialize( + auto b2 = gko::initialize( {886630.5, -172578.0, 684522.0, -65310.5, 455487.5, 607436.0}, this->exec); - auto x1 = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); - auto x2 = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + auto x1 = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + auto x2 = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); auto bc = - Mtx::create(this->exec, gko::dim<2>{this->mtx_big->get_size()[0], 2}); + Vec::create(this->exec, gko::dim<2>{this->mtx_big->get_size()[0], 2}); auto xc = - Mtx::create(this->exec, gko::dim<2>{this->mtx_big->get_size()[1], 2}); + Vec::create(this->exec, gko::dim<2>{this->mtx_big->get_size()[1], 2}); for (size_t i = 0; i < bc->get_size()[0]; ++i) { bc->at(i, 0) = b1->at(i); bc->at(i, 1) = b2->at(i); @@ -505,20 +508,20 @@ TYPED_TEST(Cg, SolvesMultipleMultiVectorSystemForDivergenceCheck) solver->apply(b1, x1); solver->apply(b2, x2); solver->apply(bc, xc); - auto mergedRes = Mtx::create(this->exec, gko::dim<2>{b1->get_size()[0], 2}); + auto mergedRes = Vec::create(this->exec, gko::dim<2>{b1->get_size()[0], 2}); for (size_t i = 0; i < mergedRes->get_size()[0]; ++i) { mergedRes->at(i, 0) = x1->at(i); mergedRes->at(i, 1) = x2->at(i); } - auto alpha = gko::initialize({1.0}, this->exec); - auto beta = gko::initialize({-1.0}, this->exec); + auto alpha = gko::initialize({1.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); - auto residual1 = Mtx::create(this->exec, b1->get_size()); + auto residual1 = Vec::create(this->exec, b1->get_size()); residual1->copy_from(b1); - auto residual2 = Mtx::create(this->exec, b2->get_size()); + auto residual2 = Vec::create(this->exec, b2->get_size()); residual2->copy_from(b2); - auto residualC = Mtx::create(this->exec, bc->get_size()); + auto residualC = Vec::create(this->exec, bc->get_size()); residualC->copy_from(bc); this->mtx_big->apply(alpha, x1, beta, residual1); @@ -545,15 +548,15 @@ TYPED_TEST(Cg, SolvesMultipleMultiVectorSystemForDivergenceCheck) TYPED_TEST(Cg, SolvesTransposedBigMultiVectorSystem) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; // the system is already out of half precision range SKIP_IF_HALF(value_type); auto solver = this->cg_factory_big->generate(this->mtx_big); - auto b = gko::initialize( + auto b = gko::initialize( {1300083.0, 1018120.5, 906410.0, -42679.5, 846779.5, 1176858.5}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); solver->transpose()->apply(b, x); @@ -564,15 +567,15 @@ TYPED_TEST(Cg, SolvesTransposedBigMultiVectorSystem) TYPED_TEST(Cg, SolvesConjTransposedBigMultiVectorSystem) { - using Mtx = typename TestFixture::Mtx; + using Vec = typename TestFixture::Vec; using value_type = typename TestFixture::value_type; // the system is already out of half precision range SKIP_IF_HALF(value_type); auto solver = this->cg_factory_big->generate(this->mtx_big); - auto b = gko::initialize( + auto b = gko::initialize( {1300083.0, 1018120.5, 906410.0, -42679.5, 846779.5, 1176858.5}, this->exec); - auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); + auto x = gko::initialize({0.0, 0.0, 0.0, 0.0, 0.0, 0.0}, this->exec); solver->conj_transpose()->apply(b, x); diff --git a/reference/test/solver/cgs_kernels.cpp b/reference/test/solver/cgs_kernels.cpp index 83d6dbf37e2..1cbe929ba5d 100644 --- a/reference/test/solver/cgs_kernels.cpp +++ b/reference/test/solver/cgs_kernels.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -27,11 +28,12 @@ class Cgs : public ::testing::Test { protected: using value_type = T; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Cgs; Cgs() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{1.0, -3.0, 0.0}, {-4.0, 1.0, -3.0}, {2.0, -1.0, 2.0}}, exec)), stopped{}, non_stopped{}, @@ -42,13 +44,13 @@ class Cgs : public ::testing::Test { .with_reduction_factor(r::value)) .on(exec)), mtx_big( - gko::initialize({{-99.0, 87.0, -67.0, -62.0, -68.0, -19.0}, - {-30.0, -17.0, -1.0, 9.0, 23.0, 77.0}, - {80.0, 89.0, 36.0, 94.0, 55.0, 34.0}, - {-31.0, 21.0, 96.0, -26.0, 24.0, -57.0}, - {60.0, 45.0, -16.0, -4.0, 96.0, 24.0}, - {69.0, 32.0, -68.0, 57.0, -30.0, -51.0}}, - exec)), + gko::initialize({{-99.0, 87.0, -67.0, -62.0, -68.0, -19.0}, + {-30.0, -17.0, -1.0, 9.0, 23.0, 77.0}, + {80.0, 89.0, 36.0, 94.0, 55.0, 34.0}, + {-31.0, 21.0, 96.0, -26.0, 24.0, -57.0}, + {60.0, 45.0, -16.0, -4.0, 96.0, 24.0}, + {69.0, 32.0, -68.0, 57.0, -30.0, -51.0}}, + exec)), cgs_factory_big( Solver::build() .with_criteria( @@ -94,8 +96,8 @@ class Cgs : public ::testing::Test { } std::shared_ptr exec; - std::shared_ptr mtx; - std::shared_ptr mtx_big; + std::shared_ptr mtx; + std::shared_ptr mtx_big; std::unique_ptr small_one; std::unique_ptr small_zero; std::unique_ptr small_prev_rho; @@ -358,25 +360,29 @@ TYPED_TEST(Cgs, SolvesMultiVectorSystemComplex) } -TYPED_TEST(Cgs, SolvesMultiVectorSystemMixedComplex) +TYPED_TEST(Cgs, SolvesMultiVectorSystemUsingAdvancedApplyMixedComplex) { - using value_type = - gko::to_complex>; - using Mtx = gko::matrix::MultiVector; + using Scalar = gko::matrix::MultiVector< + gko::next_precision>; + using Mtx = gko::to_complex; + using value_type = typename Mtx::value_type; auto solver = this->cgs_factory->generate(this->mtx); + auto alpha = gko::initialize({2.0}, this->exec); + auto beta = gko::initialize({-1.0}, this->exec); auto b = gko::initialize( {value_type{-1.0, 2.0}, value_type{3.0, -6.0}, value_type{1.0, -2.0}}, this->exec); auto x = gko::initialize( - {value_type{0.0, 0.0}, value_type{0.0, 0.0}, value_type{0.0, 0.0}}, + {value_type{-2.0, 4.0}, value_type{-0.5, 1.0}, value_type{2.0, -4.0}}, this->exec); - solver->apply(b, x); + + solver->apply(alpha, b, beta, x); GKO_ASSERT_MTX_NEAR(x, - l({value_type{-4.0, 8.0}, value_type{-1.0, 2.0}, - value_type{4.0, -8.0}}), - (r_mixed() * 1e2)); + l({value_type{-6.0, 12.0}, value_type{-1.5, 3.0}, + value_type{6.0, -12.0}}), + (r_mixed()) * 1e3); } diff --git a/reference/test/solver/chebyshev_kernels.cpp b/reference/test/solver/chebyshev_kernels.cpp index 04f2ba52656..f83d8878b05 100644 --- a/reference/test/solver/chebyshev_kernels.cpp +++ b/reference/test/solver/chebyshev_kernels.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -22,11 +23,12 @@ class Chebyshev : public ::testing::Test { protected: using value_type = T; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Chebyshev; using coeff_type = gko::solver::detail::coeff_type; Chebyshev() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{0.9, -1.0, 3.0}, {0.0, 1.0, 3.0}, {0.0, 0.0, 1.1}}, exec)), // Eigenvalues of mtx are 0.9, 1.0 and 1.1 chebyshev_factory( @@ -51,7 +53,7 @@ class Chebyshev : public ::testing::Test { {} std::shared_ptr exec; - std::shared_ptr mtx; + std::shared_ptr mtx; std::unique_ptr chebyshev_factory; coeff_type alpha; coeff_type beta; diff --git a/reference/test/solver/direct.cpp b/reference/test/solver/direct.cpp index 3ae8203f690..79ceca3b059 100644 --- a/reference/test/solver/direct.cpp +++ b/reference/test/solver/direct.cpp @@ -50,7 +50,7 @@ class Direct : public ::testing::Test { .on(exec); solver = factory->generate(mtx); std::normal_distribution<> dist(0, 1); - x = gko::test::generate_random_dense_matrix( + x = gko::test::generate_random_multi_vector( mtx->get_size()[0], nrhs, dist, rng, this->exec); x_ref = x->clone(); b = x->clone(); diff --git a/reference/test/solver/fcg_kernels.cpp b/reference/test/solver/fcg_kernels.cpp index 68969034e47..f223d0ee12f 100644 --- a/reference/test/solver/fcg_kernels.cpp +++ b/reference/test/solver/fcg_kernels.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -26,11 +27,12 @@ class Fcg : public ::testing::Test { protected: using value_type = T; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Fcg; Fcg() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{2, -1.0, 0.0}, {-1.0, 2, -1.0}, {0.0, -1.0, 2}}, exec)), stopped{}, non_stopped{}, @@ -42,7 +44,7 @@ class Fcg : public ::testing::Test { gko::stop::ResidualNorm::build() .with_reduction_factor(r::value)) .on(exec)), - mtx_big(gko::initialize( + mtx_big(gko::initialize( {{8828.0, 2673.0, 4150.0, -3139.5, 3829.5, 5856.0}, {2673.0, 10765.5, 1805.0, 73.0, 1966.0, 3919.5}, {4150.0, 1805.0, 6472.5, 2656.0, 2409.5, 3836.5}, @@ -89,8 +91,8 @@ class Fcg : public ::testing::Test { } std::shared_ptr exec; - std::shared_ptr mtx; - std::shared_ptr mtx_big; + std::shared_ptr mtx; + std::shared_ptr mtx_big; std::unique_ptr small_one; std::unique_ptr small_zero; std::unique_ptr small_prev_rho; diff --git a/reference/test/solver/gcr_kernels.cpp b/reference/test/solver/gcr_kernels.cpp index b3190d97c93..1a72732c6ab 100644 --- a/reference/test/solver/gcr_kernels.cpp +++ b/reference/test/solver/gcr_kernels.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -33,21 +34,22 @@ class Gcr : public ::testing::Test { using rc_value_type = gko::remove_complex; using Mtx = gko::matrix::MultiVector; using rc_Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Gcr; Gcr() : exec(gko::ReferenceExecutor::create()), stopped{}, non_stopped{}, - mtx(gko::initialize( + mtx(gko::initialize( {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)), mtx_medium( - gko::initialize({{-86.40, 153.30, -108.90, 8.60, -61.60}, - {7.70, -77.00, 3.30, -149.20, 74.80}, - {-121.40, 37.10, 55.30, -74.20, -19.20}, - {-111.40, -22.60, 110.10, -106.20, 88.90}, - {-0.70, 111.70, 154.40, 235.00, -76.50}}, - exec)), - mtx_big(gko::initialize( + gko::initialize({{-86.40, 153.30, -108.90, 8.60, -61.60}, + {7.70, -77.00, 3.30, -149.20, 74.80}, + {-121.40, 37.10, 55.30, -74.20, -19.20}, + {-111.40, -22.60, 110.10, -106.20, 88.90}, + {-0.70, 111.70, 154.40, 235.00, -76.50}}, + exec)), + mtx_big(gko::initialize( {{2295.7, -764.8, 1166.5, 428.9, 291.7, -774.5}, {2752.6, -1127.7, 1212.8, -299.1, 987.7, 786.8}, {138.3, 78.2, 485.5, -899.9, 392.9, 1408.9}, @@ -111,9 +113,9 @@ class Gcr : public ::testing::Test { gko::stopping_status stopped; gko::stopping_status non_stopped; - std::shared_ptr mtx; - std::shared_ptr mtx_medium; - std::shared_ptr mtx_big; + std::shared_ptr mtx; + std::shared_ptr mtx_medium; + std::shared_ptr mtx_big; std::unique_ptr gcr_factory; std::unique_ptr gcr_factory_big; std::unique_ptr gcr_factory_big2; diff --git a/reference/test/solver/gmres_kernels.cpp b/reference/test/solver/gmres_kernels.cpp index 8b1fa74a880..b7f2b5acdea 100644 --- a/reference/test/solver/gmres_kernels.cpp +++ b/reference/test/solver/gmres_kernels.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -34,12 +35,13 @@ class Gmres : public ::testing::Test { using rc_value_type = gko::remove_complex; using Mtx = gko::matrix::MultiVector; using rc_Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Gmres; Gmres() : exec(gko::ReferenceExecutor::create()), stopped{}, non_stopped{}, - mtx(gko::initialize( + mtx(gko::initialize( {{1.0, 2.0, 3.0}, {3.0, 2.0, -1.0}, {0.0, -1.0, 2}}, exec)), gmres_factory( Solver::build() @@ -51,7 +53,7 @@ class Gmres : public ::testing::Test { .with_reduction_factor(r::value)) .with_krylov_dim(3u) .on(exec)), - mtx_big(gko::initialize( + mtx_big(gko::initialize( {{2295.7, -764.8, 1166.5, 428.9, 291.7, -774.5}, {2752.6, -1127.7, 1212.8, -299.1, 987.7, 786.8}, {138.3, 78.2, 485.5, -899.9, 392.9, 1408.9}, @@ -74,12 +76,12 @@ class Gmres : public ::testing::Test { .with_reduction_factor(r::value)) .on(exec)), mtx_medium( - gko::initialize({{-86.40, 153.30, -108.90, 8.60, -61.60}, - {7.70, -77.00, 3.30, -149.20, 74.80}, - {-121.40, 37.10, 55.30, -74.20, -19.20}, - {-111.40, -22.60, 110.10, -106.20, 88.90}, - {-0.70, 111.70, 154.40, 235.00, -76.50}}, - exec)) + gko::initialize({{-86.40, 153.30, -108.90, 8.60, -61.60}, + {7.70, -77.00, 3.30, -149.20, 74.80}, + {-121.40, 37.10, 55.30, -74.20, -19.20}, + {-111.40, -22.60, 110.10, -106.20, 88.90}, + {-0.70, 111.70, 154.40, 235.00, -76.50}}, + exec)) { auto small_size = gko::dim<2>{3, 2}; constexpr gko::size_type small_restart{2}; @@ -128,9 +130,9 @@ class Gmres : public ::testing::Test { gko::stopping_status stopped; gko::stopping_status non_stopped; - std::shared_ptr mtx; - std::shared_ptr mtx_medium; - std::shared_ptr mtx_big; + std::shared_ptr mtx; + std::shared_ptr mtx_medium; + std::shared_ptr mtx_big; std::unique_ptr gmres_factory; std::unique_ptr gmres_factory_big; std::unique_ptr gmres_factory_big2; diff --git a/reference/test/solver/idr_kernels.cpp b/reference/test/solver/idr_kernels.cpp index c4d883dee05..78b3dcedaa9 100644 --- a/reference/test/solver/idr_kernels.cpp +++ b/reference/test/solver/idr_kernels.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -27,11 +28,12 @@ class Idr : public ::testing::Test { protected: using value_type = T; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Idr; Idr() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{1.0, -3.0, 0.0}, {-4.0, 1.0, -3.0}, {2.0, -1.0, 2.0}}, exec)), idr_factory(Solver::build() .with_deterministic(true) @@ -55,7 +57,7 @@ class Idr : public ::testing::Test { {} std::shared_ptr exec; - std::shared_ptr mtx; + std::shared_ptr mtx; std::unique_ptr idr_factory; std::unique_ptr idr_factory_precision; }; @@ -326,6 +328,7 @@ TYPED_TEST(Idr, SolvesMultipleMultiVectorSystemsUsingAdvancedApply) TYPED_TEST(Idr, SolvesBigMultiVectorSystemForDivergenceCheck1) { using Mtx = typename TestFixture::Mtx; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; // the internal vector t will be too large in the first run and then out of // the half precision range. @@ -333,14 +336,14 @@ TYPED_TEST(Idr, SolvesBigMultiVectorSystemForDivergenceCheck1) // rounding error for bfloat16 SKIP_IF_BFLOAT16(value_type); auto half_tol = std::sqrt(r::value); - std::shared_ptr locmtx = - gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, - {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, - {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, - {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, - {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, - {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, - this->exec); + std::shared_ptr locmtx = + gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, + {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, + {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, + {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, + {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, + {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, + this->exec); auto solver = this->idr_factory_precision->generate(locmtx); auto b = gko::initialize({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec); @@ -368,6 +371,7 @@ TYPED_TEST(Idr, SolvesBigMultiVectorSystemForDivergenceCheck1) TYPED_TEST(Idr, SolvesBigMultiVectorSystemForDivergenceCheck2) { using Mtx = typename TestFixture::Mtx; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; // the internal vector t will be too large in the first run and then out of // the half precision range. @@ -375,14 +379,14 @@ TYPED_TEST(Idr, SolvesBigMultiVectorSystemForDivergenceCheck2) // rounding error for bfloat16 SKIP_IF_BFLOAT16(value_type); auto half_tol = std::sqrt(r::value); - std::shared_ptr locmtx = - gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, - {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, - {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, - {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, - {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, - {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, - this->exec); + std::shared_ptr locmtx = + gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, + {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, + {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, + {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, + {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, + {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, + this->exec); auto solver = this->idr_factory_precision->generate(locmtx); auto b = gko::initialize({9.0, -4.0, -6.0, -10.0, 1.0, 10.0}, this->exec); @@ -401,6 +405,7 @@ TYPED_TEST(Idr, SolvesBigMultiVectorSystemForDivergenceCheck2) TYPED_TEST(Idr, SolvesMultipleMultiVectorSystemsDivergenceCheck) { using Mtx = typename TestFixture::Mtx; + using Dense = typename TestFixture::Dense; using value_type = typename TestFixture::value_type; using T = value_type; // the internal vector t will be too large in the first run and then out of @@ -408,14 +413,14 @@ TYPED_TEST(Idr, SolvesMultipleMultiVectorSystemsDivergenceCheck) SKIP_IF_HALF(value_type); // for OSX SKIP_IF_BFLOAT16(value_type); - std::shared_ptr locmtx = - gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, - {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, - {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, - {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, - {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, - {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, - this->exec); + std::shared_ptr locmtx = + gko::initialize({{-19.0, 47.0, -41.0, 35.0, -21.0, 71.0}, + {-8.0, -66.0, 29.0, -96.0, -95.0, -14.0}, + {-93.0, -58.0, -9.0, -87.0, 15.0, 35.0}, + {60.0, -86.0, 54.0, -40.0, -93.0, 56.0}, + {53.0, 94.0, -54.0, 86.0, -61.0, 4.0}, + {-42.0, 57.0, 32.0, 89.0, 89.0, -39.0}}, + this->exec); auto solver = this->idr_factory_precision->generate(locmtx); auto b1 = gko::initialize({0.0, -9.0, -2.0, 8.0, -5.0, -6.0}, this->exec); diff --git a/reference/test/solver/ir_kernels.cpp b/reference/test/solver/ir_kernels.cpp index 28bf5f4fd5c..52c545e7d05 100644 --- a/reference/test/solver/ir_kernels.cpp +++ b/reference/test/solver/ir_kernels.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -26,10 +27,11 @@ class Ir : public ::testing::Test { protected: using value_type = T; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Ir; Ir() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{0.9, -1.0, 3.0}, {0.0, 1.0, 3.0}, {0.0, 0.0, 1.1}}, exec)), // Eigenvalues of mtx are 0.9, 1.0 and 1.1 // Richardson iteration, converges since @@ -43,7 +45,7 @@ class Ir : public ::testing::Test { {} std::shared_ptr exec; - std::shared_ptr mtx; + std::shared_ptr mtx; std::unique_ptr ir_factory; }; diff --git a/reference/test/solver/lower_trs.cpp b/reference/test/solver/lower_trs.cpp index fb7914977ab..09f1709683e 100644 --- a/reference/test/solver/lower_trs.cpp +++ b/reference/test/solver/lower_trs.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -24,7 +25,7 @@ class LowerTrs : public ::testing::Test { typename std::tuple_element<0, decltype(ValueIndexType())>::type; using index_type = typename std::tuple_element<1, decltype(ValueIndexType())>::type; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using CsrMtx = gko::matrix::Csr; using Solver = gko::solver::LowerTrs; diff --git a/reference/test/solver/lower_trs_kernels.cpp b/reference/test/solver/lower_trs_kernels.cpp index 79da3a0caf9..ee705edd14f 100644 --- a/reference/test/solver/lower_trs_kernels.cpp +++ b/reference/test/solver/lower_trs_kernels.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -32,27 +33,29 @@ class LowerTrs : public ::testing::Test { using index_type = typename std::tuple_element<1, decltype(ValueIndexType())>::type; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::LowerTrs; LowerTrs() : exec(gko::ReferenceExecutor::create()), ref(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{1, 0.0, 0.0}, {3.0, 1, 0.0}, {1.0, 2.0, 1}}, exec)), - mtx2(gko::initialize( + mtx2(gko::initialize( {{2, 0.0, 0.0}, {3.0, 3, 0.0}, {1.0, 2.0, 4}}, exec)), - mtx_big_lower(gko::initialize({{124.0, 0.0, 0.0, 0.0, 0.0}, - {43.0, -789.0, 0.0, 0.0, 0.0}, - {134.5, -651.0, 654.0, 0.0, 0.0}, - {-642.0, 684.0, 68.0, 387.0, 0.0}, - {365.0, 97.0, -654.0, 8.0, 91.0}}, - exec)), + mtx_big_lower( + gko::initialize({{124.0, 0.0, 0.0, 0.0, 0.0}, + {43.0, -789.0, 0.0, 0.0, 0.0}, + {134.5, -651.0, 654.0, 0.0, 0.0}, + {-642.0, 684.0, 68.0, 387.0, 0.0}, + {365.0, 97.0, -654.0, 8.0, 91.0}}, + exec)), mtx_big_general( - gko::initialize({{124.0, 4.0, -4.0, 0.0, 2.0}, - {43.0, -789.0, 0.0, 2.0, 1.0}, - {134.5, -651.0, 654.0, 0.0, 0.5}, - {-642.0, 684.0, 68.0, 387.0, 0.0}, - {365.0, 97.0, -654.0, 8.0, 91.0}}, - exec)), + gko::initialize({{124.0, 4.0, -4.0, 0.0, 2.0}, + {43.0, -789.0, 0.0, 2.0, 1.0}, + {134.5, -651.0, 654.0, 0.0, 0.5}, + {-642.0, 684.0, 68.0, 387.0, 0.0}, + {365.0, 97.0, -654.0, 8.0, 91.0}}, + exec)), lower_trs_factory(Solver::build().on(exec)), lower_trs_syncfree_factory( Solver::build() @@ -65,10 +68,10 @@ class LowerTrs : public ::testing::Test { std::shared_ptr exec; std::shared_ptr ref; - std::shared_ptr mtx; - std::shared_ptr mtx2; - std::shared_ptr mtx_big_lower; - std::shared_ptr mtx_big_general; + std::shared_ptr mtx; + std::shared_ptr mtx2; + std::shared_ptr mtx_big_lower; + std::shared_ptr mtx_big_general; std::unique_ptr lower_trs_factory; std::unique_ptr lower_trs_syncfree_factory; std::unique_ptr lower_trs_factory_mrhs; diff --git a/reference/test/solver/minres_kernels.cpp b/reference/test/solver/minres_kernels.cpp index 932eeb1a36a..a6aa10a95ad 100644 --- a/reference/test/solver/minres_kernels.cpp +++ b/reference/test/solver/minres_kernels.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -27,11 +28,12 @@ class Minres : public ::testing::Test { protected: using value_type = T; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Minres; Minres() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{3, -1, -1, 0}, {-1, 3, -1, -1}, {-1, -1, 0, 0}, {0, -1, 0, 0}}, exec)), zero(gko::initialize(I>{{0, 0}, {0, 0}}, exec)), @@ -86,7 +88,7 @@ class Minres : public ::testing::Test { std::shared_ptr exec; - std::shared_ptr mtx; + std::shared_ptr mtx; std::unique_ptr zero; std::unique_ptr zero_scalar; diff --git a/reference/test/solver/multigrid_kernels.cpp b/reference/test/solver/multigrid_kernels.cpp index c6a0f2aece1..af021b1de95 100644 --- a/reference/test/solver/multigrid_kernels.cpp +++ b/reference/test/solver/multigrid_kernels.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -47,7 +48,7 @@ class DummyLinOp : public gko::LinOp, public: DummyLinOp(std::shared_ptr exec, gko::dim<2> size = gko::dim<2>{}) - : LinOp(exec, size) + : LinOp(exec, size, gko::precision::fp64) {} bool apply_uses_initial_guess() const override { return true; } @@ -70,7 +71,7 @@ class DummyRestrictOp : public gko::LinOp, DummyRestrictOp(std::shared_ptr exec, gko::dim<2> size = gko::dim<2>{}) - : LinOp(exec, size) + : LinOp(exec, size, gko::precision::fp64) {} bool apply_uses_initial_guess() const override { return true; } @@ -97,7 +98,7 @@ class DummyProlongOp : public gko::LinOp, DummyProlongOp(std::shared_ptr exec, gko::dim<2> size = gko::dim<2>{}) - : LinOp(exec, size) + : LinOp(exec, size, gko::precision::fp64) {} bool apply_uses_initial_guess() const override { return true; } diff --git a/reference/test/solver/pipe_cg_kernels.cpp b/reference/test/solver/pipe_cg_kernels.cpp index c13008d0562..c7d5d427555 100644 --- a/reference/test/solver/pipe_cg_kernels.cpp +++ b/reference/test/solver/pipe_cg_kernels.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -22,12 +23,13 @@ class PipeCg : public ::testing::Test { protected: using value_type = T; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::PipeCg; PipeCg() : exec(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{2, -1.0, 0.0}, {-1.0, 2, -1.0}, {0.0, -1.0, 2}}, exec)), - mtx_big(gko::initialize( + mtx_big(gko::initialize( {{8828.0, 2673.0, 4150.0, -3139.5, 3829.5, 5856.0}, {2673.0, 10765.5, 1805.0, 73.0, 1966.0, 3919.5}, {4150.0, 1805.0, 6472.5, 2656.0, 2409.5, 3836.5}, @@ -92,8 +94,8 @@ class PipeCg : public ::testing::Test { } std::shared_ptr exec; - std::shared_ptr mtx; - std::shared_ptr mtx_big; + std::shared_ptr mtx; + std::shared_ptr mtx_big; std::unique_ptr small_one; std::unique_ptr small_zero; std::unique_ptr small_prev_rho; diff --git a/reference/test/solver/upper_trs.cpp b/reference/test/solver/upper_trs.cpp index 4a6b2b85392..033dc309086 100644 --- a/reference/test/solver/upper_trs.cpp +++ b/reference/test/solver/upper_trs.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -25,7 +26,7 @@ class UpperTrs : public ::testing::Test { using index_type = typename std::tuple_element<1, decltype(ValueIndexType())>::type; using CsrMtx = gko::matrix::Csr; - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Solver = gko::solver::UpperTrs; UpperTrs() diff --git a/reference/test/solver/upper_trs_kernels.cpp b/reference/test/solver/upper_trs_kernels.cpp index 751be959e29..f8561f2a86e 100644 --- a/reference/test/solver/upper_trs_kernels.cpp +++ b/reference/test/solver/upper_trs_kernels.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -32,27 +33,29 @@ class UpperTrs : public ::testing::Test { using index_type = typename std::tuple_element<1, decltype(ValueIndexType())>::type; using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::UpperTrs; UpperTrs() : exec(gko::ReferenceExecutor::create()), ref(gko::ReferenceExecutor::create()), - mtx(gko::initialize( + mtx(gko::initialize( {{1, 3.0, 1.0}, {0.0, 1, 2.0}, {0.0, 0.0, 1}}, exec)), - mtx2(gko::initialize( + mtx2(gko::initialize( {{2, 3.0, 1.0}, {0.0, 3, 2.0}, {0.0, 0.0, 4}}, exec)), - mtx_big_upper(gko::initialize({{365.0, 97.0, -654.0, 8.0, 91.0}, - {0.0, -642.0, 684.0, 68.0, 387.0}, - {0.0, 0.0, 134, -651.0, 654.0}, - {0.0, 0.0, 0.0, 43.0, -789.0}, - {0.0, 0.0, 0.0, 0.0, 124.0}}, - exec)), + mtx_big_upper( + gko::initialize({{365.0, 97.0, -654.0, 8.0, 91.0}, + {0.0, -642.0, 684.0, 68.0, 387.0}, + {0.0, 0.0, 134, -651.0, 654.0}, + {0.0, 0.0, 0.0, 43.0, -789.0}, + {0.0, 0.0, 0.0, 0.0, 124.0}}, + exec)), mtx_big_general( - gko::initialize({{365.0, 97.0, -654.0, 8.0, 91.0}, - {6.0, -642.0, 684.0, 68.0, 387.0}, - {0.0, 0.0, 134, -651.0, 654.0}, - {0.0, 0.0, -1.0, 43.0, -789.0}, - {0.0, 2.0, 0.0, 4.0, 124.0}}, - exec)), + gko::initialize({{365.0, 97.0, -654.0, 8.0, 91.0}, + {6.0, -642.0, 684.0, 68.0, 387.0}, + {0.0, 0.0, 134, -651.0, 654.0}, + {0.0, 0.0, -1.0, 43.0, -789.0}, + {0.0, 2.0, 0.0, 4.0, 124.0}}, + exec)), upper_trs_factory(Solver::build().on(exec)), upper_trs_syncfree_factory( Solver::build() @@ -65,10 +68,10 @@ class UpperTrs : public ::testing::Test { std::shared_ptr exec; std::shared_ptr ref; - std::shared_ptr mtx; - std::shared_ptr mtx2; - std::shared_ptr mtx_big_upper; - std::shared_ptr mtx_big_general; + std::shared_ptr mtx; + std::shared_ptr mtx2; + std::shared_ptr mtx_big_upper; + std::shared_ptr mtx_big_general; std::unique_ptr upper_trs_factory; std::unique_ptr upper_trs_syncfree_factory; std::unique_ptr upper_trs_factory_mrhs; diff --git a/reference/test/stop/residual_norm_kernels.cpp b/reference/test/stop/residual_norm_kernels.cpp index 747934292b0..4dd74f9ff26 100644 --- a/reference/test/stop/residual_norm_kernels.cpp +++ b/reference/test/stop/residual_norm_kernels.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -21,6 +22,7 @@ template class ResidualNorm : public ::testing::Test { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using NormVector = gko::matrix::MultiVector>; using ValueType = T; @@ -74,13 +76,17 @@ TYPED_TEST(ResidualNorm, CanCreateFactory) TYPED_TEST(ResidualNorm, CheckIfResZeroConverges) { using Mtx = typename TestFixture::Mtx; + using Dense = typename TestFixture::Dense; using NormVector = typename TestFixture::NormVector; using T = typename TestFixture::ValueType; using gko::stop::mode; - std::shared_ptr mtx = gko::initialize({1.0}, this->exec_); - std::shared_ptr rhs = gko::initialize({0.0}, this->exec_); - std::shared_ptr x = gko::initialize({0.0}, this->exec_); - std::shared_ptr res_norm = + std::shared_ptr mtx = + gko::initialize({1.0}, this->exec_); + std::shared_ptr rhs = + gko::initialize({0.0}, this->exec_); + std::shared_ptr x = + gko::initialize({0.0}, this->exec_); + std::shared_ptr res_norm = gko::initialize({0.0}, this->exec_); for (auto baseline : @@ -123,7 +129,7 @@ TYPED_TEST(ResidualNorm, CannotCreateCriterionWithoutNeededInput) TYPED_TEST(ResidualNorm, CanCreateCriterionWithNeededInput) { using Mtx = typename TestFixture::Mtx; - std::shared_ptr scalar = + std::shared_ptr scalar = gko::initialize({1.0}, this->exec_); auto rhs_criterion = this->rhs_factory_->generate(nullptr, scalar, nullptr, nullptr); @@ -141,7 +147,7 @@ TYPED_TEST(ResidualNorm, CanCreateCriterionWithNeededInput) TYPED_TEST(ResidualNorm, CanIgorneResidualNorm) { using Mtx = typename TestFixture::Mtx; - std::shared_ptr scalar = + std::shared_ptr scalar = gko::initialize({1.0}, this->exec_); auto criterion = this->rhs_factory_->generate(nullptr, scalar, nullptr, nullptr); @@ -164,7 +170,8 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal) using NormVector = typename TestFixture::NormVector; using T_nc = gko::remove_complex; auto initial_res = gko::initialize({100.0}, this->exec_); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec_); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec_); auto rhs_criterion = this->rhs_factory_->generate(nullptr, rhs, nullptr, initial_res.get()); auto rel_criterion = @@ -252,7 +259,7 @@ TYPED_TEST(ResidualNorm, SelfCalculatesThrowWithoutMatrix) auto initial_res = gko::initialize({100.0}, this->exec_); T rhs_val = 10.0; - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({rhs_val}, this->exec_); auto rhs_criterion = this->rhs_factory_->generate(nullptr, rhs, nullptr, initial_res.get()); @@ -328,14 +335,15 @@ TYPED_TEST(ResidualNorm, RelativeSelfCalculatesThrowWithoutRhs) TYPED_TEST(ResidualNorm, SelfCalculatesAndWaitsTillResidualGoal) { using Mtx = typename TestFixture::Mtx; + using Dense = typename TestFixture::Dense; using NormVector = typename TestFixture::NormVector; using T = TypeParam; using T_nc = gko::remove_complex; auto initial_res = gko::initialize({100.0}, this->exec_); - auto system_mtx = share(gko::initialize({1.0}, this->exec_)); + auto system_mtx = share(gko::initialize({1.0}, this->exec_)); T rhs_val = 10.0; - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({rhs_val}, this->exec_); auto rhs_criterion = this->rhs_factory_->generate(system_mtx, rhs, nullptr, initial_res.get()); @@ -425,7 +433,7 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoalMultipleRHS) using T = TypeParam; using T_nc = gko::remove_complex; auto res = gko::initialize({I{100.0, 100.0}}, this->exec_); - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({I{10.0, 10.0}}, this->exec_); auto rhs_criterion = this->rhs_factory_->generate(nullptr, rhs, nullptr, res.get()); @@ -517,7 +525,8 @@ TYPED_TEST(ResidualNorm, WorksWithMinIterationCount) using NormVector = typename TestFixture::NormVector; using T_nc = gko::remove_complex; auto initial_res = gko::initialize({100.0}, this->exec_); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec_); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec_); auto min_factory = gko::stop::min_iters( 10, gko::stop::ResidualNorm::build() .with_baseline(gko::stop::mode::absolute) @@ -576,7 +585,8 @@ TYPED_TEST(ResidualNorm, SimplifiedInterface) using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec_); auto initial_guess = gko::initialize({1000.0}, this->exec_); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec_); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec_); auto factory_abs = gko::stop::absolute_residual_norm(0.5).on(this->exec_); auto factory_rel = gko::stop::relative_residual_norm(0.5).on(this->exec_); @@ -608,6 +618,7 @@ template class ResidualNormWithInitialResnorm : public ::testing::Test { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using NormVector = gko::matrix::MultiVector>; ResidualNormWithInitialResnorm() @@ -631,9 +642,13 @@ TYPED_TEST(ResidualNormWithInitialResnorm, CanCreateCriterionWithMtxRhsXWithoutInitialRes) { using Mtx = typename TestFixture::Mtx; - std::shared_ptr x = gko::initialize({100.0}, this->exec_); - std::shared_ptr mtx = gko::initialize({1.0}, this->exec_); - std::shared_ptr b = gko::initialize({10.0}, this->exec_); + using Dense = typename TestFixture::Dense; + std::shared_ptr x = + gko::initialize({100.0}, this->exec_); + std::shared_ptr mtx = + gko::initialize({1.0}, this->exec_); + std::shared_ptr b = + gko::initialize({10.0}, this->exec_); auto criterion = this->factory_->generate(mtx, b, x.get()); @@ -646,7 +661,8 @@ TYPED_TEST(ResidualNormWithInitialResnorm, WaitsTillResidualGoal) using Mtx = typename TestFixture::Mtx; using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec_); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec_); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec_); auto res_norm = gko::initialize({100.0}, this->exec_); auto init_res_val = res_norm->at(0, 0); auto criterion = @@ -678,14 +694,16 @@ TYPED_TEST(ResidualNormWithInitialResnorm, { using T = TypeParam; using Mtx = typename TestFixture::Mtx; + using Dense = typename TestFixture::Dense; using NormVector = typename TestFixture::NormVector; T initial_res = 100; T rhs_val = 10; - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({rhs_val}, this->exec_); std::shared_ptr x = gko::initialize({rhs_val - initial_res}, this->exec_); - std::shared_ptr mtx = gko::initialize({1.0}, this->exec_); + std::shared_ptr mtx = + gko::initialize({1.0}, this->exec_); auto criterion = this->factory_->generate(mtx, rhs, x.get()); bool one_changed{}; @@ -719,7 +737,7 @@ TYPED_TEST(ResidualNormWithInitialResnorm, WaitsTillResidualGoalMultipleRHS) auto res = gko::initialize({I{100.0, 100.0}}, this->exec_); auto res_norm = gko::initialize({I{100.0, 100.0}}, this->exec_); - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({I{10.0, 10.0}}, this->exec_); auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get()); bool one_changed{}; @@ -787,7 +805,7 @@ TYPED_TEST(ResidualNormWithRhsNorm, CannotCreateCriterionWithoutB) TYPED_TEST(ResidualNormWithRhsNorm, CanCreateCriterionWithB) { using Mtx = typename TestFixture::Mtx; - std::shared_ptr scalar = + std::shared_ptr scalar = gko::initialize({1.0}, this->exec_); auto criterion = this->factory_->generate(nullptr, scalar, nullptr, nullptr); @@ -803,7 +821,8 @@ TYPED_TEST(ResidualNormWithRhsNorm, WaitsTillResidualGoal) using Mtx = typename TestFixture::Mtx; using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec_); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec_); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec_); auto rhs_norm = gko::initialize({I{0.0}}, this->exec_); gko::as(rhs)->compute_norm2(rhs_norm); auto res_norm = gko::initialize({100.0}, this->exec_); @@ -840,7 +859,7 @@ TYPED_TEST(ResidualNormWithRhsNorm, WaitsTillResidualGoalMultipleRHS) auto res = gko::initialize({I{100.0, 100.0}}, this->exec_); auto res_norm = gko::initialize({I{100.0, 100.0}}, this->exec_); - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({I{10.0, 10.0}}, this->exec_); auto rhs_norm = gko::initialize({I{0.0, 0.0}}, this->exec_); @@ -873,6 +892,7 @@ template class ImplicitResidualNorm : public ::testing::Test { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using NormVector = gko::matrix::MultiVector>; using ValueType = T; @@ -922,12 +942,16 @@ TYPED_TEST(ImplicitResidualNorm, CanCreateFactory) TYPED_TEST(ImplicitResidualNorm, CheckIfResZeroConverges) { using Mtx = typename TestFixture::Mtx; + using Dense = typename TestFixture::Dense; using T = typename TestFixture::ValueType; using gko::stop::mode; - std::shared_ptr mtx = gko::initialize({1.0}, this->exec_); - std::shared_ptr rhs = gko::initialize({0.0}, this->exec_); - std::shared_ptr x = gko::initialize({0.0}, this->exec_); - std::shared_ptr implicit_sq_res_norm = + std::shared_ptr mtx = + gko::initialize({1.0}, this->exec_); + std::shared_ptr rhs = + gko::initialize({0.0}, this->exec_); + std::shared_ptr x = + gko::initialize({0.0}, this->exec_); + std::shared_ptr implicit_sq_res_norm = gko::initialize({0.0}, this->exec_); for (auto baseline : @@ -965,7 +989,7 @@ TYPED_TEST(ImplicitResidualNorm, CannotCreateCriterionWithoutBAndInitRes) TYPED_TEST(ImplicitResidualNorm, CanCreateCriterionWithB) { using Mtx = typename TestFixture::Mtx; - std::shared_ptr scalar = + std::shared_ptr scalar = gko::initialize({1.0}, this->exec_); auto criterion = this->factory_->generate(nullptr, scalar, nullptr, nullptr); @@ -991,7 +1015,8 @@ TYPED_TEST(ImplicitResidualNorm, WaitsTillResidualGoal) using Mtx = typename TestFixture::Mtx; using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec_); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec_); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec_); auto res_norm = gko::initialize({100.0}, this->exec_); auto rhs_norm = gko::initialize({I{0.0}}, this->exec_); gko::as(rhs)->compute_norm2(rhs_norm); @@ -1027,7 +1052,7 @@ TYPED_TEST(ImplicitResidualNorm, WaitsTillResidualGoalMultipleRHS) using T_nc = gko::remove_complex; auto res = gko::initialize({I{100.0, 100.0}}, this->exec_); auto res_norm = gko::initialize({I{100.0, 100.0}}, this->exec_); - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({I{10.0, 10.0}}, this->exec_); auto rhs_norm = gko::initialize({I{0.0, 0.0}}, this->exec_); @@ -1064,7 +1089,8 @@ TYPED_TEST(ImplicitResidualNorm, SimplifiedInterface) using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec_); auto initial_guess = gko::initialize({1000.0}, this->exec_); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec_); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec_); auto factory_abs = gko::stop::absolute_implicit_residual_norm(0.5).on(this->exec_); @@ -1137,7 +1163,7 @@ TYPED_TEST(ResidualNormWithAbsolute, CannotCreateCriterionWithoutB) TYPED_TEST(ResidualNormWithAbsolute, CanCreateCriterionWithB) { using Mtx = typename TestFixture::Mtx; - std::shared_ptr scalar = + std::shared_ptr scalar = gko::initialize({1.0}, this->exec_); auto criterion = this->factory_->generate(nullptr, scalar, nullptr, nullptr); @@ -1151,7 +1177,8 @@ TYPED_TEST(ResidualNormWithAbsolute, WaitsTillResidualGoal) using Mtx = typename TestFixture::Mtx; using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec_); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec_); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec_); auto res_norm = gko::initialize({100.0}, this->exec_); auto criterion = this->factory_->generate(nullptr, rhs, nullptr, initial_res.get()); @@ -1186,7 +1213,7 @@ TYPED_TEST(ResidualNormWithAbsolute, WaitsTillResidualGoalMultipleRHS) auto res = gko::initialize({I{100.0, 100.0}}, this->exec_); auto res_norm = gko::initialize({I{100.0, 100.0}}, this->exec_); - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({I{10.0, 10.0}}, this->exec_); auto criterion = this->factory_->generate(nullptr, rhs, nullptr, res.get()); bool one_changed{}; diff --git a/reference/test/utils/assertions_test.cpp b/reference/test/utils/assertions_test.cpp index 25a85411ab6..dfb01969d9d 100644 --- a/reference/test/utils/assertions_test.cpp +++ b/reference/test/utils/assertions_test.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include "core/test/utils.hpp" @@ -23,7 +24,7 @@ TYPED_TEST_SUITE(MatricesNear, gko::test::ValueTypes, TypenameNameGenerator); TYPED_TEST(MatricesNear, CanPassAnyMatrixType) { auto exec = gko::ReferenceExecutor::create(); - auto mtx = gko::initialize>( + auto mtx = gko::initialize>( {{1.0, 2.0, 3.0}, {0.0, 4.0, 0.0}}, exec); auto csr_mtx = gko::matrix::Csr::create(exec); diff --git a/test/base/batch_multi_vector_kernels.cpp b/test/base/batch_multi_vector_kernels.cpp index 360bac96d60..f58f7867bca 100644 --- a/test/base/batch_multi_vector_kernels.cpp +++ b/test/base/batch_multi_vector_kernels.cpp @@ -11,6 +11,7 @@ #include #include +#include #include "core/base/batch_utilities.hpp" #include "core/test/utils.hpp" diff --git a/test/factorization/cholesky_kernels.cpp b/test/factorization/cholesky_kernels.cpp index 5210c4b82b9..db7a94dbc07 100644 --- a/test/factorization/cholesky_kernels.cpp +++ b/test/factorization/cholesky_kernels.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2017 - 2025 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "core/components/disjoint_sets.hpp" diff --git a/test/factorization/ic_kernels.cpp b/test/factorization/ic_kernels.cpp index 5b5d8f4f225..d6cf4c08ba6 100644 --- a/test/factorization/ic_kernels.cpp +++ b/test/factorization/ic_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "core/test/utils.hpp" #include "core/test/utils/unsort_matrix.hpp" diff --git a/test/factorization/ilu_kernels.cpp b/test/factorization/ilu_kernels.cpp index 3b0e8d34fe1..26cd88e36d9 100644 --- a/test/factorization/ilu_kernels.cpp +++ b/test/factorization/ilu_kernels.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "core/test/utils.hpp" #include "core/test/utils/unsort_matrix.hpp" diff --git a/test/factorization/par_ict_kernels.cpp b/test/factorization/par_ict_kernels.cpp index f0c37abe1aa..0a0a9f53432 100644 --- a/test/factorization/par_ict_kernels.cpp +++ b/test/factorization/par_ict_kernels.cpp @@ -112,8 +112,8 @@ TYPED_TEST(ParIct, KernelAddCandidatesIsEquivalentToRef) } this->dmtx_l->copy_from(this->mtx_l); } - auto mtx_llh = Csr::create(this->ref, this->mtx_size); - this->mtx_l->apply(this->mtx_l->conj_transpose(), mtx_llh); + auto mtx_llh = + this->mtx_l->multiply(gko::as(this->mtx_l->conj_transpose())); auto dmtx_llh = Csr::create(this->exec, this->mtx_size); dmtx_llh->copy_from(mtx_llh); auto res_mtx_l = Csr::create(this->ref, this->mtx_size); diff --git a/test/factorization/par_ilut_kernels.cpp b/test/factorization/par_ilut_kernels.cpp index 2940357316e..4e7fd76c609 100644 --- a/test/factorization/par_ilut_kernels.cpp +++ b/test/factorization/par_ilut_kernels.cpp @@ -448,8 +448,7 @@ TYPED_TEST(ParIlut, KernelAddCandidatesIsEquivalentToRef) this->dmtx_u->copy_from(this->mtx_u); } auto square_size = this->mtx_square->get_size(); - auto mtx_lu = Csr::create(this->ref, square_size); - this->mtx_l2->apply(this->mtx_u, mtx_lu); + auto mtx_lu = this->mtx_l2->multiply(this->mtx_u); auto dmtx_lu = Csr::create(this->exec, square_size); dmtx_lu->copy_from(mtx_lu); auto res_mtx_l = Csr::create(this->ref, square_size); diff --git a/test/matrix/CMakeLists.txt b/test/matrix/CMakeLists.txt index 89e4a9da6b6..8afff93a5c5 100644 --- a/test/matrix/CMakeLists.txt +++ b/test/matrix/CMakeLists.txt @@ -6,6 +6,7 @@ ginkgo_create_common_test(csr_builder DISABLE_EXECUTORS omp) ginkgo_create_common_device_test(csr_kernels) ginkgo_create_common_test(csr_kernels2) ginkgo_create_common_test(coo_kernels) +ginkgo_create_common_test(dense_kernels) ginkgo_create_common_device_test(device_views) ginkgo_create_common_test(diagonal_kernels) ginkgo_create_common_test(ell_kernels) diff --git a/test/matrix/coo_kernels.cpp b/test/matrix/coo_kernels.cpp index eb6585353cb..f9e005645e3 100644 --- a/test/matrix/coo_kernels.cpp +++ b/test/matrix/coo_kernels.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -288,11 +289,11 @@ TEST_F(Coo, ApplyAddToComplexIsEquivalentToRef) } -TEST_F(Coo, ConvertToMultiVectorIsEquivalentToRef) +TEST_F(Coo, ConvertToDenseIsEquivalentToRef) { set_up_apply_data(); - auto dense_mtx = gko::matrix::MultiVector::create(ref); - auto ddense_mtx = gko::matrix::MultiVector::create(exec); + auto dense_mtx = gko::matrix::Dense::create(ref); + auto ddense_mtx = gko::matrix::Dense::create(exec); mtx->convert_to(dense_mtx); dmtx->convert_to(ddense_mtx); @@ -304,7 +305,7 @@ TEST_F(Coo, ConvertToMultiVectorIsEquivalentToRef) TEST_F(Coo, ConvertToCsrIsEquivalentToRef) { set_up_apply_data(); - auto dense_mtx = gko::matrix::MultiVector::create(ref); + auto dense_mtx = gko::matrix::Dense::create(ref); auto csr_mtx = gko::matrix::Csr::create(ref); auto dcsr_mtx = gko::matrix::Csr::create(exec); diff --git a/test/matrix/csr_kernels.cpp b/test/matrix/csr_kernels.cpp index a33afaebe50..e7b87e68dee 100644 --- a/test/matrix/csr_kernels.cpp +++ b/test/matrix/csr_kernels.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include "common/unified/base/kernel_launch.hpp" diff --git a/test/matrix/csr_kernels2.cpp b/test/matrix/csr_kernels2.cpp index f29f5e6b342..ea9e774a4c1 100644 --- a/test/matrix/csr_kernels2.cpp +++ b/test/matrix/csr_kernels2.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -79,9 +80,9 @@ class Csr : public CommonTestFixture { void set_up_apply_data(int num_vectors = 1) { mtx = Mtx::create(ref, strategy); - mtx->move_from(gen_mtx(mtx_size[0], mtx_size[1], 1)); + mtx->move_from(gen_mtx(mtx_size[0], mtx_size[1], 1)); square_mtx = Mtx::create(ref, strategy); - square_mtx->move_from(gen_mtx(mtx_size[0], mtx_size[0], 1)); + square_mtx->move_from(gen_mtx(mtx_size[0], mtx_size[0], 1)); expected = gen_mtx(mtx_size[0], num_vectors, 1); y = gen_mtx(mtx_size[1], num_vectors, 1); alpha = gko::initialize({2.0}, ref); @@ -125,7 +126,7 @@ class Csr : public CommonTestFixture { { complex_mtx = ComplexMtx::create(ref, strategy); complex_mtx->move_from( - gen_mtx(mtx_size[0], mtx_size[1], 1)); + gen_mtx(mtx_size[0], mtx_size[1], 1)); dcomplex_mtx = ComplexMtx::create(exec, strategy); dcomplex_mtx->copy_from(complex_mtx); } @@ -500,21 +501,6 @@ TEST_F(Csr, OneAutomaticWorksWithDifferentMatrices) #endif -TEST_F(Csr, AdvancedApplyToCsrMatrixIsEquivalentToRef) -{ - set_up_apply_data(); - auto trans = mtx->transpose(); - auto dtrans = dmtx->transpose(); - - mtx->apply(alpha, trans, beta, square_mtx); - dmtx->apply(dalpha, dtrans, dbeta, dsquare_mtx); - - GKO_ASSERT_MTX_EQ_SPARSITY(dsquare_mtx, square_mtx); - GKO_ASSERT_MTX_NEAR(dsquare_mtx, square_mtx, r::value); - ASSERT_TRUE(dsquare_mtx->is_sorted_by_column_index()); -} - - TEST_F(Csr, MultiplyAddIsEquivalentToRef) { set_up_apply_data(); @@ -583,21 +569,6 @@ TEST_F(Csr, MultiplyAddReuseUpdateCrossExecutor) } -TEST_F(Csr, SimpleApplyToCsrMatrixIsEquivalentToRef) -{ - set_up_apply_data(); - auto trans = mtx->transpose(); - auto dtrans = dmtx->transpose(); - - mtx->apply(trans, square_mtx); - dmtx->apply(dtrans, dsquare_mtx); - - GKO_ASSERT_MTX_EQ_SPARSITY(dsquare_mtx, square_mtx); - GKO_ASSERT_MTX_NEAR(dsquare_mtx, square_mtx, r::value); - ASSERT_TRUE(dsquare_mtx->is_sorted_by_column_index()); -} - - TEST_F(Csr, MultiplyIsEquivalentToRefCrossExecutor) { set_up_apply_data(); @@ -696,26 +667,6 @@ TEST_F(Csr, MultiplyReuseUpdateCrossExecutor) } -TEST_F(Csr, AdvancedApplyToIdentityMatrixIsEquivalentToRef) -{ - set_up_apply_data(); - auto a = gen_mtx(mtx_size[0], mtx_size[1], 0); - auto b = gen_mtx(mtx_size[0], mtx_size[1], 0); - auto da = gko::clone(exec, a); - auto db = gko::clone(exec, b); - auto id = gko::matrix::Identity::create(ref, mtx_size[1]); - auto did = - gko::matrix::Identity::create(exec, mtx_size[1]); - - a->apply(alpha, id, beta, b); - da->apply(dalpha, did, dbeta, db); - - GKO_ASSERT_MTX_NEAR(b, db, r::value); - GKO_ASSERT_MTX_EQ_SPARSITY(b, db); - ASSERT_TRUE(db->is_sorted_by_column_index()); -} - - TEST_F(Csr, ScaleAddZeroIsEquivalentToRef) { set_up_apply_data(); @@ -941,11 +892,11 @@ TEST_F(Csr, ConjugateTranspose64IsEquivalentToRef) } -TEST_F(Csr, ConvertToMultiVectorIsEquivalentToRef) +TEST_F(Csr, ConvertToDenseIsEquivalentToRef) { set_up_apply_data(); - auto dense_mtx = gko::matrix::MultiVector::create(ref); - auto ddense_mtx = gko::matrix::MultiVector::create(exec); + auto dense_mtx = gko::matrix::Dense::create(ref); + auto ddense_mtx = gko::matrix::Dense::create(exec); mtx->convert_to(dense_mtx); dmtx->convert_to(ddense_mtx); @@ -954,11 +905,11 @@ TEST_F(Csr, ConvertToMultiVectorIsEquivalentToRef) } -TEST_F(Csr, MoveToMultiVectorIsEquivalentToRef) +TEST_F(Csr, MoveToDenseIsEquivalentToRef) { set_up_apply_data(); - auto dense_mtx = gko::matrix::MultiVector::create(ref); - auto ddense_mtx = gko::matrix::MultiVector::create(exec); + auto dense_mtx = gko::matrix::Dense::create(ref); + auto ddense_mtx = gko::matrix::Dense::create(exec); mtx->move_to(dense_mtx); dmtx->move_to(ddense_mtx); diff --git a/test/matrix/dense_kernels.cpp b/test/matrix/dense_kernels.cpp new file mode 100644 index 00000000000..28b01ee604f --- /dev/null +++ b/test/matrix/dense_kernels.cpp @@ -0,0 +1,676 @@ +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors +// +// SPDX-License-Identifier: BSD-3-Clause + +#include "core/matrix/dense_kernels.hpp" + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core/components/fill_array_kernels.hpp" +#include "core/test/utils.hpp" +#include "test/utils/common_fixture.hpp" + + +class Dense : public CommonTestFixture { +protected: + // in single mode, mixed_type will be the same as value_type + using mixed_type = float; + using Mtx = gko::matrix::Dense; + using ComplexMtx = gko::matrix::Dense>; + using Diagonal = gko::matrix::Diagonal; + using Vec = gko::matrix::MultiVector; + using MixedVec = gko::matrix::MultiVector; + using ComplexVec = gko::matrix::MultiVector>; + + Dense() : rand_engine(15) {} + + template + std::unique_ptr gen_mtx(int num_rows, int num_cols) + { + return gko::test::generate_random_matrix( + num_rows, num_cols, + std::uniform_int_distribution<>(num_cols, num_cols), + std::normal_distribution>(0.0, 1.0), + rand_engine, ref); + } + + template + std::unique_ptr gen_mtx(int num_rows, int num_cols, + int min_nnz_row) + { + return gko::test::generate_random_matrix( + num_rows, num_cols, + std::uniform_int_distribution<>(min_nnz_row, num_cols), + std::normal_distribution>(-1.0, + 1.0), + rand_engine, ref); + } + + void set_up_vector_data(gko::size_type num_vecs, + bool different_alpha = false) + { + x = gen_mtx(1000, num_vecs); + y = gen_mtx(1000, num_vecs); + c_x = gen_mtx(1000, num_vecs); + if (different_alpha) { + alpha = gen_mtx(1, num_vecs); + } else { + alpha = gko::initialize({2.0}, ref); + } + dx = gko::clone(exec, x); + dy = gko::clone(exec, y); + dc_x = gko::clone(exec, c_x); + dalpha = gko::clone(exec, alpha); + result = Vec::create(ref, gko::dim<2>{1, num_vecs}); + dresult = Vec::create(exec, gko::dim<2>{1, num_vecs}); + } + + + void set_up_apply_data() + { + x = gen_mtx(65, 25); + y = gen_mtx(25, 35); + c_x = gen_mtx(65, 25); + alpha = gko::initialize({2.0}, ref); + beta = gko::initialize({-1.0}, ref); + result = gen_mtx(65, 35); + c_x = gen_mtx(65, 25); + dx = gko::clone(exec, x); + dy = gko::clone(exec, y); + dc_x = gko::clone(exec, c_x); + dresult = gko::clone(exec, result); + dalpha = gko::clone(exec, alpha); + dbeta = gko::clone(exec, beta); + dc_x = gko::clone(exec, c_x); + } + + template + std::unique_ptr convert(InputType&& input) + { + auto result = ConvertedType::create(input->get_executor()); + input->convert_to(result); + return result; + } + + std::default_random_engine rand_engine; + + std::unique_ptr x; + std::unique_ptr c_x; + std::unique_ptr y; + std::unique_ptr alpha; + std::unique_ptr beta; + std::unique_ptr result; + std::unique_ptr dx; + std::unique_ptr dc_x; + std::unique_ptr dy; + std::unique_ptr dalpha; + std::unique_ptr dbeta; + std::unique_ptr dresult; +}; + + +TEST_F(Dense, SimpleApplyIsEquivalentToRef) +{ + set_up_apply_data(); + + x->apply(y, result); + dx->apply(dy, dresult); + + GKO_ASSERT_MTX_NEAR(dresult, result, r::value); +} + + +TEST_F(Dense, SimpleApplyMixedIsEquivalentToRef) +{ + set_up_apply_data(); + + x->apply(convert(y), convert(result)); + dx->apply(convert(dy), convert(dresult)); + + GKO_ASSERT_MTX_NEAR(dresult, result, 1e-7); +} + + +TEST_F(Dense, AdvancedApplyIsEquivalentToRef) +{ + set_up_apply_data(); + + x->apply(alpha, y, beta, result); + dx->apply(dalpha, dy, dbeta, dresult); + + GKO_ASSERT_MTX_NEAR(dresult, result, r::value); +} + + +TEST_F(Dense, AdvancedApplyMixedIsEquivalentToRef) +{ + set_up_apply_data(); + + x->apply(convert(alpha), convert(y), + convert(beta), convert(result)); + dx->apply(convert(dalpha), convert(dy), + convert(dbeta), convert(dresult)); + + GKO_ASSERT_MTX_NEAR(dresult, result, 1e-7); +} + + +TEST_F(Dense, ApplyToComplexIsEquivalentToRef) +{ + set_up_apply_data(); + auto complex_b = gen_mtx(x->get_size()[1], 1); + auto dcomplex_b = gko::clone(exec, complex_b); + auto complex_x = gen_mtx(x->get_size()[0], 1); + auto dcomplex_x = gko::clone(exec, complex_x); + + x->apply(complex_b, complex_x); + dx->apply(dcomplex_b, dcomplex_x); + + GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r::value); +} + + +TEST_F(Dense, ApplyToMixedComplexIsEquivalentToRef) +{ + set_up_apply_data(); + auto complex_b = gen_mtx(x->get_size()[1], 1); + auto dcomplex_b = gko::clone(exec, complex_b); + auto complex_x = gen_mtx(x->get_size()[0], 1); + auto dcomplex_x = gko::clone(exec, complex_x); + + x->apply(complex_b, complex_x); + dx->apply(dcomplex_b, dcomplex_x); + + GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 2e-7); +} + + +TEST_F(Dense, AdvancedApplyToComplexIsEquivalentToRef) +{ + set_up_apply_data(); + auto complex_b = gen_mtx(x->get_size()[1], 1); + auto dcomplex_b = gko::clone(exec, complex_b); + auto complex_x = gen_mtx(x->get_size()[0], 1); + auto dcomplex_x = gko::clone(exec, complex_x); + + x->apply(alpha, complex_b, beta, complex_x); + dx->apply(dalpha, dcomplex_b, dbeta, dcomplex_x); + + GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, r::value); +} + + +TEST_F(Dense, AdvancedApplyToMixedComplexIsEquivalentToRef) +{ + set_up_apply_data(); + auto complex_b = gen_mtx(x->get_size()[1], 1); + auto dcomplex_b = gko::clone(exec, complex_b); + auto complex_x = gen_mtx(x->get_size()[0], 1); + auto dcomplex_x = gko::clone(exec, complex_x); + + x->apply(convert(alpha), complex_b, convert(beta), + complex_x); + dx->apply(convert(dalpha), dcomplex_b, convert(dbeta), + dcomplex_x); + + GKO_ASSERT_MTX_NEAR(dcomplex_x, complex_x, 2e-7); +} + + +TEST_F(Dense, IsTransposable) +{ + set_up_apply_data(); + + auto trans = x->transpose(); + auto dtrans = dx->transpose(); + + GKO_ASSERT_MTX_NEAR(static_cast(dtrans.get()), + static_cast(trans.get()), 0); +} + + +TEST_F(Dense, IsTransposableIntoDenseCrossExecutor) +{ + set_up_apply_data(); + auto row_span = gko::local_span{0, x->get_size()[0] - 2}; + auto col_span = gko::local_span{0, x->get_size()[1] - 2}; + auto sub_x = x->create_subview(row_span, col_span); + auto sub_dx = dx->create_subview(row_span, col_span); + // create the target matrices on another executor to + // force temporary clone + auto trans = Mtx::create(ref, gko::transpose(sub_x->get_size())); + auto dtrans = Mtx::create(ref, gko::transpose(sub_x->get_size()), + sub_x->get_size()[0] + 4); + + sub_x->transpose(trans); + sub_dx->transpose(dtrans); + + GKO_ASSERT_MTX_NEAR(dtrans, trans, 0); +} + + +TEST_F(Dense, IsConjugateTransposable) +{ + set_up_apply_data(); + + auto trans = c_x->conj_transpose(); + auto dtrans = dc_x->conj_transpose(); + + GKO_ASSERT_MTX_NEAR(static_cast(dtrans.get()), + static_cast(trans.get()), 0); +} + + +TEST_F(Dense, IsConjugateTransposableIntoDenseCrossExecutor) +{ + set_up_apply_data(); + auto row_span = gko::local_span{0, c_x->get_size()[0] - 2}; + auto col_span = gko::local_span{0, c_x->get_size()[1] - 2}; + auto sub_x = c_x->create_subview(row_span, col_span); + auto sub_dx = dc_x->create_subview(row_span, col_span); + // create the target matrices on another executor to + // force temporary clone + auto trans = ComplexMtx::create(ref, gko::transpose(sub_x->get_size())); + auto dtrans = ComplexMtx::create(ref, gko::transpose(sub_x->get_size()), + sub_x->get_size()[0] + 4); + + sub_x->conj_transpose(trans); + sub_dx->conj_transpose(dtrans); + + GKO_ASSERT_MTX_NEAR(dtrans, trans, 0); +} + + +TEST_F(Dense, ExtractDiagonalOnTallSkinnyIsEquivalentToRef) +{ + set_up_apply_data(); + + auto diag = x->extract_diagonal(); + auto ddiag = dx->extract_diagonal(); + + GKO_ASSERT_MTX_NEAR(diag, ddiag, 0); +} + + +TEST_F(Dense, ExtractDiagonalOnTallSkinnyIntoDenseCrossExecutor) +{ + set_up_apply_data(); + auto diag = Diagonal::create(ref, x->get_size()[1]); + // test make_temporary_clone + auto ddiag = Diagonal::create(ref, x->get_size()[1]); + + x->extract_diagonal(diag); + dx->extract_diagonal(ddiag); + + GKO_ASSERT_MTX_NEAR(diag, ddiag, 0); +} + + +TEST_F(Dense, ExtractDiagonalOnShortFatIsEquivalentToRef) +{ + set_up_apply_data(); + + auto diag = y->as_const_dense_view()->extract_diagonal(); + auto ddiag = dy->as_const_dense_view()->extract_diagonal(); + + GKO_ASSERT_MTX_NEAR(diag, ddiag, 0); +} + + +TEST_F(Dense, ExtractDiagonalOnShortFatIntoDenseCrossExecutor) +{ + set_up_apply_data(); + auto diag = Diagonal::create(ref, y->get_size()[0]); + // test make_temporary_clone + auto ddiag = Diagonal::create(ref, y->get_size()[0]); + + y->as_const_dense_view()->extract_diagonal(diag); + dy->as_const_dense_view()->extract_diagonal(ddiag); + + GKO_ASSERT_MTX_NEAR(diag, ddiag, 0); +} + + +TEST_F(Dense, AddsScaledDiagIsEquivalentToRef) +{ + auto mat = gen_mtx(532, 532); + gko::array diag_values(this->ref, 532); + gko::kernels::reference::components::fill_array( + this->ref, diag_values.get_data(), 532, Mtx::value_type{2.0}); + auto diag = gko::matrix::Diagonal::create(this->ref, 532, + diag_values); + auto alpha = gko::initialize({2.0}, this->ref); + auto dmat = gko::clone(this->exec, mat); + auto ddiag = gko::clone(this->exec, diag); + auto dalpha = gko::clone(this->exec, alpha); + + mat->add_scaled(alpha, diag); + dmat->add_scaled(dalpha, ddiag); + + GKO_ASSERT_MTX_NEAR(mat, dmat, r::value); +} + + +TEST_F(Dense, SubtractScaledDiagIsEquivalentToRef) +{ + auto mat = gen_mtx(532, 532); + gko::array diag_values(this->ref, 532); + gko::kernels::reference::components::fill_array( + this->ref, diag_values.get_data(), 532, Mtx::value_type{2.0}); + auto diag = gko::matrix::Diagonal::create(this->ref, 532, + diag_values); + auto alpha = gko::initialize({2.0}, this->ref); + auto dmat = gko::clone(this->exec, mat); + auto ddiag = gko::clone(this->exec, diag); + auto dalpha = gko::clone(this->exec, alpha); + + mat->sub_scaled(alpha, diag); + dmat->sub_scaled(dalpha, ddiag); + + GKO_ASSERT_MTX_NEAR(mat, dmat, r::value); +} + + +TEST_F(Dense, AddScaledIdentityToNonSquare) +{ + set_up_apply_data(); + + x->add_scaled_identity(alpha, beta); + dx->add_scaled_identity(dalpha, dbeta); + + GKO_ASSERT_MTX_NEAR(x, dx, r::value); +} + + +TEST_F(Dense, AddScaledIdentityToNonSquareOnDifferentExecutor) +{ + set_up_apply_data(); + + x->add_scaled_identity(alpha, beta); + dx->add_scaled_identity(alpha, beta); + + GKO_ASSERT_MTX_NEAR(x, dx, r::value); +} + + +TEST_F(Dense, CopyRespectsStride) +{ + set_up_vector_data(3); + auto stride = dx->get_size()[1] + 1; + auto result = Mtx::create(exec, dx->get_size(), stride); + value_type val = 1234567.0; + auto original_data = result->get_values(); + auto padding_ptr = original_data + dx->get_size()[1]; + exec->copy_from(ref, 1, &val, padding_ptr); + + dx->convert_to(result); + + GKO_ASSERT_MTX_NEAR(result, dx, 0); + ASSERT_EQ(result->get_stride(), stride); + ASSERT_EQ(exec->copy_val_to_host(padding_ptr), val); + ASSERT_EQ(result->get_values(), original_data); +} + + +TEST_F(Dense, FillIsEquivalentToRef) +{ + set_up_vector_data(3); + + x->fill(42); + dx->fill(42); + + GKO_ASSERT_MTX_NEAR(dx, x, 0); +} + + +TEST_F(Dense, StridedFillIsEquivalentToRef) +{ + using T = value_type; + auto x = gko::initialize>( + 4, {I{1.0, 2.0}, I{3.0, 4.0}, I{5.0, 6.0}}, ref); + auto dx = gko::initialize>( + 4, {I{1.0, 2.0}, I{3.0, 4.0}, I{5.0, 6.0}}, exec); + + x->fill(42); + dx->fill(42); + + GKO_ASSERT_MTX_NEAR(dx, x, 0); +} + + +TEST_F(Dense, ConvertToCooIsEquivalentToRef) +{ + set_up_apply_data(); + auto coo_mtx = gko::matrix::Coo::create(ref); + auto dcoo_mtx = gko::matrix::Coo::create(exec); + + x->convert_to(coo_mtx); + dx->convert_to(dcoo_mtx); + + ASSERT_EQ(dcoo_mtx->get_num_stored_elements(), + coo_mtx->get_num_stored_elements()); + GKO_ASSERT_MTX_NEAR(dcoo_mtx, coo_mtx, 0); +} + + +TEST_F(Dense, MoveToCooIsEquivalentToRef) +{ + set_up_apply_data(); + auto coo_mtx = gko::matrix::Coo::create(ref); + auto dcoo_mtx = gko::matrix::Coo::create(exec); + + x->move_to(coo_mtx); + dx->move_to(dcoo_mtx); + + ASSERT_EQ(dcoo_mtx->get_num_stored_elements(), + coo_mtx->get_num_stored_elements()); + GKO_ASSERT_MTX_NEAR(dcoo_mtx, coo_mtx, 0); +} + + +TEST_F(Dense, ConvertToCsrIsEquivalentToRef) +{ + set_up_apply_data(); + auto csr_mtx = gko::matrix::Csr::create(ref); + auto dcsr_mtx = gko::matrix::Csr::create(exec); + + x->convert_to(csr_mtx); + dx->convert_to(dcsr_mtx); + + GKO_ASSERT_MTX_NEAR(dcsr_mtx, csr_mtx, 0); +} + + +TEST_F(Dense, MoveToCsrIsEquivalentToRef) +{ + set_up_apply_data(); + auto csr_mtx = gko::matrix::Csr::create(ref); + auto dcsr_mtx = gko::matrix::Csr::create(exec); + + x->move_to(csr_mtx); + dx->move_to(dcsr_mtx); + + GKO_ASSERT_MTX_NEAR(dcsr_mtx, csr_mtx, 0); +} + + +TEST_F(Dense, ConvertToSparsityCsrIsEquivalentToRef) +{ + set_up_apply_data(); + auto sparsity_mtx = gko::matrix::SparsityCsr::create(ref); + auto d_sparsity_mtx = gko::matrix::SparsityCsr::create(exec); + + x->convert_to(sparsity_mtx); + dx->convert_to(d_sparsity_mtx); + + GKO_ASSERT_MTX_NEAR(d_sparsity_mtx, sparsity_mtx, 0); +} + + +TEST_F(Dense, MoveToSparsityCsrIsEquivalentToRef) +{ + set_up_apply_data(); + auto sparsity_mtx = gko::matrix::SparsityCsr::create(ref); + auto d_sparsity_mtx = gko::matrix::SparsityCsr::create(exec); + + x->move_to(sparsity_mtx); + dx->move_to(d_sparsity_mtx); + + GKO_ASSERT_MTX_NEAR(d_sparsity_mtx, sparsity_mtx, 0); +} + + +TEST_F(Dense, ConvertToEllIsEquivalentToRef) +{ + set_up_apply_data(); + auto ell_mtx = gko::matrix::Ell::create(ref); + auto dell_mtx = gko::matrix::Ell::create(exec); + + x->convert_to(ell_mtx); + dx->convert_to(dell_mtx); + + GKO_ASSERT_MTX_NEAR(dell_mtx, ell_mtx, 0); +} + + +TEST_F(Dense, MoveToEllIsEquivalentToRef) +{ + set_up_apply_data(); + auto ell_mtx = gko::matrix::Ell::create(ref); + auto dell_mtx = gko::matrix::Ell::create(exec); + + x->move_to(ell_mtx); + dx->move_to(dell_mtx); + + GKO_ASSERT_MTX_NEAR(dell_mtx, ell_mtx, 0); +} + + +TEST_F(Dense, ConvertToHybridIsEquivalentToRef) +{ + auto rmtx = gen_mtx(532, 231); + auto omtx = gko::clone(exec, rmtx); + auto srmtx = gko::matrix::Hybrid::create(ref); + auto somtx = gko::matrix::Hybrid::create(exec); + auto drmtx = Mtx::create(ref); + auto domtx = Mtx::create(exec); + + rmtx->convert_to(srmtx); + omtx->convert_to(somtx); + srmtx->convert_to(drmtx); + somtx->convert_to(domtx); + + GKO_ASSERT_MTX_NEAR(drmtx, domtx, 0); + GKO_ASSERT_MTX_NEAR(srmtx, somtx, 0); + GKO_ASSERT_MTX_NEAR(domtx, omtx, 0); +} + + +TEST_F(Dense, MoveToHybridIsEquivalentToRef) +{ + auto rmtx = gen_mtx(532, 231); + auto omtx = gko::clone(exec, rmtx); + auto srmtx = gko::matrix::Hybrid::create(ref); + auto somtx = gko::matrix::Hybrid::create(exec); + auto drmtx = Mtx::create(ref); + auto domtx = Mtx::create(exec); + + rmtx->move_to(srmtx); + omtx->move_to(somtx); + srmtx->move_to(drmtx); + somtx->move_to(domtx); + + GKO_ASSERT_MTX_NEAR(drmtx, domtx, 0); + GKO_ASSERT_MTX_NEAR(srmtx, somtx, 0); + GKO_ASSERT_MTX_NEAR(domtx, omtx, 0); +} + + +TEST_F(Dense, ConvertToSellpIsEquivalentToRef) +{ + set_up_apply_data(); + auto sellp_mtx = gko::matrix::Sellp::create(ref); + auto dsellp_mtx = gko::matrix::Sellp::create(exec); + + x->convert_to(sellp_mtx); + dx->convert_to(dsellp_mtx); + + GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 0); +} + + +TEST_F(Dense, MoveToSellpIsEquivalentToRef) +{ + set_up_apply_data(); + auto sellp_mtx = gko::matrix::Sellp::create(ref); + auto dsellp_mtx = gko::matrix::Sellp::create(exec); + + x->move_to(sellp_mtx); + dx->move_to(dsellp_mtx); + + GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 0); +} + + +TEST_F(Dense, ConvertsEmptyToSellp) +{ + auto dempty_mtx = Mtx::create(exec); + auto dsellp_mtx = gko::matrix::Sellp::create(exec); + + dempty_mtx->convert_to(dsellp_mtx); + + ASSERT_EQ(exec->copy_val_to_host(dsellp_mtx->get_const_slice_sets()), 0); + ASSERT_FALSE(dsellp_mtx->get_size()); +} + + +TEST_F(Dense, CalculateNNZPerRowIsEquivalentToRef) +{ + set_up_apply_data(); + gko::array nnz_per_row(ref); + nnz_per_row.resize_and_reset(x->get_size()[0]); + gko::array dnnz_per_row(exec); + dnnz_per_row.resize_and_reset(dx->get_size()[0]); + + gko::kernels::reference::dense::count_nonzeros_per_row( + ref, x->get_const_device_view(), nnz_per_row.get_data()); + gko::kernels::GKO_DEVICE_NAMESPACE::dense::count_nonzeros_per_row( + exec, dx->get_const_device_view(), dnnz_per_row.get_data()); + + auto tmp = gko::array(ref, dnnz_per_row); + for (gko::size_type i = 0; i < nnz_per_row.get_size(); i++) { + ASSERT_EQ(nnz_per_row.get_const_data()[i], tmp.get_const_data()[i]); + } +} + + +TEST_F(Dense, ComputeMaxNNZPerRowIsEquivalentToRef) +{ + set_up_apply_data(); + gko::size_type max_nnz; + gko::size_type dmax_nnz; + + gko::kernels::reference::dense::compute_max_nnz_per_row( + ref, x->get_const_device_view(), max_nnz); + gko::kernels::GKO_DEVICE_NAMESPACE::dense::compute_max_nnz_per_row( + exec, dx->get_const_device_view(), dmax_nnz); + + ASSERT_EQ(max_nnz, dmax_nnz); +} diff --git a/test/matrix/ell_kernels.cpp b/test/matrix/ell_kernels.cpp index 76b0d1ff33e..067e39d2201 100644 --- a/test/matrix/ell_kernels.cpp +++ b/test/matrix/ell_kernels.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -45,13 +46,13 @@ class Ell : public CommonTestFixture { { mtx = Mtx::create(ref, gko::dim<2>{}, num_stored_elements_per_row, stride); - mtx->move_from(gen_mtx(num_rows, num_cols)); + mtx->move_from(gen_mtx(num_rows, num_cols)); expected = gen_mtx(num_rows, num_vectors); expected2 = Vec2::create(ref); - expected2->copy_from(expected); + expected->convert_to(expected2.get()); y = gen_mtx(num_cols, num_vectors); y2 = Vec2::create(ref); - y2->copy_from(y); + y->convert_to(y2); alpha = gko::initialize({2.0}, ref); alpha2 = gko::initialize({2.0}, ref); beta = gko::initialize({-1.0}, ref); @@ -493,12 +494,12 @@ TEST_F(Ell, AdvancedApplyToComplexIsEquivalentToRef) } -TEST_F(Ell, ConvertToMultiVectorIsEquivalentToRef) +TEST_F(Ell, ConvertToDenseIsEquivalentToRef) { set_up_apply_data(); - auto dense_mtx = gko::matrix::MultiVector::create(ref); - auto ddense_mtx = gko::matrix::MultiVector::create(exec); + auto dense_mtx = gko::matrix::Dense::create(ref); + auto ddense_mtx = gko::matrix::Dense::create(exec); mtx->convert_to(dense_mtx); dmtx->convert_to(ddense_mtx); diff --git a/test/matrix/fft_kernels.cpp b/test/matrix/fft_kernels.cpp index b78961ec378..24e4887734c 100644 --- a/test/matrix/fft_kernels.cpp +++ b/test/matrix/fft_kernels.cpp @@ -40,8 +40,8 @@ class Fft : public CommonTestFixture { std::normal_distribution<>(-1.0, 1.0), rand_engine, ref); ddata = Vec::create(exec); ddata->copy_from(this->data); - data_strided = data->create_submatrix({0, n}, {0, subcols}); - ddata_strided = ddata->create_submatrix({0, n}, {0, subcols}); + data_strided = data->create_subview({0, n}, {0, subcols}); + ddata_strided = ddata->create_subview({0, n}, {0, subcols}); out = data->clone(); dout = data->clone(); out_strided = Vec::create(ref, data_strided->get_size(), out_stride); diff --git a/test/matrix/hybrid_kernels.cpp b/test/matrix/hybrid_kernels.cpp index c9628240b0a..fa7e7bd754b 100644 --- a/test/matrix/hybrid_kernels.cpp +++ b/test/matrix/hybrid_kernels.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,7 @@ class Hybrid : public CommonTestFixture { protected: using Mtx = gko::matrix::Hybrid; + using Dense = gko::matrix::Dense; using Vec = gko::matrix::MultiVector; using ComplexVec = gko::matrix::MultiVector>; @@ -49,7 +51,7 @@ class Hybrid : public CommonTestFixture { std::make_shared()) { mtx = Mtx::create(ref, strategy); - mtx->move_from(gen_mtx(532, 231, 1)); + mtx->move_from(gen_mtx(532, 231, 1)); expected = gen_mtx(532, num_vectors, 1); y = gen_mtx(231, num_vectors, 1); alpha = gko::initialize({2.0}, ref); @@ -169,7 +171,7 @@ TEST_F(Hybrid, ConvertEmptyCooToCsrIsEquivalentToRef) { auto balanced_mtx = Mtx::create(ref, std::make_shared(4)); - balanced_mtx->move_from(gen_mtx(400, 200, 4, 4)); + balanced_mtx->move_from(gen_mtx(400, 200, 4, 4)); auto dbalanced_mtx = Mtx::create(exec, std::make_shared(4)); dbalanced_mtx->copy_from(balanced_mtx); @@ -186,7 +188,7 @@ TEST_F(Hybrid, ConvertEmptyCooToCsrIsEquivalentToRef) TEST_F(Hybrid, ConvertWithEmptyFirstAndLastRowToCsrIsEquivalentToRef) { // create a dense matrix for easier manipulation - auto dense_mtx = gen_mtx(400, 200, 0, 4); + auto dense_mtx = gen_mtx(400, 200, 0, 4); // set first and last row to zero for (gko::size_type col = 0; col < dense_mtx->get_size()[1]; col++) { dense_mtx->at(0, col) = gko::zero(); diff --git a/test/matrix/matrix.cpp b/test/matrix/matrix.cpp index 9f6f7a41574..10d3ce28960 100644 --- a/test/matrix/matrix.cpp +++ b/test/matrix/matrix.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -41,7 +42,7 @@ struct SimpleMatrixTest { static std::unique_ptr create( std::shared_ptr exec, gko::dim<2> size) { - return matrix_type::create(exec->get_master(), size); + return matrix_type::create(exec, size); } static void modify_data( @@ -59,8 +60,8 @@ struct SimpleMatrixTest { } }; -struct MultiVectorWithDefaultStride - : SimpleMatrixTest> { +struct DenseWithDefaultStride + : SimpleMatrixTest> { static bool preserves_zeros() { return false; } static void assert_empty_state(gko::ptr_param mtx) @@ -72,7 +73,7 @@ struct MultiVectorWithDefaultStride } }; -struct MultiVectorWithCustomStride : MultiVectorWithDefaultStride { +struct DenseWithCustomStride : DenseWithDefaultStride { static std::unique_ptr create( std::shared_ptr exec, gko::dim<2> size) { @@ -745,21 +746,21 @@ class Matrix : public CommonTestFixture { // create slightly bigger vectors auto in_padded = gen_in_vec(mtx, in_stride); auto out_padded = gen_out_vec(mtx, out_stride); - const auto in_rows = gko::span(0, mtx.ref->get_size()[1]); - const auto out_rows = gko::span(0, mtx.ref->get_size()[0]); - const auto cols = gko::span(0, rhs); - const auto out_pad_cols = gko::span(rhs, out_stride); + const auto in_rows = gko::local_span(0, mtx.ref->get_size()[1]); + const auto out_rows = gko::local_span(0, mtx.ref->get_size()[0]); + const auto cols = gko::local_span(0, rhs); + const auto out_pad_cols = gko::local_span(rhs, out_stride); // create views of the padding and in/out vectors auto out_padding = test_pair{ - out_padded.ref->create_submatrix(out_rows, out_pad_cols), - out_padded.dev->create_submatrix(out_rows, out_pad_cols)}; + out_padded.ref->create_subview(out_rows, out_pad_cols), + out_padded.dev->create_subview(out_rows, out_pad_cols)}; auto orig_padding = out_padding.ref->clone(); auto in = - test_pair{in_padded.ref->create_submatrix(in_rows, cols), - in_padded.dev->create_submatrix(in_rows, cols)}; + test_pair{in_padded.ref->create_subview(in_rows, cols), + in_padded.dev->create_subview(in_rows, cols)}; auto out = test_pair{ - out_padded.ref->create_submatrix(out_rows, cols), - out_padded.dev->create_submatrix(out_rows, cols)}; + out_padded.ref->create_subview(out_rows, cols), + out_padded.dev->create_subview(out_rows, cols)}; fn(std::move(in), std::move(out)); // check that padding was unmodified GKO_ASSERT_MTX_NEAR(out_padding.ref, orig_padding, 0.0); @@ -831,8 +832,7 @@ class Matrix : public CommonTestFixture { }; using MatrixTypes = ::testing::Types< - MultiVectorWithDefaultStride, MultiVectorWithCustomStride, Coo, - CsrWithDefaultStrategy, + DenseWithDefaultStride, DenseWithCustomStride, Coo, CsrWithDefaultStrategy, #if defined(GKO_COMPILING_CUDA) || defined(GKO_COMPILING_HIP) || \ defined(GKO_COMPILING_DPCPP) || defined(GKO_COMPILING_OMP) CsrWithClassicalStrategy, CsrWithMergePathStrategy, @@ -1098,25 +1098,25 @@ TYPED_TEST(Matrix, MoveFromCsrIsEquivalentToRef) } -TYPED_TEST(Matrix, ConvertToMultiVectorIsEquivalentToRef) +TYPED_TEST(Matrix, ConvertToDenseIsEquivalentToRef) { using Mtx = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; this->forall_matrix_scenarios([&](auto mtx) { const auto size = mtx.ref->get_size(); const auto stride = size[1] + 5; const auto padded_size = gko::dim<2>{size[0], stride}; - auto ref_padded = MultiVector::create(this->ref, padded_size); - auto dev_padded = MultiVector::create(this->exec, padded_size); + auto ref_padded = Dense::create(this->ref, padded_size); + auto dev_padded = Dense::create(this->exec, padded_size); ref_padded->fill(12345); dev_padded->fill(12345); - const auto rows = gko::span{0, size[0]}; - const auto cols = gko::span{0, size[1]}; - const auto pad_cols = gko::span{size[1], stride}; - auto ref_result = ref_padded->create_submatrix(rows, cols); - auto dev_result = dev_padded->create_submatrix(rows, cols); - auto ref_padding = ref_padded->create_submatrix(rows, pad_cols); - auto dev_padding = dev_padded->create_submatrix(rows, pad_cols); + const auto rows = gko::local_span{0, size[0]}; + const auto cols = gko::local_span{0, size[1]}; + const auto pad_cols = gko::local_span{size[1], stride}; + auto ref_result = ref_padded->create_subview(rows, cols); + auto dev_result = dev_padded->create_subview(rows, cols); + auto ref_padding = ref_padded->create_subview(rows, pad_cols); + auto dev_padding = dev_padded->create_subview(rows, pad_cols); auto orig_padding = ref_padding->clone(); mtx.ref->convert_to(ref_result); @@ -1131,13 +1131,13 @@ TYPED_TEST(Matrix, ConvertToMultiVectorIsEquivalentToRef) } -TYPED_TEST(Matrix, MoveToMultiVectorIsEquivalentToRef) +TYPED_TEST(Matrix, MoveToDenseIsEquivalentToRef) { using Mtx = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; this->forall_matrix_scenarios([&](auto mtx) { - auto ref_result = MultiVector::create(this->ref); - auto dev_result = MultiVector::create(this->exec); + auto ref_result = Dense::create(this->ref); + auto dev_result = Dense::create(this->exec); mtx.ref->move_to(ref_result); mtx.dev->move_to(dev_result); @@ -1147,15 +1147,15 @@ TYPED_TEST(Matrix, MoveToMultiVectorIsEquivalentToRef) } -TYPED_TEST(Matrix, ConvertFromMultiVectorIsEquivalentToRef) +TYPED_TEST(Matrix, ConvertFromDenseIsEquivalentToRef) { using TestConfig = typename TestFixture::Config; using Mtx = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; this->forall_matrix_data_scenarios([&](auto data) { const auto stride = data.size[1] + 2; - auto ref_src = MultiVector::create(this->ref, data.size, stride); - auto dev_src = MultiVector::create(this->exec, data.size, stride); + auto ref_src = Dense::create(this->ref, data.size, stride); + auto dev_src = Dense::create(this->exec, data.size, stride); ref_src->read(data); dev_src->read(data); ASSERT_EQ(ref_src->get_stride(), stride); @@ -1172,15 +1172,15 @@ TYPED_TEST(Matrix, ConvertFromMultiVectorIsEquivalentToRef) } -TYPED_TEST(Matrix, MoveFromMultiVectorIsEquivalentToRef) +TYPED_TEST(Matrix, MoveFromDenseIsEquivalentToRef) { using TestConfig = typename TestFixture::Config; using Mtx = typename TestFixture::Mtx; - using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; this->forall_matrix_data_scenarios([&](auto data) { const auto stride = data.size[1] + 2; - auto ref_src = MultiVector::create(this->ref, data.size, stride); - auto dev_src = MultiVector::create(this->exec, data.size, stride); + auto ref_src = Dense::create(this->ref, data.size, stride); + auto dev_src = Dense::create(this->exec, data.size, stride); ref_src->read(data); dev_src->read(data); ASSERT_EQ(ref_src->get_stride(), stride); diff --git a/test/matrix/multivector_kernels.cpp b/test/matrix/multivector_kernels.cpp index c5df909b421..97a5332e3f1 100644 --- a/test/matrix/multivector_kernels.cpp +++ b/test/matrix/multivector_kernels.cpp @@ -13,16 +13,11 @@ #include #include -#include -#include +#include #include -#include -#include #include #include #include -#include -#include #include "core/components/fill_array_kernels.hpp" #include "core/test/utils.hpp" @@ -40,7 +35,6 @@ class MultiVector : public CommonTestFixture { using Arr = gko::array; using ComplexMtx = gko::matrix::MultiVector>; using Diagonal = gko::matrix::Diagonal; - using MixedComplexMtx = gko::matrix::MultiVector>; using Permutation = gko::matrix::Permutation; using ScaledPermutation = gko::matrix::ScaledPermutation; @@ -404,227 +398,6 @@ TEST_F(MultiVector, ComputeConjDotComplexIsEquivalentToRef) } -TEST_F(MultiVector, ConvertToCooIsEquivalentToRef) -{ - set_up_apply_data(); - auto coo_mtx = gko::matrix::Coo::create(ref); - auto dcoo_mtx = gko::matrix::Coo::create(exec); - - x->convert_to(coo_mtx); - dx->convert_to(dcoo_mtx); - - ASSERT_EQ(dcoo_mtx->get_num_stored_elements(), - coo_mtx->get_num_stored_elements()); - GKO_ASSERT_MTX_NEAR(dcoo_mtx, coo_mtx, 0); -} - - -TEST_F(MultiVector, MoveToCooIsEquivalentToRef) -{ - set_up_apply_data(); - auto coo_mtx = gko::matrix::Coo::create(ref); - auto dcoo_mtx = gko::matrix::Coo::create(exec); - - x->move_to(coo_mtx); - dx->move_to(dcoo_mtx); - - ASSERT_EQ(dcoo_mtx->get_num_stored_elements(), - coo_mtx->get_num_stored_elements()); - GKO_ASSERT_MTX_NEAR(dcoo_mtx, coo_mtx, 0); -} - - -TEST_F(MultiVector, ConvertToCsrIsEquivalentToRef) -{ - set_up_apply_data(); - auto csr_mtx = gko::matrix::Csr::create(ref); - auto dcsr_mtx = gko::matrix::Csr::create(exec); - - x->convert_to(csr_mtx); - dx->convert_to(dcsr_mtx); - - GKO_ASSERT_MTX_NEAR(dcsr_mtx, csr_mtx, 0); -} - - -TEST_F(MultiVector, MoveToCsrIsEquivalentToRef) -{ - set_up_apply_data(); - auto csr_mtx = gko::matrix::Csr::create(ref); - auto dcsr_mtx = gko::matrix::Csr::create(exec); - - x->move_to(csr_mtx); - dx->move_to(dcsr_mtx); - - GKO_ASSERT_MTX_NEAR(dcsr_mtx, csr_mtx, 0); -} - - -TEST_F(MultiVector, ConvertToSparsityCsrIsEquivalentToRef) -{ - set_up_apply_data(); - auto sparsity_mtx = gko::matrix::SparsityCsr::create(ref); - auto d_sparsity_mtx = gko::matrix::SparsityCsr::create(exec); - - x->convert_to(sparsity_mtx); - dx->convert_to(d_sparsity_mtx); - - GKO_ASSERT_MTX_NEAR(d_sparsity_mtx, sparsity_mtx, 0); -} - - -TEST_F(MultiVector, MoveToSparsityCsrIsEquivalentToRef) -{ - set_up_apply_data(); - auto sparsity_mtx = gko::matrix::SparsityCsr::create(ref); - auto d_sparsity_mtx = gko::matrix::SparsityCsr::create(exec); - - x->move_to(sparsity_mtx); - dx->move_to(d_sparsity_mtx); - - GKO_ASSERT_MTX_NEAR(d_sparsity_mtx, sparsity_mtx, 0); -} - - -TEST_F(MultiVector, ConvertToEllIsEquivalentToRef) -{ - set_up_apply_data(); - auto ell_mtx = gko::matrix::Ell::create(ref); - auto dell_mtx = gko::matrix::Ell::create(exec); - - x->convert_to(ell_mtx); - dx->convert_to(dell_mtx); - - GKO_ASSERT_MTX_NEAR(dell_mtx, ell_mtx, 0); -} - - -TEST_F(MultiVector, MoveToEllIsEquivalentToRef) -{ - set_up_apply_data(); - auto ell_mtx = gko::matrix::Ell::create(ref); - auto dell_mtx = gko::matrix::Ell::create(exec); - - x->move_to(ell_mtx); - dx->move_to(dell_mtx); - - GKO_ASSERT_MTX_NEAR(dell_mtx, ell_mtx, 0); -} - - -TEST_F(MultiVector, ConvertToHybridIsEquivalentToRef) -{ - auto rmtx = gen_mtx(532, 231); - auto omtx = gko::clone(exec, rmtx); - auto srmtx = gko::matrix::Hybrid::create(ref); - auto somtx = gko::matrix::Hybrid::create(exec); - auto drmtx = Mtx::create(ref); - auto domtx = Mtx::create(exec); - - rmtx->convert_to(srmtx); - omtx->convert_to(somtx); - srmtx->convert_to(drmtx); - somtx->convert_to(domtx); - - GKO_ASSERT_MTX_NEAR(drmtx, domtx, 0); - GKO_ASSERT_MTX_NEAR(srmtx, somtx, 0); - GKO_ASSERT_MTX_NEAR(domtx, omtx, 0); -} - - -TEST_F(MultiVector, MoveToHybridIsEquivalentToRef) -{ - auto rmtx = gen_mtx(532, 231); - auto omtx = gko::clone(exec, rmtx); - auto srmtx = gko::matrix::Hybrid::create(ref); - auto somtx = gko::matrix::Hybrid::create(exec); - auto drmtx = Mtx::create(ref); - auto domtx = Mtx::create(exec); - - rmtx->move_to(srmtx); - omtx->move_to(somtx); - srmtx->move_to(drmtx); - somtx->move_to(domtx); - - GKO_ASSERT_MTX_NEAR(drmtx, domtx, 0); - GKO_ASSERT_MTX_NEAR(srmtx, somtx, 0); - GKO_ASSERT_MTX_NEAR(domtx, omtx, 0); -} - - -TEST_F(MultiVector, ConvertToSellpIsEquivalentToRef) -{ - set_up_apply_data(); - auto sellp_mtx = gko::matrix::Sellp::create(ref); - auto dsellp_mtx = gko::matrix::Sellp::create(exec); - - x->convert_to(sellp_mtx); - dx->convert_to(dsellp_mtx); - - GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 0); -} - - -TEST_F(MultiVector, MoveToSellpIsEquivalentToRef) -{ - set_up_apply_data(); - auto sellp_mtx = gko::matrix::Sellp::create(ref); - auto dsellp_mtx = gko::matrix::Sellp::create(exec); - - x->move_to(sellp_mtx); - dx->move_to(dsellp_mtx); - - GKO_ASSERT_MTX_NEAR(sellp_mtx, dsellp_mtx, 0); -} - - -TEST_F(MultiVector, ConvertsEmptyToSellp) -{ - auto dempty_mtx = Mtx::create(exec); - auto dsellp_mtx = gko::matrix::Sellp::create(exec); - - dempty_mtx->convert_to(dsellp_mtx); - - ASSERT_EQ(exec->copy_val_to_host(dsellp_mtx->get_const_slice_sets()), 0); - ASSERT_FALSE(dsellp_mtx->get_size()); -} - - -TEST_F(MultiVector, CalculateNNZPerRowIsEquivalentToRef) -{ - set_up_apply_data(); - gko::array nnz_per_row(ref); - nnz_per_row.resize_and_reset(x->get_size()[0]); - gko::array dnnz_per_row(exec); - dnnz_per_row.resize_and_reset(dx->get_size()[0]); - - gko::kernels::reference::multivector::count_nonzeros_per_row( - ref, x->get_const_device_view(), nnz_per_row.get_data()); - gko::kernels::GKO_DEVICE_NAMESPACE::multivector::count_nonzeros_per_row( - exec, dx->get_const_device_view(), dnnz_per_row.get_data()); - - auto tmp = gko::array(ref, dnnz_per_row); - for (gko::size_type i = 0; i < nnz_per_row.get_size(); i++) { - ASSERT_EQ(nnz_per_row.get_const_data()[i], tmp.get_const_data()[i]); - } -} - - -TEST_F(MultiVector, ComputeMaxNNZPerRowIsEquivalentToRef) -{ - set_up_apply_data(); - gko::size_type max_nnz; - gko::size_type dmax_nnz; - - gko::kernels::reference::multivector::compute_max_nnz_per_row( - ref, x->get_const_device_view(), max_nnz); - gko::kernels::GKO_DEVICE_NAMESPACE::multivector::compute_max_nnz_per_row( - exec, dx->get_const_device_view(), dmax_nnz); - - ASSERT_EQ(max_nnz, dmax_nnz); -} - - TEST_F(MultiVector, IsTransposable) { set_up_apply_data(); @@ -1140,46 +913,6 @@ TEST_F( } -TEST_F(MultiVector, AddsScaledDiagIsEquivalentToRef) -{ - auto mat = gen_mtx(532, 532); - gko::array diag_values(this->ref, 532); - gko::kernels::reference::components::fill_array( - this->ref, diag_values.get_data(), 532, Mtx::value_type{2.0}); - auto diag = gko::matrix::Diagonal::create(this->ref, 532, - diag_values); - auto alpha = gko::initialize({2.0}, this->ref); - auto dmat = gko::clone(this->exec, mat); - auto ddiag = gko::clone(this->exec, diag); - auto dalpha = gko::clone(this->exec, alpha); - - mat->add_scaled(alpha, diag); - dmat->add_scaled(dalpha, ddiag); - - GKO_ASSERT_MTX_NEAR(mat, dmat, r::value); -} - - -TEST_F(MultiVector, SubtractScaledDiagIsEquivalentToRef) -{ - auto mat = gen_mtx(532, 532); - gko::array diag_values(this->ref, 532); - gko::kernels::reference::components::fill_array( - this->ref, diag_values.get_data(), 532, Mtx::value_type{2.0}); - auto diag = gko::matrix::Diagonal::create(this->ref, 532, - diag_values); - auto alpha = gko::initialize({2.0}, this->ref); - auto dmat = gko::clone(this->exec, mat); - auto ddiag = gko::clone(this->exec, diag); - auto dalpha = gko::clone(this->exec, alpha); - - mat->sub_scaled(alpha, diag); - dmat->sub_scaled(dalpha, ddiag); - - GKO_ASSERT_MTX_NEAR(mat, dmat, r::value); -} - - TEST_F(MultiVector, CanGatherRows) { set_up_apply_data(); @@ -1617,56 +1350,6 @@ TEST_F(MultiVector, IsInverseColPermutableIntoMultiVectorCrossExecutor) } -TEST_F(MultiVector, ExtractDiagonalOnTallSkinnyIsEquivalentToRef) -{ - set_up_apply_data(); - - auto diag = x->extract_diagonal(); - auto ddiag = dx->extract_diagonal(); - - GKO_ASSERT_MTX_NEAR(diag, ddiag, 0); -} - - -TEST_F(MultiVector, ExtractDiagonalOnTallSkinnyIntoMultiVectorCrossExecutor) -{ - set_up_apply_data(); - auto diag = Diagonal::create(ref, x->get_size()[1]); - // test make_temporary_clone - auto ddiag = Diagonal::create(ref, x->get_size()[1]); - - x->extract_diagonal(diag); - dx->extract_diagonal(ddiag); - - GKO_ASSERT_MTX_NEAR(diag, ddiag, 0); -} - - -TEST_F(MultiVector, ExtractDiagonalOnShortFatIsEquivalentToRef) -{ - set_up_apply_data(); - - auto diag = y->extract_diagonal(); - auto ddiag = dy->extract_diagonal(); - - GKO_ASSERT_MTX_NEAR(diag, ddiag, 0); -} - - -TEST_F(MultiVector, ExtractDiagonalOnShortFatIntoMultiVectorCrossExecutor) -{ - set_up_apply_data(); - auto diag = Diagonal::create(ref, y->get_size()[0]); - // test make_temporary_clone - auto ddiag = Diagonal::create(ref, y->get_size()[0]); - - y->extract_diagonal(diag); - dy->extract_diagonal(ddiag); - - GKO_ASSERT_MTX_NEAR(diag, ddiag, 0); -} - - TEST_F(MultiVector, ComputeDotIsEquivalentToRef) { set_up_vector_data(1); @@ -1979,28 +1662,6 @@ TEST_F(MultiVector, GetImagIntoMultiVectorCrossExecutor) } -TEST_F(MultiVector, AddScaledIdentityToNonSquare) -{ - set_up_apply_data(); - - x->add_scaled_identity(alpha, beta); - dx->add_scaled_identity(dalpha, dbeta); - - GKO_ASSERT_MTX_NEAR(x, dx, r::value); -} - - -TEST_F(MultiVector, AddScaledIdentityToNonSquareOnDifferentExecutor) -{ - set_up_apply_data(); - - x->add_scaled_identity(alpha, beta); - dx->add_scaled_identity(alpha, beta); - - GKO_ASSERT_MTX_NEAR(x, dx, r::value); -} - - TEST_F(MultiVector, ComputeNorm2SquaredIsEquivalentToRef) { set_up_apply_data(); diff --git a/test/matrix/permutation_kernels.cpp b/test/matrix/permutation_kernels.cpp index 248ff1f20dd..e0d8903d212 100644 --- a/test/matrix/permutation_kernels.cpp +++ b/test/matrix/permutation_kernels.cpp @@ -7,6 +7,7 @@ #include +#include #include #include diff --git a/test/matrix/scaled_permutation_kernels.cpp b/test/matrix/scaled_permutation_kernels.cpp index dbec451e2ae..405951dcce5 100644 --- a/test/matrix/scaled_permutation_kernels.cpp +++ b/test/matrix/scaled_permutation_kernels.cpp @@ -7,6 +7,7 @@ #include +#include #include #include "core/test/utils.hpp" diff --git a/test/matrix/sellp_kernels.cpp b/test/matrix/sellp_kernels.cpp index 438f6eb21a8..84a4d183733 100644 --- a/test/matrix/sellp_kernels.cpp +++ b/test/matrix/sellp_kernels.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -191,11 +192,11 @@ TEST_F(Sellp, AdvancedApplyToComplexIsEquivalentToRef) } -TEST_F(Sellp, ConvertToMultiVectorIsEquivalentToRef) +TEST_F(Sellp, ConvertToDenseIsEquivalentToRef) { set_up_apply_matrix(64); - auto dense_mtx = gko::matrix::MultiVector::create(ref); - auto ddense_mtx = gko::matrix::MultiVector::create(exec); + auto dense_mtx = gko::matrix::Dense::create(ref); + auto ddense_mtx = gko::matrix::Dense::create(exec); mtx->convert_to(dense_mtx); dmtx->convert_to(ddense_mtx); @@ -217,11 +218,11 @@ TEST_F(Sellp, ConvertToCsrIsEquivalentToRef) } -TEST_F(Sellp, ConvertEmptyToMultiVectorIsEquivalentToRef) +TEST_F(Sellp, ConvertEmptyToDenseIsEquivalentToRef) { set_up_apply_matrix(64); - auto dense_mtx = gko::matrix::MultiVector::create(ref); - auto ddense_mtx = gko::matrix::MultiVector::create(exec); + auto dense_mtx = gko::matrix::Dense::create(ref); + auto ddense_mtx = gko::matrix::Dense::create(exec); empty->convert_to(dense_mtx); dempty->convert_to(ddense_mtx); diff --git a/test/matrix/sparsity_csr_kernels.cpp b/test/matrix/sparsity_csr_kernels.cpp index ba517856279..26bf0b127ae 100644 --- a/test/matrix/sparsity_csr_kernels.cpp +++ b/test/matrix/sparsity_csr_kernels.cpp @@ -101,11 +101,11 @@ TEST_F(SparsityCsr, ToAdjacencyMatrixIsEquivalentToRef) } -TEST_F(SparsityCsr, ConvertToMultiVectorIsEquivalentToRef) +TEST_F(SparsityCsr, ConvertToDenseIsEquivalentToRef) { - const auto out_dense = gko::matrix::MultiVector::create( + const auto out_dense = gko::matrix::Dense::create( exec, mtx->get_size(), mtx->get_size()[1] + 2); - const auto dout_dense = gko::matrix::MultiVector::create( + const auto dout_dense = gko::matrix::Dense::create( exec, mtx->get_size(), mtx->get_size()[1] + 2); mtx->convert_to(out_dense); diff --git a/test/mpi/distributed/matrix.cpp b/test/mpi/distributed/matrix.cpp index 9232be272d7..67db5da9622 100644 --- a/test/mpi/distributed/matrix.cpp +++ b/test/mpi/distributed/matrix.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include "core/test/utils.hpp" #include "test/utils/mpi/common_fixture.hpp" diff --git a/test/mpi/distributed/row_gatherer.cpp b/test/mpi/distributed/row_gatherer.cpp index 91a8e20155f..82aec2ff0ae 100644 --- a/test/mpi/distributed/row_gatherer.cpp +++ b/test/mpi/distributed/row_gatherer.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include "core/test/utils.hpp" #include "test/utils/mpi/common_fixture.hpp" diff --git a/test/mpi/preconditioner/schwarz.cpp b/test/mpi/preconditioner/schwarz.cpp index 40033ef58e1..0f7948bb60f 100644 --- a/test/mpi/preconditioner/schwarz.cpp +++ b/test/mpi/preconditioner/schwarz.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include diff --git a/test/mpi/solver/solver.cpp b/test/mpi/solver/solver.cpp index 2cac0e394f5..0f2ebeb6394 100644 --- a/test/mpi/solver/solver.cpp +++ b/test/mpi/solver/solver.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -51,7 +52,7 @@ struct SimpleSolverTest { using global_index_type = gko::int64; using dist_matrix_type = gko::experimental::distributed::Matrix; + global_index_type>; using non_dist_matrix_type = gko::matrix::Csr; using dist_vector_type = gko::experimental::distributed::Vector; diff --git a/test/multigrid/fixed_coarsening_kernels.cpp b/test/multigrid/fixed_coarsening_kernels.cpp index b76555d9cef..a2610c1f6aa 100644 --- a/test/multigrid/fixed_coarsening_kernels.cpp +++ b/test/multigrid/fixed_coarsening_kernels.cpp @@ -31,7 +31,7 @@ class FixedCoarsening : public CommonTestFixture { protected: - using Mtx = gko::matrix::MultiVector; + using Mtx = gko::matrix::Dense; using Csr = gko::matrix::Csr; FixedCoarsening() : rand_engine(30), m{597} {} diff --git a/test/preconditioner/isai_kernels.cpp b/test/preconditioner/isai_kernels.cpp index a90a9f4706f..cf01dd2279f 100644 --- a/test/preconditioner/isai_kernels.cpp +++ b/test/preconditioner/isai_kernels.cpp @@ -28,6 +28,7 @@ class Isai : public CommonTestFixture { protected: using Csr = gko::matrix::Csr; using MultiVector = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; Isai() : rand_engine(42) {} std::unique_ptr clone_allocations(const Csr* csr_mtx) @@ -53,19 +54,18 @@ class Isai : public CommonTestFixture { auto val_dist = std::uniform_real_distribution(-1., 1.); mtx = Csr::create(ref); if (type == matrix_type::general) { - auto dense_mtx = gko::test::generate_random_matrix( + auto dense_mtx = gko::test::generate_random_matrix( n, n, nz_dist, val_dist, rand_engine, ref, gko::dim<2>{n, n}); ensure_diagonal(dense_mtx.get()); mtx->copy_from(dense_mtx); } else if (type == matrix_type::spd) { - auto dense_mtx = - gko::test::generate_random_band_matrix( - n, row_limit / 4, row_limit / 4, val_dist, rand_engine, ref, - gko::dim<2>{n, n}); - auto transp = gko::as(dense_mtx->transpose()); + auto dense_mtx = gko::test::generate_random_band_matrix( + n, row_limit / 4, row_limit / 4, val_dist, rand_engine, ref, + gko::dim<2>{n, n}); + auto transp = gko::as(dense_mtx->transpose()); auto spd_mtx = MultiVector::create(ref, gko::dim<2>{n, n}); - dense_mtx->apply(transp, spd_mtx); - mtx->copy_from(spd_mtx); + dense_mtx->apply(transp->as_const_multivector_view(), spd_mtx); + spd_mtx->as_const_dense_view()->convert_to(mtx); } else { mtx = gko::test::generate_random_triangular_matrix( n, true, for_lower_tm, nz_dist, val_dist, rand_engine, ref, @@ -81,7 +81,7 @@ class Isai : public CommonTestFixture { { auto val_dist = std::uniform_real_distribution(0., 1.); mtx = Csr::create(ref); - auto dense_mtx = gko::test::generate_random_band_matrix( + auto dense_mtx = gko::test::generate_random_band_matrix( n, 1, 1, val_dist, rand_engine, ref); ensure_diagonal(dense_mtx.get()); mtx->copy_from(dense_mtx); @@ -91,10 +91,10 @@ class Isai : public CommonTestFixture { d_inverse = gko::clone(exec, inverse); } - void ensure_diagonal(MultiVector* mtx) + void ensure_diagonal(Dense* mtx) { for (int i = 0; i < mtx->get_size()[0]; ++i) { - mtx->at(i, i) = gko::one(); + mtx->at(i, i) = gko::one(); } } diff --git a/test/preconditioner/jacobi_kernels.cpp b/test/preconditioner/jacobi_kernels.cpp index 2e7b8f892fa..5027e6f1751 100644 --- a/test/preconditioner/jacobi_kernels.cpp +++ b/test/preconditioner/jacobi_kernels.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -21,6 +22,7 @@ class Jacobi : public CommonTestFixture { using Bj = gko::preconditioner::Jacobi<>; using Mtx = gko::matrix::Csr<>; using Vec = gko::matrix::MultiVector<>; + using Dense = gko::matrix::Dense; using mtx_data = gko::matrix_data<>; using value_type = typename Mtx::value_type; using index_type = typename Mtx::index_type; @@ -489,7 +491,7 @@ TEST_F(Jacobi, ScalarApplyEquivalentToRef) dim, dim, std::uniform_int_distribution<>(1, dim), std::normal_distribution<>(1.0, 2.0), engine); gko::utils::make_diag_dominant(dense_data); - auto dense_smtx = gko::share(Vec::create(ref)); + auto dense_smtx = gko::share(Dense::create(ref)); dense_smtx->read(dense_data); auto smtx = gko::share(Mtx::create(ref)); smtx->copy_from(dense_smtx); @@ -523,7 +525,7 @@ TEST_F(Jacobi, ScalarL1ApplyEquivalentToRef) dim, dim, std::uniform_int_distribution<>(1, dim), std::normal_distribution<>(1.0, 2.0), engine); gko::utils::make_diag_dominant(dense_data, 1.001); - auto dense_smtx = gko::share(Vec::create(ref)); + auto dense_smtx = gko::share(Dense::create(ref)); dense_smtx->read(dense_data); auto smtx = gko::share(Mtx::create(ref)); smtx->copy_from(dense_smtx); @@ -597,7 +599,7 @@ TEST_F(Jacobi, ScalarLinearCombinationApplyEquivalentToRef) dim, dim, std::uniform_int_distribution<>(1, dim), std::normal_distribution<>(1.0, 2.0), engine); gko::utils::make_diag_dominant(dense_data); - auto dense_smtx = gko::share(Vec::create(ref)); + auto dense_smtx = gko::share(Dense::create(ref)); dense_smtx->read(dense_data); auto smtx = gko::share(Mtx::create(ref)); smtx->copy_from(dense_smtx); @@ -1022,7 +1024,7 @@ TEST_F( TEST_F(Jacobi, ScalarJacobiHandleZero) { auto mtx = gko::share( - gko::initialize({{0, 0, 0}, {0, 2, 0}, {0, 0, 0}}, ref)); + gko::initialize({{0, 0, 0}, {0, 2, 0}, {0, 0, 0}}, ref)); auto b = gko::initialize({1, 2, 3}, ref); auto x = Vec::create(ref, gko::dim<2>(3, 1)); auto d_b = b->clone(exec); diff --git a/test/reorder/mc64.cpp b/test/reorder/mc64.cpp index f05b13d19c0..f079cb56c8e 100644 --- a/test/reorder/mc64.cpp +++ b/test/reorder/mc64.cpp @@ -1,10 +1,11 @@ -// SPDX-FileCopyrightText: 2017 - 2024 The Ginkgo authors +// SPDX-FileCopyrightText: 2017 - 2026 The Ginkgo authors // // SPDX-License-Identifier: BSD-3-Clause #include #include +#include #include #include "core/test/utils/assertions.hpp" diff --git a/test/solver/bicg_kernels.cpp b/test/solver/bicg_kernels.cpp index f2f7f7a5d78..7cacc76ca71 100644 --- a/test/solver/bicg_kernels.cpp +++ b/test/solver/bicg_kernels.cpp @@ -25,12 +25,13 @@ class Bicg : public CommonTestFixture { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; Bicg() : rand_engine(30) { std::string file_name(gko::matrices::location_ani1_mtx); auto input_file = std::ifstream(file_name, std::ios::in); - mtx_ani = gko::read(input_file, ref); + mtx_ani = gko::read(input_file, ref); d_mtx_ani = gko::clone(exec, mtx_ani.get()); } @@ -105,7 +106,7 @@ class Bicg : public CommonTestFixture { std::unique_ptr beta; std::unique_ptr prev_rho; std::unique_ptr rho; - std::shared_ptr mtx_ani; + std::shared_ptr mtx_ani; gko::array stop_status; std::unique_ptr d_b; @@ -121,7 +122,7 @@ class Bicg : public CommonTestFixture { std::unique_ptr d_beta; std::unique_ptr d_prev_rho; std::unique_ptr d_rho; - std::shared_ptr d_mtx_ani; + std::shared_ptr d_mtx_ani; gko::array d_stop_status; }; @@ -210,7 +211,7 @@ TEST_F(Bicg, ApplyWithSpdMatrixIsEquivalentToRef) gko::dim<2>{50, 50}, std::normal_distribution(-1.0, 1.0), rand_engine); gko::utils::make_hpd(data); - auto mtx = Mtx::create(ref, data.size, 53); + auto mtx = Dense::create(ref, data.size, 53); mtx->read(data); auto x = gen_mtx(50, 3, 5); auto b = gen_mtx(50, 3, 4); diff --git a/test/solver/bicgstab_kernels.cpp b/test/solver/bicgstab_kernels.cpp index 1b69e88f0da..6cf1632081e 100644 --- a/test/solver/bicgstab_kernels.cpp +++ b/test/solver/bicgstab_kernels.cpp @@ -25,6 +25,7 @@ class Bicgstab : public CommonTestFixture { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Bicgstab; Bicgstab() : rand_engine(30) @@ -33,7 +34,7 @@ class Bicgstab : public CommonTestFixture { gko::dim<2>{123, 123}, std::normal_distribution(-1.0, 1.0), rand_engine); gko::utils::make_diag_dominant(data); - mtx = Mtx::create(ref, data.size, 125); + mtx = Dense::create(ref, data.size, 125); mtx->read(data); d_mtx = gko::clone(exec, mtx); exec_bicgstab_factory = @@ -119,8 +120,8 @@ class Bicgstab : public CommonTestFixture { std::default_random_engine rand_engine; - std::shared_ptr mtx; - std::shared_ptr d_mtx; + std::shared_ptr mtx; + std::shared_ptr d_mtx; std::unique_ptr exec_bicgstab_factory; std::unique_ptr ref_bicgstab_factory; diff --git a/test/solver/cg_kernels.cpp b/test/solver/cg_kernels.cpp index 231bde62f3a..185331e4e79 100644 --- a/test/solver/cg_kernels.cpp +++ b/test/solver/cg_kernels.cpp @@ -24,6 +24,7 @@ class Cg : public CommonTestFixture { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; Cg() : rand_engine(30) {} @@ -169,7 +170,7 @@ TEST_F(Cg, ApplyIsEquivalentToRef) gko::dim<2>{50, 50}, std::normal_distribution(-1.0, 1.0), rand_engine); gko::utils::make_hpd(data); - auto mtx = Mtx::create(ref, data.size, 53); + auto mtx = Dense::create(ref, data.size, 53); mtx->read(data); auto x = gen_mtx(50, 3, 5); auto b = gen_mtx(50, 3, 4); diff --git a/test/solver/cgs_kernels.cpp b/test/solver/cgs_kernels.cpp index 03630dec81c..c84e4b14da5 100644 --- a/test/solver/cgs_kernels.cpp +++ b/test/solver/cgs_kernels.cpp @@ -24,6 +24,7 @@ class Cgs : public CommonTestFixture { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Cgs; Cgs() : rand_engine(30) @@ -32,7 +33,7 @@ class Cgs : public CommonTestFixture { gko::dim<2>{123, 123}, std::normal_distribution(-1.0, 1.0), rand_engine); gko::utils::make_diag_dominant(data); - mtx = Mtx::create(ref, data.size, 125); + mtx = Dense::create(ref, data.size, 125); mtx->read(data); d_mtx = gko::clone(exec, mtx); exec_cgs_factory = @@ -112,8 +113,8 @@ class Cgs : public CommonTestFixture { std::default_random_engine rand_engine; - std::shared_ptr mtx; - std::shared_ptr d_mtx; + std::shared_ptr mtx; + std::shared_ptr d_mtx; std::unique_ptr exec_cgs_factory; std::unique_ptr ref_cgs_factory; diff --git a/test/solver/chebyshev_kernels.cpp b/test/solver/chebyshev_kernels.cpp index 547c7667a78..deeb24d107b 100644 --- a/test/solver/chebyshev_kernels.cpp +++ b/test/solver/chebyshev_kernels.cpp @@ -24,18 +24,22 @@ class Chebyshev : public CommonTestFixture { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using coeff_type = gko::solver::detail::coeff_type; Chebyshev() : rand_engine(30) {} - std::unique_ptr gen_mtx(gko::size_type num_rows, - gko::size_type num_cols, gko::size_type stride) + template + std::unique_ptr gen_mtx(gko::size_type num_rows, + gko::size_type num_cols, + gko::size_type stride) { - auto tmp_mtx = gko::test::generate_random_matrix( + auto tmp_mtx = gko::test::generate_random_matrix( num_rows, num_cols, std::uniform_int_distribution<>(num_cols, num_cols), std::normal_distribution(-1.0, 1.0), rand_engine, ref); - auto result = Mtx::create(ref, gko::dim<2>{num_rows, num_cols}, stride); + auto result = + MtxType::create(ref, gko::dim<2>{num_rows, num_cols}, stride); result->copy_from(tmp_mtx); return result; } @@ -94,7 +98,7 @@ TEST_F(Chebyshev, KernelUpdate) TEST_F(Chebyshev, ApplyIsEquivalentToRef) { - auto mtx = gen_mtx(50, 50, 52); + auto mtx = gen_mtx(50, 50, 52); auto x = gen_mtx(50, 3, 8); auto b = gen_mtx(50, 3, 5); auto d_mtx = gko::clone(exec, mtx); @@ -123,7 +127,7 @@ TEST_F(Chebyshev, ApplyIsEquivalentToRef) TEST_F(Chebyshev, ApplyWithIterativeInnerSolverIsEquivalentToRef) { - auto mtx = gen_mtx(50, 50, 54); + auto mtx = gen_mtx(50, 50, 54)->as_dense_view()->clone(); auto x = gen_mtx(50, 3, 6); auto b = gen_mtx(50, 3, 10); auto d_mtx = gko::clone(exec, mtx); @@ -159,7 +163,7 @@ TEST_F(Chebyshev, ApplyWithIterativeInnerSolverIsEquivalentToRef) TEST_F(Chebyshev, ApplyWithGivenInitialGuessModeIsEquivalentToRef) { using initial_guess_mode = gko::solver::initial_guess_mode; - auto mtx = gko::share(gen_mtx(50, 50, 52)); + auto mtx = gko::share(gen_mtx(50, 50, 52)); auto b = gen_mtx(50, 3, 7); auto d_mtx = gko::share(clone(exec, mtx)); auto d_b = gko::clone(exec, b); diff --git a/test/solver/fcg_kernels.cpp b/test/solver/fcg_kernels.cpp index f60e62f60a1..dfad9ad3060 100644 --- a/test/solver/fcg_kernels.cpp +++ b/test/solver/fcg_kernels.cpp @@ -24,6 +24,7 @@ class Fcg : public CommonTestFixture { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Fcg; Fcg() : rand_engine(30) {} @@ -180,7 +181,7 @@ TEST_F(Fcg, ApplyIsEquivalentToRef) gko::dim<2>{50, 50}, std::normal_distribution(-1.0, 1.0), rand_engine); gko::utils::make_hpd(data, 1.5); - auto mtx = Mtx::create(ref, data.size, 53); + auto mtx = Dense::create(ref, data.size, 53); mtx->read(data); auto x = gen_mtx(50, 3, 4); auto b = gen_mtx(50, 3, 5); diff --git a/test/solver/gcr_kernels.cpp b/test/solver/gcr_kernels.cpp index 50de0cb289f..fd480d60e16 100644 --- a/test/solver/gcr_kernels.cpp +++ b/test/solver/gcr_kernels.cpp @@ -25,6 +25,7 @@ class Gcr : public CommonTestFixture { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Gcr; using norm_type = gko::remove_complex; using NormVector = gko::matrix::MultiVector; @@ -34,7 +35,7 @@ class Gcr : public CommonTestFixture { Gcr() : rand_engine(30) { - mtx = gen_mtx(123, 123); + mtx = gen_mtx(123, 123)->as_dense_view()->clone(); mtx->write(data); gko::utils::make_spd(data, 1.0001); mtx->read(data); @@ -109,9 +110,9 @@ class Gcr : public CommonTestFixture { std::default_random_engine rand_engine; - std::shared_ptr mtx; + std::shared_ptr mtx; mtx_data data; - std::shared_ptr d_mtx; + std::shared_ptr d_mtx; std::unique_ptr exec_gcr_factory; std::unique_ptr ref_gcr_factory; diff --git a/test/solver/gmres_kernels.cpp b/test/solver/gmres_kernels.cpp index d195bb55cd7..59c5eb71767 100644 --- a/test/solver/gmres_kernels.cpp +++ b/test/solver/gmres_kernels.cpp @@ -25,6 +25,7 @@ class Gmres : public CommonTestFixture { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Gmres; using norm_type = gko::remove_complex; using NormVector = gko::matrix::MultiVector; @@ -33,7 +34,7 @@ class Gmres : public CommonTestFixture { Gmres() : rand_engine(30) { - mtx = gen_mtx(123, 123); + mtx = gen_mtx(123, 123)->as_dense_view()->clone(); d_mtx = gko::clone(exec, mtx); exec_gmres_factory = Solver::build() @@ -112,8 +113,8 @@ class Gmres : public CommonTestFixture { std::default_random_engine rand_engine; - std::shared_ptr mtx; - std::shared_ptr d_mtx; + std::shared_ptr mtx; + std::shared_ptr d_mtx; std::unique_ptr exec_gmres_factory; std::unique_ptr ref_gmres_factory; @@ -293,23 +294,24 @@ TEST_F(Gmres, GmresKernelMultiAxpyIsEquivalentToRef) TEST_F(Gmres, GmresKernelMultiDotIsEquivalentToRef) { initialize_data(); - auto krylov_basis = krylov_bases->create_submatrix( - gko::span{0, x->get_size()[0] * gko::solver::gmres_default_krylov_dim}, - gko::span{0, x->get_size()[1]}); - auto d_krylov_basis = d_krylov_bases->create_submatrix( - gko::span{0, - d_x->get_size()[0] * gko::solver::gmres_default_krylov_dim}, - gko::span{0, d_x->get_size()[1]}); - auto next_krylov = krylov_bases->create_submatrix( - gko::span{ + auto krylov_basis = krylov_bases->create_subview( + gko::local_span{ + 0, x->get_size()[0] * gko::solver::gmres_default_krylov_dim}, + gko::local_span{0, x->get_size()[1]}); + auto d_krylov_basis = d_krylov_bases->create_subview( + gko::local_span{ + 0, d_x->get_size()[0] * gko::solver::gmres_default_krylov_dim}, + gko::local_span{0, d_x->get_size()[1]}); + auto next_krylov = krylov_bases->create_subview( + gko::local_span{ x->get_size()[0] * gko::solver::gmres_default_krylov_dim, x->get_size()[0] * (gko::solver::gmres_default_krylov_dim + 1)}, - gko::span{0, x->get_size()[1]}); - auto d_next_krylov = d_krylov_bases->create_submatrix( - gko::span{ + gko::local_span{0, x->get_size()[1]}); + auto d_next_krylov = d_krylov_bases->create_subview( + gko::local_span{ d_x->get_size()[0] * gko::solver::gmres_default_krylov_dim, d_x->get_size()[0] * (gko::solver::gmres_default_krylov_dim + 1)}, - gko::span{0, d_x->get_size()[1]}); + gko::local_span{0, d_x->get_size()[1]}); gko::kernels::reference::gmres::multi_dot( ref, krylov_basis->get_const_device_view(), diff --git a/test/solver/idr_kernels.cpp b/test/solver/idr_kernels.cpp index 0309e8e0828..55543bbdd2c 100644 --- a/test/solver/idr_kernels.cpp +++ b/test/solver/idr_kernels.cpp @@ -38,6 +38,7 @@ using rr = typename gko::test::reduction_factor; class Idr : public CommonTestFixture { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; using Solver = gko::solver::Idr; Idr() : rand_engine(30) @@ -68,7 +69,7 @@ class Idr : public CommonTestFixture { { nrhs = input_nrhs; int s = 4; - mtx = gen_mtx(size, size); + mtx = gen_mtx(size, size)->as_dense_view()->clone(); x = gen_mtx(size, nrhs); b = gen_mtx(size, nrhs); r = gen_mtx(size, nrhs); @@ -108,8 +109,8 @@ class Idr : public CommonTestFixture { std::default_random_engine rand_engine; - std::shared_ptr mtx; - std::shared_ptr d_mtx; + std::shared_ptr mtx; + std::shared_ptr d_mtx; std::unique_ptr exec_idr_factory; std::unique_ptr ref_idr_factory; diff --git a/test/solver/ir_kernels.cpp b/test/solver/ir_kernels.cpp index cf708c1cc87..8cd1b908fd4 100644 --- a/test/solver/ir_kernels.cpp +++ b/test/solver/ir_kernels.cpp @@ -62,7 +62,7 @@ TEST_F(Ir, InitializeIsEquivalentToRef) TEST_F(Ir, ApplyIsEquivalentToRef) { - auto mtx = gen_mtx(50, 50, 52); + auto mtx = gen_mtx(50, 50, 52)->as_dense_view()->clone(); auto x = gen_mtx(50, 3, 8); auto b = gen_mtx(50, 3, 5); auto d_mtx = clone(exec, mtx); @@ -91,7 +91,7 @@ TEST_F(Ir, ApplyIsEquivalentToRef) TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef) { - auto mtx = gen_mtx(50, 50, 54); + auto mtx = gen_mtx(50, 50, 54)->as_dense_view()->clone(); auto x = gen_mtx(50, 3, 6); auto b = gen_mtx(50, 3, 10); auto d_mtx = clone(exec, mtx); @@ -125,7 +125,7 @@ TEST_F(Ir, ApplyWithIterativeInnerSolverIsEquivalentToRef) TEST_F(Ir, RichardsonApplyIsEquivalentToRef) { - auto mtx = gen_mtx(50, 50, 54); + auto mtx = gen_mtx(50, 50, 54)->as_dense_view()->clone(); auto x = gen_mtx(50, 3, 4); auto b = gen_mtx(50, 3, 3); auto d_mtx = clone(exec, mtx); @@ -156,7 +156,7 @@ TEST_F(Ir, RichardsonApplyIsEquivalentToRef) TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef) { - auto mtx = gen_mtx(50, 50, 52); + auto mtx = gen_mtx(50, 50, 52)->as_dense_view()->clone(); auto x = gen_mtx(50, 3, 4); auto b = gen_mtx(50, 3, 7); auto d_mtx = clone(exec, mtx); @@ -192,7 +192,7 @@ TEST_F(Ir, RichardsonApplyWithIterativeInnerSolverIsEquivalentToRef) TEST_F(Ir, ApplyWithGivenInitialGuessModeIsEquivalentToRef) { using initial_guess_mode = gko::solver::initial_guess_mode; - auto mtx = gko::share(gen_mtx(50, 50, 52)); + auto mtx = gko::share(gen_mtx(50, 50, 52)->as_dense_view()->clone()); auto b = gen_mtx(50, 3, 7); auto d_mtx = gko::share(clone(exec, mtx)); auto d_b = clone(exec, b); diff --git a/test/solver/minres_kernels.cpp b/test/solver/minres_kernels.cpp index ed9515dd3a4..5276fe22db4 100644 --- a/test/solver/minres_kernels.cpp +++ b/test/solver/minres_kernels.cpp @@ -266,7 +266,7 @@ TEST_F(Minres, MinresStep2IsEquivalentToRef) TEST_F(Minres, ApplyIsEquivalentToRef) { - auto mtx = gen_mtx(50, 50, 53, true); + auto mtx = gen_mtx(50, 50, 53, true)->as_dense_view()->clone(); auto x = gen_mtx(50, 1, 5, false); auto b = gen_mtx(50, 1, 4, false); auto d_mtx = gko::clone(exec, mtx); @@ -300,7 +300,7 @@ TEST_F(Minres, ApplyIsEquivalentToRef) TEST_F(Minres, PreconditionedApplyIsEquivalentToRef) { - auto mtx = gen_mtx(50, 50, 53, true); + auto mtx = gen_mtx(50, 50, 53, true)->as_dense_view()->clone(); auto x = gen_mtx(50, 1, 5, false); auto b = gen_mtx(50, 1, 4, false); auto d_mtx = gko::clone(exec, mtx); diff --git a/test/solver/pipe_cg_kernels.cpp b/test/solver/pipe_cg_kernels.cpp index 0d9aa7c586a..dddc9c9a531 100644 --- a/test/solver/pipe_cg_kernels.cpp +++ b/test/solver/pipe_cg_kernels.cpp @@ -24,6 +24,7 @@ class PipeCg : public CommonTestFixture { protected: using Mtx = gko::matrix::MultiVector; + using Dense = gko::matrix::Dense; PipeCg() : rand_engine(30) {} @@ -239,7 +240,7 @@ TEST_F(PipeCg, ApplyIsEquivalentToRef) gko::dim<2>{50, 50}, std::normal_distribution(-1.0, 1.0), rand_engine); gko::utils::make_hpd(data); - auto mtx = Mtx::create(ref, data.size, 53); + auto mtx = Dense::create(ref, data.size, 53); mtx->read(data); auto x = gen_mtx(50, 3, 5); auto b = gen_mtx(50, 3, 4); diff --git a/test/solver/solver.cpp b/test/solver/solver.cpp index 5eb90055ae3..0a8a8b089d3 100644 --- a/test/solver/solver.cpp +++ b/test/solver/solver.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -128,7 +129,7 @@ struct Fcg : SimpleSolverTest> { struct PipeCg : SimpleSolverTest> { - static double tolerance() { return 1e7 * r::value; } + static double tolerance() { return 4e7 * r::value; } }; diff --git a/test/stop/residual_norm_kernels.cpp b/test/stop/residual_norm_kernels.cpp index 98c693494d3..445e9dda61f 100644 --- a/test/stop/residual_norm_kernels.cpp +++ b/test/stop/residual_norm_kernels.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include "core/test/utils.hpp" @@ -66,7 +67,8 @@ TYPED_TEST(ResidualNorm, CanIgorneResidualNorm) using Mtx = typename TestFixture::Mtx; using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec); auto criterion = this->factory->generate(nullptr, rhs, nullptr, initial_res.get()); constexpr gko::uint8 RelativeStoppingId{1}; @@ -92,9 +94,11 @@ TYPED_TEST(ResidualNorm, CheckIfResZeroConverges) using Csr = gko::matrix::Csr; using mode = gko::stop::mode; std::shared_ptr mtx = gko::initialize({1.0}, this->exec); - std::shared_ptr rhs = gko::initialize({0.0}, this->exec); - std::shared_ptr x = gko::initialize({0.0}, this->exec); - std::shared_ptr res_norm = + std::shared_ptr rhs = + gko::initialize({0.0}, this->exec); + std::shared_ptr x = + gko::initialize({0.0}, this->exec); + std::shared_ptr res_norm = gko::initialize({0.0}, this->exec); for (auto baseline : @@ -127,7 +131,8 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoal) using Mtx = typename TestFixture::Mtx; using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec); auto criterion = this->factory->generate(nullptr, rhs, nullptr, initial_res.get()); auto rel_criterion = @@ -227,7 +232,7 @@ TYPED_TEST(ResidualNorm, WaitsTillResidualGoalMultipleRHS) using T = TypeParam; using T_nc = gko::remove_complex; auto res = gko::initialize({I{100.0, 100.0}}, this->exec); - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({I{10.0, 10.0}}, this->exec); auto criterion = this->factory->generate(nullptr, rhs, nullptr, res.get()); auto rel_criterion = @@ -353,7 +358,8 @@ TYPED_TEST(ResidualNormWithInitialResnorm, WaitsTillResidualGoal) using Mtx = typename TestFixture::Mtx; using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec); auto res_norm = gko::initialize({100.0}, this->exec); auto criterion = this->factory->generate(nullptr, rhs, nullptr, initial_res.get()); @@ -393,7 +399,7 @@ TYPED_TEST(ResidualNormWithInitialResnorm, WaitsTillResidualGoalMultipleRHS) auto res = gko::initialize({I{100.0, 100.0}}, this->exec); auto res_norm = gko::initialize({I{100.0, 100.0}}, this->exec); - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({I{10.0, 10.0}}, this->exec); auto criterion = this->factory->generate(nullptr, rhs, nullptr, res.get()); bool one_changed{}; @@ -452,7 +458,8 @@ TYPED_TEST(ResidualNormWithRhsNorm, WaitsTillResidualGoal) using Mtx = typename TestFixture::Mtx; using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec); auto rhs_norm = gko::initialize({I{0.0}}, this->exec); gko::as(rhs)->compute_norm2(rhs_norm); auto res_norm = gko::initialize({100.0}, this->exec); @@ -494,7 +501,7 @@ TYPED_TEST(ResidualNormWithRhsNorm, WaitsTillResidualGoalMultipleRHS) auto res = gko::initialize({I{100.0, 100.0}}, this->exec); auto res_norm = gko::initialize({I{100.0, 100.0}}, this->exec); - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({I{10.0, 10.0}}, this->exec); auto rhs_norm = gko::initialize({I{0.0, 0.0}}, this->exec); @@ -558,9 +565,11 @@ TYPED_TEST(ImplicitResidualNorm, CheckIfResZeroConverges) using Csr = gko::matrix::Csr; using gko::stop::mode; std::shared_ptr mtx = gko::initialize({1.0}, this->exec); - std::shared_ptr rhs = gko::initialize({0.0}, this->exec); - std::shared_ptr x = gko::initialize({0.0}, this->exec); - std::shared_ptr implicit_sq_res_norm = + std::shared_ptr rhs = + gko::initialize({0.0}, this->exec); + std::shared_ptr x = + gko::initialize({0.0}, this->exec); + std::shared_ptr implicit_sq_res_norm = gko::initialize({0.0}, this->exec); for (auto baseline : @@ -596,7 +605,8 @@ TYPED_TEST(ImplicitResidualNorm, WaitsTillResidualGoal) using Mtx = typename TestFixture::Mtx; using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec); auto res_norm = gko::initialize({100.0}, this->exec); auto rhs_norm = gko::initialize({I{0.0}}, this->exec); gko::as(rhs)->compute_norm2(rhs_norm); @@ -639,7 +649,7 @@ TYPED_TEST(ImplicitResidualNorm, WaitsTillResidualGoalMultipleRHS) using T_nc = gko::remove_complex; auto res = gko::initialize({I{100.0, 100.0}}, this->exec); auto res_norm = gko::initialize({I{100.0, 100.0}}, this->exec); - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({I{10.0, 10.0}}, this->exec); auto rhs_norm = gko::initialize({I{0.0, 0.0}}, this->exec); @@ -701,7 +711,8 @@ TYPED_TEST(ResidualNormWithAbsolute, WaitsTillResidualGoal) using Mtx = typename TestFixture::Mtx; using NormVector = typename TestFixture::NormVector; auto initial_res = gko::initialize({100.0}, this->exec); - std::shared_ptr rhs = gko::initialize({10.0}, this->exec); + std::shared_ptr rhs = + gko::initialize({10.0}, this->exec); auto res_norm = gko::initialize({100.0}, this->exec); auto criterion = this->factory->generate(nullptr, rhs, nullptr, initial_res.get()); @@ -741,7 +752,7 @@ TYPED_TEST(ResidualNormWithAbsolute, WaitsTillResidualGoalMultipleRHS) auto res = gko::initialize({I{100.0, 100.0}}, this->exec); auto res_norm = gko::initialize({I{100.0, 100.0}}, this->exec); - std::shared_ptr rhs = + std::shared_ptr rhs = gko::initialize({I{10.0, 10.0}}, this->exec); auto criterion = this->factory->generate(nullptr, rhs, nullptr, res.get()); bool one_changed{}; diff --git a/test/test_install/test_install.cpp b/test/test_install/test_install.cpp index 5f204cf8333..db27e98218c 100644 --- a/test/test_install/test_install.cpp +++ b/test/test_install/test_install.cpp @@ -364,9 +364,9 @@ int main() Mtx::create(exec, gko::matrix::csr::spmv_strategy::classical); } - // core/matrix/multivector.hpp + // core/matrix/dense.hpp { - using Mtx = gko::matrix::MultiVector<>; + using Mtx = gko::matrix::Dense<>; check_spmv(exec, A_raw, b, x); }