From bd46488aaad0fd22189481fc918150b274d12f55 Mon Sep 17 00:00:00 2001 From: Radzivon Bartoshyk Date: Sun, 29 Mar 2026 18:32:48 +0100 Subject: [PATCH] CI update, lots of improvements --- .github/workflows/build_push.yml | 28 +- .github/workflows/mirroring.yaml | 25 + .github/workflows/publish_release.yml | 2 +- Cargo.toml | 8 +- app/Cargo.toml | 2 +- app/benches/gauss/main.rs | 40 +- app/src/main.rs | 26 +- fuzz/Cargo.toml | 2 +- src/adaptive_blur.rs | 10 +- src/avx/fast_gaussian.rs | 776 +++++---- src/avx/fast_gaussian_f32.rs | 709 ++++---- src/avx/fast_gaussian_next.rs | 1074 ++++++------ src/avx/fast_gaussian_next_f32.rs | 814 ++++----- src/avx/fast_gaussian_next_u16.rs | 954 ++++++----- src/avx/fast_gaussian_u16.rs | 806 ++++----- src/avx/pack.rs | 548 +++--- src/avx/utils.rs | 60 +- src/avx/v_load.rs | 94 +- src/avx/v_store.rs | 74 +- src/bilateral/bp8.rs | 4 +- src/bilateral/mod.rs | 2 +- src/bilateral/neon/bp8.rs | 46 +- src/box_filter/avx/hrgba8.rs | 78 +- src/box_filter/avx/rgba_f32.rs | 54 +- src/box_filter/avx/vrgba8.rs | 8 +- src/box_filter/avx/vrgba8_low_radius.rs | 895 +++++----- src/box_filter/avx/vsum8.rs | 10 +- src/box_filter/box_blur.rs | 47 +- src/box_filter/neon/mod.rs | 12 +- src/box_filter/neon/rgba16.rs | 568 +++--- src/box_filter/neon/rgba8.rs | 1525 +++++++++-------- src/box_filter/neon/rgba8_q0_31.rs | 91 +- src/box_filter/neon/rgba_f32.rs | 506 +++--- src/box_filter/neon/vsum8.rs | 16 +- src/box_filter/sse/mod.rs | 8 +- src/box_filter/sse/rgba16.rs | 754 ++++---- src/box_filter/sse/rgba8.rs | 843 ++++----- src/box_filter/sse/rgba_f32.rs | 2 +- src/box_filter/sse/vsum16.rs | 8 +- src/box_filter/sse/vsum8.rs | 8 +- src/box_filter/sse/vsum_f32.rs | 8 +- src/fast_bilateral_filter.rs | 4 +- src/fast_bilateral_image.rs | 4 +- src/fast_gaussian.rs | 54 +- src/fast_gaussian_image.rs | 4 +- src/fast_gaussian_image_next.rs | 4 +- src/fast_gaussian_next.rs | 50 +- src/filter1d/avx/filter_column.rs | 336 ++-- src/filter1d/avx/filter_column_approx.rs | 164 +- .../avx/filter_column_complex_u16_f32.rs | 2 +- .../avx/filter_column_complex_u16_i32.rs | 48 +- .../avx/filter_column_complex_u8_f32.rs | 78 +- .../avx/filter_column_complex_u8_i32.rs | 42 +- src/filter1d/avx/filter_column_f32.rs | 4 +- src/filter1d/avx/filter_column_symm.rs | 369 ++-- src/filter1d/avx/filter_column_symm_approx.rs | 57 +- .../avx/filter_column_symm_approx_uq0_7.rs | 65 +- .../avx/filter_column_symm_approx_x2.rs | 54 +- src/filter1d/avx/filter_column_symm_f32.rs | 6 +- .../avx/filter_column_symm_f32_f64.rs | 22 +- src/filter1d/avx/filter_column_symm_u16.rs | 465 ++--- src/filter1d/avx/filter_column_symm_u16_x2.rs | 500 +++--- .../avx/filter_column_symm_uq15_u16.rs | 21 +- .../avx/filter_column_symm_uq15_u16_x2.rs | 23 +- src/filter1d/avx/filter_column_symm_x2.rs | 368 ++-- src/filter1d/avx/filter_row.rs | 264 +-- src/filter1d/avx/filter_row_approx.rs | 186 +- src/filter1d/avx/filter_row_complex_u8_i32.rs | 56 +- src/filter1d/avx/filter_row_f32.rs | 200 +-- src/filter1d/avx/filter_row_f32_f64_symm.rs | 373 ++-- src/filter1d/avx/filter_row_f32_symm.rs | 258 +-- src/filter1d/avx/filter_row_symm.rs | 290 ++-- src/filter1d/avx/filter_row_symm_approx.rs | 259 +-- src/filter1d/avx/filter_row_symm_u16.rs | 326 ++-- src/filter1d/avx/filter_row_symm_uq15_u16.rs | 333 ++-- src/filter1d/avx/mod.rs | 11 +- src/filter1d/avx/row_symm_approx_binter.rs | 378 ++-- .../avx/row_symm_approx_binter3_uq0_7.rs | 449 +++-- .../avx/row_symm_approx_binter5_uq0_7.rs | 576 +++---- .../avx/row_symm_approx_binter_uq0_7.rs | 533 +++--- src/filter1d/avx/sse_utils.rs | 828 ++++----- src/filter1d/avx/utils.rs | 351 ++-- .../avx512/filter_column_symm_approx.rs | 69 +- .../avx512/filter_column_symm_approx_uq0_7.rs | 65 +- .../avx512/filter_row_symm_uq15_u16.rs | 355 ++-- src/filter1d/avx512/mod.rs | 1 + src/filter1d/avx512/pack.rs | 410 ++--- src/filter1d/avx512/row_symm_approx_binter.rs | 419 ++--- .../avx512/row_symm_approx_binter3_uq0_7.rs | 526 +++--- src/filter1d/avx512/sse_utils.rs | 708 ++++---- src/filter1d/avx512/utils.rs | 194 ++- src/filter1d/avx512/v_load.rs | 122 +- src/filter1d/avx512/v_store.rs | 94 +- src/filter1d/filter.rs | 2 +- src/filter1d/filter_1d_approx.rs | 207 +-- src/filter1d/filter_1d_column_handler.rs | 2 +- .../filter_1d_column_handler_approx.rs | 577 ++++--- src/filter1d/filter_1d_row_handler.rs | 2 +- src/filter1d/filter_1d_row_handler_approx.rs | 4 +- src/filter1d/filter_column_approx.rs | 23 +- .../filter_column_approx_symmetric.rs | 27 +- src/filter1d/filter_complex.rs | 2 +- src/filter1d/filter_complex_dispatch.rs | 4 +- src/filter1d/filter_complex_dispatch_q.rs | 2 +- src/filter1d/filter_complex_q.rs | 2 +- src/filter1d/mod.rs | 4 +- src/filter1d/neon/filter_column_approx.rs | 33 +- src/filter1d/neon/filter_column_approx_rdm.rs | 45 +- .../neon/filter_column_complex_u16_i32.rs | 26 +- .../neon/filter_column_complex_u8_f32.rs | 40 +- .../neon/filter_column_complex_u8_f32_fcma.rs | 18 +- .../neon/filter_column_complex_u8_i32.rs | 84 +- .../neon/filter_column_symm_approx.rs | 57 +- .../neon/filter_column_symm_approx_rdm.rs | 39 +- .../neon/filter_column_symm_approx_rdm_x3.rs | 27 +- .../neon/filter_column_symm_approx_uq0_7.rs | 48 +- .../neon/filter_column_symm_u15_u16_x3.rs | 19 +- .../neon/filter_column_symm_uq15_u16.rs | 19 +- .../neon/filter_row_symm_approx_rdm.rs | 4 +- src/filter1d/neon/mod.rs | 17 +- src/filter1d/neon/rom_symm_approx3.rs | 23 +- src/filter1d/neon/rom_symm_approx5.rs | 31 +- src/filter1d/neon/row_symm_app_binter_rdm.rs | 99 +- src/filter1d/neon/row_symm_approx_binter.rs | 55 +- .../neon/row_symm_approx_binter_uq0_7.rs | 33 +- src/filter1d/neon/utils.rs | 601 ++++--- src/filter1d/row_handler_small_approx.rs | 275 +-- src/filter1d/row_symm_approx_binter.rs | 2 +- src/filter1d/sse/filter_column.rs | 294 ++-- src/filter1d/sse/filter_column_approx.rs | 35 +- .../sse/filter_column_complex_u16_i32.rs | 26 +- .../sse/filter_column_complex_u8_f32.rs | 39 +- .../sse/filter_column_complex_u8_i32.rs | 20 +- src/filter1d/sse/filter_column_symm.rs | 384 ++--- src/filter1d/sse/filter_column_symm_approx.rs | 63 +- .../sse/filter_column_symm_approx_uq0_7.rs | 53 +- src/filter1d/sse/filter_column_symm_u16.rs | 371 ++-- src/filter1d/sse/filter_column_symm_u8_i16.rs | 350 ++-- .../sse/filter_column_symm_uq15_u16.rs | 17 +- src/filter1d/sse/filter_column_u8_i16.rs | 304 ++-- src/filter1d/sse/filter_row.rs | 225 +-- src/filter1d/sse/filter_row_approx.rs | 204 +-- src/filter1d/sse/filter_row_complex_u8_i32.rs | 22 +- src/filter1d/sse/filter_row_f32.rs | 130 +- src/filter1d/sse/filter_row_f32_f64.rs | 168 +- src/filter1d/sse/filter_row_symm.rs | 342 ++-- src/filter1d/sse/filter_row_symm_approx.rs | 316 ++-- src/filter1d/sse/filter_row_symm_u16.rs | 231 +-- src/filter1d/sse/filter_row_symm_u8_i16.rs | 222 +-- src/filter1d/sse/filter_row_symm_uq15_u16.rs | 199 +-- src/filter1d/sse/filter_row_u8_i16.rs | 150 +- src/filter1d/sse/mod.rs | 13 +- .../sse/row_symm_approx_binter_uq0_7.rs | 420 +++-- src/filter1d/sse/utils.rs | 711 ++++---- src/filter2d/avx/convolve_op_u8_i16_fp.rs | 554 +++--- src/filter2d/convolve_op.rs | 2 +- src/filter2d/convolve_op_fp.rs | 2 +- src/filter2d/fft_utils.rs | 16 +- src/filter2d/filter_2d.rs | 2 +- src/filter2d/filter_2d_fft.rs | 2 +- src/filter2d/filter_2d_fft_real.rs | 2 +- src/filter2d/filter_2d_handler.rs | 2 +- src/filter2d/filter_2d_rgb.rs | 2 +- src/filter2d/filter_2d_rgb_fft.rs | 2 +- src/filter2d/filter_2d_rgba.rs | 2 +- src/filter2d/neon/convolve_op_u8_f32.rs | 4 +- src/filter2d/neon/convolve_op_u8_i16.rs | 4 +- src/filter2d/neon/convolve_op_u8_i16_fp.rs | 2 +- src/filter2d/sse/convolve_op_u8_f32.rs | 264 +-- src/filter2d/sse/convolve_op_u8_i16.rs | 4 +- src/gaussian/declaration.rs | 8 +- src/gaussian/gaussian_util.rs | 4 +- src/gaussian/mod.rs | 2 +- src/gaussian_blur_image.rs | 5 +- src/image.rs | 20 +- src/laplacian.rs | 4 +- src/lens.rs | 4 +- src/lib.rs | 41 +- src/motion_blur.rs | 2 +- src/neon/f16_utils.rs | 58 +- src/neon/fast_gaussian.rs | 22 +- src/neon/fast_gaussian_f16.rs | 14 +- src/neon/fast_gaussian_f32.rs | 18 +- src/neon/fast_gaussian_next.rs | 18 +- src/neon/fast_gaussian_next_f16.rs | 12 +- src/neon/fast_gaussian_next_f32.rs | 80 +- src/neon/fast_gaussian_next_q0_31.rs | 32 +- src/neon/fast_gaussian_next_u16.rs | 10 +- src/neon/fast_gaussian_next_u16_q0_31.rs | 12 +- src/neon/fast_gaussian_q0_31.rs | 52 +- src/neon/fast_gaussian_u16.rs | 6 +- src/neon/utils.rs | 785 +++++---- src/sobel.rs | 4 +- src/sse/fast_gaussian.rs | 816 ++++----- src/sse/fast_gaussian_f16.rs | 193 ++- src/sse/fast_gaussian_f32.rs | 627 +++---- src/sse/fast_gaussian_next.rs | 1046 +++++------ src/sse/fast_gaussian_next_f16.rs | 273 ++- src/sse/fast_gaussian_next_f32.rs | 744 ++++---- src/sse/fast_gaussian_next_u16.rs | 1038 +++++------ src/sse/fast_gaussian_u16.rs | 804 ++++----- src/sse/packing.rs | 254 +-- src/sse/utils.rs | 558 +++--- src/sse/v_load_store.rs | 203 ++- src/stack_blur_image.rs | 4 +- src/stackblur/avx/horizontal.rs | 663 +++---- src/stackblur/avx/horizontal_f32.rs | 420 ++--- src/stackblur/avx/vertical.rs | 938 +++++----- src/stackblur/avx/vertical_f32.rs | 4 +- src/stackblur/horizontal.rs | 11 +- src/stackblur/neon/horizontal.rs | 4 +- src/stackblur/neon/horizontal_f16.rs | 64 +- src/stackblur/neon/horizontal_f32.rs | 4 +- src/stackblur/neon/horizontal_q0_31.rs | 6 +- src/stackblur/neon/vertical.rs | 4 +- src/stackblur/neon/vertical_f16.rs | 64 +- src/stackblur/neon/vertical_f32.rs | 9 +- src/stackblur/neon/vertical_q0_31.rs | 4 +- src/stackblur/sse/horizontal.rs | 389 ++--- src/stackblur/sse/horizontal_f16.rs | 68 +- src/stackblur/sse/horizontal_f32.rs | 148 +- src/stackblur/sse/vertical.rs | 513 +++--- src/stackblur/sse/vertical_f16.rs | 64 +- src/stackblur/stack_blur.rs | 4 +- src/stackblur/stack_blur_f16.rs | 27 +- src/stackblur/wasm/horizontal.rs | 218 +-- src/stackblur/wasm/mod.rs | 4 +- src/stackblur/wasm/vertical.rs | 241 ++- src/unsafe_slice.rs | 4 +- src/util.rs | 83 + src/wasm32/fast_gaussian.rs | 262 ++- src/wasm32/fast_gaussian_next.rs | 346 ++-- src/wasm32/mod.rs | 4 +- src/wasm32/utils.rs | 90 +- 234 files changed, 22405 insertions(+), 21579 deletions(-) create mode 100644 .github/workflows/mirroring.yaml diff --git a/.github/workflows/build_push.yml b/.github/workflows/build_push.yml index 300483e9..d9c243c2 100644 --- a/.github/workflows/build_push.yml +++ b/.github/workflows/build_push.yml @@ -23,7 +23,7 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: actions-rust-lang/setup-rust-toolchain@v1 - run: rustup target add aarch64-unknown-linux-gnu x86_64-unknown-linux-gnu i686-unknown-linux-gnu powerpc-unknown-linux-gnu armv7-unknown-linux-gnueabi - run: RUSTFLAGS="-C target-feature=+neon" cargo build --target aarch64-unknown-linux-gnu --features image @@ -43,7 +43,7 @@ jobs: os: [ ubuntu-latest, macos-latest ] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable - run: rustup component add clippy - run: cargo clippy --all-features -- -D warnings @@ -52,7 +52,7 @@ jobs: name: Clippy WASM runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@stable - run: rustup target add wasm32-unknown-unknown - run: rustup component add clippy @@ -65,7 +65,7 @@ jobs: matrix: feature: [ neon, rdm, "" ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo test --features "${{ matrix.feature }}" --features fft --no-default-features @@ -76,7 +76,7 @@ jobs: feature: [ sse, avx, "" ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo test --features "${{ matrix.feature }}" --features fft --no-default-features @@ -87,7 +87,7 @@ jobs: matrix: feature: [ neon, rdm, "" ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo install cargo-fuzz - run: cargo fuzz run box --features "${{ matrix.feature }}" -- -max_total_time=14 @@ -101,7 +101,7 @@ jobs: feature: [ avx, sse ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo install cargo-fuzz - run: cargo fuzz run box --features "${{ matrix.feature }}" -- -max_total_time=14 @@ -115,7 +115,7 @@ jobs: matrix: feature: [ neon, rdm ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo install cargo-fuzz - run: cargo fuzz run gauss --features ${{ matrix.feature }} -- -max_total_time=17 @@ -130,7 +130,7 @@ jobs: name: Fuzzing Filter 1D/2D ARM FCMA runs-on: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo install cargo-fuzz - run: cargo fuzz run filter_complex --features neon,nightly_fcma -- -max_total_time=17 @@ -143,7 +143,7 @@ jobs: feature: [ "", sse, avx ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo install cargo-fuzz - run: cargo fuzz run gauss --features "${{ matrix.feature }}" -- -max_total_time=17 @@ -161,7 +161,7 @@ jobs: matrix: feature: [ neon, rdm ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo install cargo-fuzz - run: cargo fuzz run stack_blur --features ${{ matrix.feature }} -- -max_total_time=17 @@ -175,7 +175,7 @@ jobs: feature: [ sse, avx, "" ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo install cargo-fuzz - run: cargo fuzz run stack_blur --features "${{ matrix.feature }}" -- -max_total_time=17 @@ -189,7 +189,7 @@ jobs: matrix: feature: [ neon, rdm ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo install cargo-fuzz - run: cargo fuzz run fast_gaussian --features ${{ matrix.feature }} -- -max_total_time=14 @@ -206,7 +206,7 @@ jobs: feature: [ sse, avx, "" ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: dtolnay/rust-toolchain@nightly - run: cargo install cargo-fuzz - run: cargo fuzz run fast_gaussian --features "${{ matrix.feature }}" -- -max_total_time=14 diff --git a/.github/workflows/mirroring.yaml b/.github/workflows/mirroring.yaml new file mode 100644 index 00000000..f1d0718f --- /dev/null +++ b/.github/workflows/mirroring.yaml @@ -0,0 +1,25 @@ +name: Mirror to GitLab and Codeberg + +on: + push: + branches: [ 'master' ] + tags: + - '*' + +jobs: + mirror: + runs-on: ubuntu-latest + environment: mirroring + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Mirror to GitLab + run: | + git remote add gitlab https://oauth2:${{ secrets.GITLAB_TOKEN }}@gitlab.com/awxkee/libblur + git remote add codeberg https://awxkee:${{ secrets.CODEBERG_TOKEN }}@codeberg.org/awxkee/libblur.git + git push gitlab --all + git push gitlab --tags + git push codeberg --all + git push codeberg --tags \ No newline at end of file diff --git a/.github/workflows/publish_release.yml b/.github/workflows/publish_release.yml index 6286f694..b9d8fd1a 100644 --- a/.github/workflows/publish_release.yml +++ b/.github/workflows/publish_release.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest environment: Cargo steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: actions-rust-lang/setup-rust-toolchain@v1 - run: rustup target add aarch64-unknown-linux-gnu x86_64-unknown-linux-gnu i686-unknown-linux-gnu powerpc-unknown-linux-gnu armv7-unknown-linux-gnueabi - run: RUSTFLAGS="-C target-feature=+neon" cargo build --target aarch64-unknown-linux-gnu --features image diff --git a/Cargo.toml b/Cargo.toml index 4b1db0f2..68b5e01e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,8 @@ workspace = { members = ["app/accelerate", "app"], exclude = ["fuzz"] } [package] name = "libblur" -version = "0.23.3" -edition = "2021" +version = "0.24.0" +edition = "2024" description = "Fast image blurring in pure Rust" readme = "./README.md" keywords = ["blur", "gauss_blur", "image_blur", "fast_blur", "stack_blur"] @@ -15,7 +15,7 @@ homepage = "https://github.com/awxkee/libblur" repository = "https://github.com/awxkee/libblur.git" exclude = ["*.jpg", "./assets/*", "*.png"] include = ["/src/", "/README.md", "/LICENSE.md", "/LICENSE-APACHE.md"] -rust-version = "1.82.0" +rust-version = "1.89.0" [lib] name = "libblur" @@ -36,7 +36,7 @@ fft = ["dep:zaft"] # Enables NEON RDM intrinsics rdm = ["neon"] # Enables AVX-512. Requires nightly on rust compiler before 1.89. -nightly_avx512 = [] +avx512 = [] # Enables NEON fcma, used in FFT and filter 1d complex. Requires nightly. nightly_fcma = ["zaft?/fcma"] # Enables core::f16 support. Requires nightly. diff --git a/app/Cargo.toml b/app/Cargo.toml index 0403847d..17423bd0 100644 --- a/app/Cargo.toml +++ b/app/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] half = "2.4.1" image = "0.25.5" -libblur = { path = "../", features = ["nightly_f16", "sse", "avx", "fft", "neon", "rdm", "nightly_avx512", "nightly_fcma", "image"], default-features = false } +libblur = { path = "../", features = ["nightly_f16", "sse", "avx", "fft", "neon", "rdm", "avx512", "nightly_fcma", "image"], default-features = false } accelerate = { path = "accelerate" } rayon = "1.10.0" num-complex = "0.4" diff --git a/app/benches/gauss/main.rs b/app/benches/gauss/main.rs index 6b0982c4..2717abad 100644 --- a/app/benches/gauss/main.rs +++ b/app/benches/gauss/main.rs @@ -52,7 +52,9 @@ pub fn criterion_benchmark(c: &mut Criterion) { FastBlurChannels::Channels4, ); - c.bench_function("RGBA gauss blur kernel clip: 3", |b| { + opencv::core::set_num_threads(1).unwrap(); + + c.bench_function("RGBA gauss blur kernel clip Single Thread: 3", |b| { let mut dst_bytes = BlurImageMut::alloc(img.width(), img.height(), FastBlurChannels::Channels4); b.iter(|| { @@ -61,7 +63,7 @@ pub fn criterion_benchmark(c: &mut Criterion) { &mut dst_bytes, GaussianBlurParams::new_from_kernel(3.), EdgeMode::Clamp.as_2d(), - ThreadingPolicy::Adaptive, + ThreadingPolicy::Single, ConvolutionMode::FixedPoint, ) .unwrap(); @@ -83,6 +85,40 @@ pub fn criterion_benchmark(c: &mut Criterion) { ) .unwrap(); + c.bench_function("OpenCV RGBA Gaussian Single Thread: 3", |b| { + b.iter(|| { + let mut dst = Mat::default(); + opencv::imgproc::gaussian_blur( + &src, + &mut dst, + Size::new(3, 3), + 3., + 3., + BORDER_DEFAULT, + AlgorithmHint::ALGO_HINT_ACCURATE, + ) + .unwrap(); + }) + }); + + opencv::core::set_num_threads(-1).unwrap(); + + c.bench_function("RGBA gauss blur kernel clip: 3", |b| { + let mut dst_bytes = + BlurImageMut::alloc(img.width(), img.height(), FastBlurChannels::Channels4); + b.iter(|| { + libblur::gaussian_blur( + &src_image, + &mut dst_bytes, + GaussianBlurParams::new_from_kernel(3.), + EdgeMode::Clamp.as_2d(), + ThreadingPolicy::Adaptive, + ConvolutionMode::FixedPoint, + ) + .unwrap(); + }) + }); + c.bench_function("OpenCV RGBA Gaussian: 3", |b| { b.iter(|| { let mut dst = Mat::default(); diff --git a/app/src/main.rs b/app/src/main.rs index eab01962..710637f8 100644 --- a/app/src/main.rs +++ b/app/src/main.rs @@ -128,7 +128,7 @@ fn main() { // .iter() // .map(|&x| (x as f32 * (1. / 255.))) // .collect::>(); - let mut cvt = BlurImage::borrow( + let mut cvt = BlurImageMut::borrow( &mut v_vec, dyn_image.width(), dyn_image.height(), @@ -148,7 +148,7 @@ fn main() { let gaussian_kernel = gaussian_kernel_2d(500., 500 / 2); // let gaussian_kernel = complex_gaussian_kernel(500., 0.75, 5.); - let mut dst_image = BlurImageMut::default(); //cvt.clone_as_mut(); + // let mut dst_image = BlurImageMut::default(); //cvt.clone_as_mut(); // gaussian_blur( // &cvt, @@ -160,17 +160,17 @@ fn main() { // ) // .unwrap(); // - gaussian_blur( - &cvt, - &mut dst_image, - GaussianBlurParams::new_from_sigma(31.), - EdgeMode::Reflect.as_2d(), - ThreadingPolicy::Single, - ConvolutionMode::Exact, - ) - .unwrap(); + // gaussian_blur( + // &cvt, + // &mut dst_image, + // GaussianBlurParams::new_from_kernel(3.), + // EdgeMode::Reflect.as_2d(), + // ThreadingPolicy::Single, + // ConvolutionMode::FixedPoint, + // ) + // .unwrap(); - // stack_blur_f32(&mut cvt, AnisotropicRadius::new(8), ThreadingPolicy::Single).unwrap(); + stack_blur(&mut cvt, AnisotropicRadius::new(8), ThreadingPolicy::Single).unwrap(); // filter_2d_rgb_fft::( // &cvt, @@ -238,7 +238,7 @@ fn main() { // dst_image = vzd.gamma8(TransferFunction::Srgb, false).unwrap(); // - dst_bytes = dst_image + dst_bytes = cvt .data .borrow_mut() .iter() diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 37602aa9..4736ee3e 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -18,7 +18,7 @@ neon = ["libblur/neon"] avx = ["libblur/avx"] sse = ["libblur/sse"] rdm = ["libblur/rdm", "neon"] -avx512 = ["libblur/nightly_avx512"] +avx512 = ["libblur/avx512"] nightly_fcma = ["libblur/nightly_fcma"] [[bin]] diff --git a/src/adaptive_blur.rs b/src/adaptive_blur.rs index 150e0451..7ff496d9 100644 --- a/src/adaptive_blur.rs +++ b/src/adaptive_blur.rs @@ -29,9 +29,9 @@ #![allow(clippy::manual_clamp)] use crate::{ - filter_2d, filter_2d_fft, gaussian_blur, stack_blur, AnisotropicRadius, BlurError, BlurImage, - BlurImageMut, ConvolutionMode, EdgeMode, EdgeMode2D, FastBlurChannels, GaussianBlurParams, - KernelShape, Scalar, ThreadingPolicy, TransferFunction, + AnisotropicRadius, BlurError, BlurImage, BlurImageMut, ConvolutionMode, EdgeMode, EdgeMode2D, + FastBlurChannels, GaussianBlurParams, KernelShape, Scalar, ThreadingPolicy, TransferFunction, + filter_2d, filter_2d_fft, gaussian_blur, stack_blur, }; use num_traits::AsPrimitive; use std::fmt::Debug; @@ -123,7 +123,7 @@ impl Blur for u8 { channels: FastBlurChannels, ) { let mut rad = (radius / 2).saturating_sub(1).max(1); - if rad % 2 == 0 { + if rad.is_multiple_of(2) { rad += 1; } let src = BlurImage::borrow(image, width as u32, height as u32, channels); @@ -249,7 +249,7 @@ impl Edges for u8 { let mut dst = vec![0u8; width * height]; let mut radius = radius; - if radius % 2 == 0 { + if radius.is_multiple_of(2) { radius += 1; } diff --git a/src/avx/fast_gaussian.rs b/src/avx/fast_gaussian.rs index bc3c21fa..d9840295 100644 --- a/src/avx/fast_gaussian.rs +++ b/src/avx/fast_gaussian.rs @@ -25,16 +25,16 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::avx::fast_gaussian_next::AvxSseI32x8; use crate::edge_mode::clamp_edge; use crate::sse::store_u8_u32; use crate::sse::utils::load_u8_s32_fast; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::x86_64::*; -pub(crate) fn fg_horizontal_pass_sse_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_horizontal_pass_sse_u8( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -44,15 +44,14 @@ pub(crate) fn fg_horizontal_pass_sse_u8( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); fg_horizontal_pass_avx_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } #[target_feature(enable = "avx2")] -unsafe fn fg_horizontal_pass_avx_def( +fn fg_horizontal_pass_avx_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -68,7 +67,7 @@ unsafe fn fg_horizontal_pass_avx_def( } #[inline(always)] -unsafe fn fg_horizontal_pass_avx_u8_impl( +fn fg_horizontal_pass_avx_u8_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -78,7 +77,7 @@ unsafe fn fg_horizontal_pass_avx_u8_impl( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseI32x8::default(); 1024 * 3]); + let mut full_buffer = [AvxSseI32x8::default(); 1024 * 3]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, buffer2) = rem.split_at_mut(1024); @@ -88,225 +87,233 @@ unsafe fn fg_horizontal_pass_avx_u8_impl( let radius_64 = radius as i64; let width_wide = width as i64; - let v_weight = _mm256_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let mut yy = start; - - while yy + 6 < height.min(end) { - let mut diffs0 = _mm256_setzero_si256(); - let mut diffs1 = _mm256_setzero_si256(); - let mut diffs2 = _mm256_setzero_si256(); - - let mut summs0 = _mm256_set1_epi32(initial_sum); - let mut summs1 = _mm256_set1_epi32(initial_sum); - let mut summs2 = _mm256_set1_epi32(initial_sum); - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; - let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; - let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; - let current_y4 = ((yy as i64 + 4) * (stride as i64)) as usize; - let current_y5 = ((yy as i64 + 5) * (stride as i64)) as usize; - - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = (x as u32 * CN as u32) as usize; - - let ps01 = _mm256_cvtepi32_ps(summs0); - let ps23 = _mm256_cvtepi32_ps(summs1); - let ps45 = _mm256_cvtepi32_ps(summs1); - - let r01 = _mm256_mul_ps(ps01, v_weight); - let r23 = _mm256_mul_ps(ps23, v_weight); - let r45 = _mm256_mul_ps(ps45, v_weight); - - let e01 = _mm256_cvtps_epi32(r01); - let e23 = _mm256_cvtps_epi32(r23); - let e45 = _mm256_cvtps_epi32(r45); - - let prepared_px0 = _mm256_castsi256_si128(e01); - let prepared_px1 = _mm256_extracti128_si256::<1>(e01); - let prepared_px2 = _mm256_castsi256_si128(e23); - let prepared_px3 = _mm256_extracti128_si256::<1>(e23); - let prepared_px4 = _mm256_castsi256_si128(e45); - let prepared_px5 = _mm256_extracti128_si256::<1>(e45); - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y0 + current_px); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y1 + current_px); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y2 + current_px); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y3 + current_px); - let dst_ptr4 = (bytes.slice.as_ptr() as *mut u8).add(current_y4 + current_px); - let dst_ptr5 = (bytes.slice.as_ptr() as *mut u8).add(current_y5 + current_px); - - store_u8_u32::(dst_ptr0, prepared_px0); - store_u8_u32::(dst_ptr1, prepared_px1); - store_u8_u32::(dst_ptr2, prepared_px2); - store_u8_u32::(dst_ptr3, prepared_px3); - store_u8_u32::(dst_ptr4, prepared_px4); - store_u8_u32::(dst_ptr5, prepared_px5); - - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let mut d_stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - let mut d_stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - let mut d_stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - - d_stored0 = _mm256_slli_epi32::<1>(d_stored0); - d_stored1 = _mm256_slli_epi32::<1>(d_stored1); - d_stored2 = _mm256_slli_epi32::<1>(d_stored2); - - let a_stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + unsafe { + let v_weight = _mm256_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let mut yy = start; + + while yy + 6 <= height.min(end) { + let mut diffs0 = _mm256_setzero_si256(); + let mut diffs1 = _mm256_setzero_si256(); + let mut diffs2 = _mm256_setzero_si256(); + + let mut summs0 = _mm256_set1_epi32(initial_sum); + let mut summs1 = _mm256_set1_epi32(initial_sum); + let mut summs2 = _mm256_set1_epi32(initial_sum); + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; + let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; + let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + let current_y4 = ((yy as i64 + 4) * (stride as i64)) as usize; + let current_y5 = ((yy as i64 + 5) * (stride as i64)) as usize; + + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = (x as u32 * CN as u32) as usize; + + let ps01 = _mm256_cvtepi32_ps(summs0); + let ps23 = _mm256_cvtepi32_ps(summs1); + let ps45 = _mm256_cvtepi32_ps(summs1); + + let r01 = _mm256_mul_ps(ps01, v_weight); + let r23 = _mm256_mul_ps(ps23, v_weight); + let r45 = _mm256_mul_ps(ps45, v_weight); + + let e01 = _mm256_cvtps_epi32(r01); + let e23 = _mm256_cvtps_epi32(r23); + let e45 = _mm256_cvtps_epi32(r45); + + let prepared_px0 = _mm256_castsi256_si128(e01); + let prepared_px1 = _mm256_extracti128_si256::<1>(e01); + let prepared_px2 = _mm256_castsi256_si128(e23); + let prepared_px3 = _mm256_extracti128_si256::<1>(e23); + let prepared_px4 = _mm256_castsi256_si128(e45); + let prepared_px5 = _mm256_extracti128_si256::<1>(e45); + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y0 + current_px); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y1 + current_px); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y2 + current_px); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y3 + current_px); + let dst_ptr4 = (bytes.slice.as_ptr() as *mut u8).add(current_y4 + current_px); + let dst_ptr5 = (bytes.slice.as_ptr() as *mut u8).add(current_y5 + current_px); + + store_u8_u32::(dst_ptr0, prepared_px0); + store_u8_u32::(dst_ptr1, prepared_px1); + store_u8_u32::(dst_ptr2, prepared_px2); + store_u8_u32::(dst_ptr3, prepared_px3); + store_u8_u32::(dst_ptr4, prepared_px4); + store_u8_u32::(dst_ptr5, prepared_px5); + + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let mut d_stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + + d_stored0 = _mm256_slli_epi32::<1>(d_stored0); + d_stored1 = _mm256_slli_epi32::<1>(d_stored1); + d_stored2 = _mm256_slli_epi32::<1>(d_stored2); + + let a_stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm256_add_epi32(diffs0, _mm256_sub_epi32(a_stored0, d_stored0)); + diffs1 = _mm256_add_epi32(diffs1, _mm256_sub_epi32(a_stored1, d_stored1)); + diffs2 = _mm256_add_epi32(diffs2, _mm256_sub_epi32(a_stored2, d_stored2)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let mut stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let mut stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let mut stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + stored0 = _mm256_slli_epi32::<1>(stored0); + stored1 = _mm256_slli_epi32::<1>(stored1); + stored2 = _mm256_slli_epi32::<1>(stored2); + + diffs0 = _mm256_sub_epi32(diffs0, stored0); + diffs1 = _mm256_sub_epi32(diffs1, stored1); + diffs2 = _mm256_sub_epi32(diffs2, stored2); + } + + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u8; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u8; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u8; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u8; + let s_ptr4 = bytes.slice.as_ptr().add(current_y4 + next_row_px) as *mut u8; + let s_ptr5 = bytes.slice.as_ptr().add(current_y5 + next_row_px) as *mut u8; + + let pixel_color0 = load_u8_s32_fast::(s_ptr0); + let pixel_color1 = load_u8_s32_fast::(s_ptr1); + let pixel_color2 = load_u8_s32_fast::(s_ptr2); + let pixel_color3 = load_u8_s32_fast::(s_ptr3); + let pixel_color4 = load_u8_s32_fast::(s_ptr4); + let pixel_color5 = load_u8_s32_fast::(s_ptr5); + + let arr_index = ((x + radius_64) & 1023) as usize; + + let px01 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color0), + pixel_color1, ); - let a_stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px23 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color2), + pixel_color3, ); - let a_stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px45 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color4), + pixel_color5, ); - diffs0 = _mm256_add_epi32(diffs0, _mm256_sub_epi32(a_stored0, d_stored0)); - diffs1 = _mm256_add_epi32(diffs1, _mm256_sub_epi32(a_stored1, d_stored1)); - diffs2 = _mm256_add_epi32(diffs2, _mm256_sub_epi32(a_stored2, d_stored2)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let mut stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + _mm256_store_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px01, ); - let mut stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + _mm256_store_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px23, ); - let mut stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + _mm256_store_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px45, ); - stored0 = _mm256_slli_epi32::<1>(stored0); - stored1 = _mm256_slli_epi32::<1>(stored1); - stored2 = _mm256_slli_epi32::<1>(stored2); + diffs0 = _mm256_add_epi32(diffs0, px01); + diffs1 = _mm256_add_epi32(diffs1, px23); + diffs2 = _mm256_add_epi32(diffs2, px45); - diffs0 = _mm256_sub_epi32(diffs0, stored0); - diffs1 = _mm256_sub_epi32(diffs1, stored1); - diffs2 = _mm256_sub_epi32(diffs2, stored2); + summs0 = _mm256_add_epi32(summs0, diffs0); + summs1 = _mm256_add_epi32(summs1, diffs1); + summs2 = _mm256_add_epi32(summs2, diffs2); } - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u8; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u8; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u8; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u8; - let s_ptr4 = bytes.slice.as_ptr().add(current_y4 + next_row_px) as *mut u8; - let s_ptr5 = bytes.slice.as_ptr().add(current_y5 + next_row_px) as *mut u8; - - let pixel_color0 = load_u8_s32_fast::(s_ptr0); - let pixel_color1 = load_u8_s32_fast::(s_ptr1); - let pixel_color2 = load_u8_s32_fast::(s_ptr2); - let pixel_color3 = load_u8_s32_fast::(s_ptr3); - let pixel_color4 = load_u8_s32_fast::(s_ptr4); - let pixel_color5 = load_u8_s32_fast::(s_ptr5); - - let arr_index = ((x + radius_64) & 1023) as usize; - - let px01 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color0), pixel_color1); - let px23 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color2), pixel_color3); - let px45 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color4), pixel_color5); - - _mm256_store_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px01, - ); - _mm256_store_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px23, - ); - _mm256_store_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px45, - ); - - diffs0 = _mm256_add_epi32(diffs0, px01); - diffs1 = _mm256_add_epi32(diffs1, px23); - diffs2 = _mm256_add_epi32(diffs2, px45); - - summs0 = _mm256_add_epi32(summs0, diffs0); - summs1 = _mm256_add_epi32(summs1, diffs1); - summs2 = _mm256_add_epi32(summs2, diffs2); + yy += 6; } - yy += 6; - } + for y in yy..height.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut summs = _mm_set1_epi32(initial_sum); - for y in yy..height.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut summs = _mm_set1_epi32(initial_sum); + let current_y = ((y as i64) * (stride as i64)) as usize; - let current_y = ((y as i64) * (stride as i64)) as usize; + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = (x as u32 * CN as u32) as usize; - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = (x as u32 * CN as u32) as usize; + let pixel_f32 = + _mm_mul_ps(_mm_cvtepi32_ps(summs), _mm256_castps256_ps128(v_weight)); + let pixel_u32 = _mm_cvtps_epi32(pixel_f32); - let pixel_f32 = - _mm_mul_ps(_mm_cvtepi32_ps(summs), _mm256_castps256_ps128(v_weight)); - let pixel_u32 = _mm_cvtps_epi32(pixel_f32); + let bytes_offset = current_y + current_px; - let bytes_offset = current_y + current_px; + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + store_u8_u32::(dst_ptr, pixel_u32); - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - store_u8_u32::(dst_ptr, pixel_u32); + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let d_buf_ptr = buffer0.as_mut_ptr().add(d_arr_index) as *const i32; + let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); + d_stored = _mm_slli_epi32::<1>(d_stored); - let d_buf_ptr = buffer0.as_mut_ptr().add(d_arr_index) as *const i32; - let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); - d_stored = _mm_slli_epi32::<1>(d_stored); + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let a_stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let a_stored = _mm_load_si128(buf_ptr as *const __m128i); + diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let mut stored = _mm_load_si128(buf_ptr as *const __m128i); + stored = _mm_slli_epi32::<1>(stored); + diffs = _mm_sub_epi32(diffs, stored); + } - diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let mut stored = _mm_load_si128(buf_ptr as *const __m128i); - stored = _mm_slli_epi32::<1>(stored); - diffs = _mm_sub_epi32(diffs, stored); - } + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; + let pixel_color = load_u8_s32_fast::(s_ptr); - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; - let pixel_color = load_u8_s32_fast::(s_ptr); - - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); - summs = _mm_add_epi32(summs, diffs); + diffs = _mm_add_epi32(diffs, pixel_color); + summs = _mm_add_epi32(summs, diffs); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } -pub(crate) fn fg_vertical_pass_avx_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_vertical_pass_avx_u8( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -316,15 +323,14 @@ pub(crate) fn fg_vertical_pass_avx_u8( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); fg_vertical_pass_avx_u8_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } #[target_feature(enable = "avx2")] -unsafe fn fg_vertical_pass_avx_u8_def( +fn fg_vertical_pass_avx_u8_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -334,7 +340,7 @@ unsafe fn fg_vertical_pass_avx_u8_def( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseI32x8::default(); 1024 * 3]); + let mut full_buffer = [AvxSseI32x8::default(); 1024 * 3]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, buffer2) = rem.split_at_mut(1024); @@ -349,219 +355,227 @@ unsafe fn fg_vertical_pass_avx_u8_def( let mut xx = start; - while xx + 6 < width.min(end) { - let mut diffs0 = _mm256_setzero_si256(); - let mut diffs1 = _mm256_setzero_si256(); - let mut diffs2 = _mm256_setzero_si256(); - - let mut summs0 = _mm256_set1_epi32(initial_sum); - let mut summs1 = _mm256_set1_epi32(initial_sum); - let mut summs2 = _mm256_set1_epi32(initial_sum); - - let start_y = 0 - 2 * radius as i64; - - let current_px0 = (xx * CN as u32) as usize; - let current_px1 = ((xx + 1) * CN as u32) as usize; - let current_px2 = ((xx + 2) * CN as u32) as usize; - let current_px3 = ((xx + 3) * CN as u32) as usize; - let current_px4 = ((xx + 4) * CN as u32) as usize; - let current_px5 = ((xx + 5) * CN as u32) as usize; - - for y in start_y..height_wide { - if y >= 0 { - let ps01 = _mm256_cvtepi32_ps(summs0); - let ps23 = _mm256_cvtepi32_ps(summs1); - let ps45 = _mm256_cvtepi32_ps(summs2); - - let r01 = _mm256_mul_ps(ps01, v_weight); - let r23 = _mm256_mul_ps(ps23, v_weight); - let r45 = _mm256_mul_ps(ps45, v_weight); - - let e01 = _mm256_cvtps_epi32(r01); - let e23 = _mm256_cvtps_epi32(r23); - let e45 = _mm256_cvtps_epi32(r45); - - let prepared_px0 = _mm256_castsi256_si128(e01); - let prepared_px1 = _mm256_extracti128_si256::<1>(e01); - let prepared_px2 = _mm256_castsi256_si128(e23); - let prepared_px3 = _mm256_extracti128_si256::<1>(e23); - let prepared_px4 = _mm256_castsi256_si128(e45); - let prepared_px5 = _mm256_extracti128_si256::<1>(e45); - - let current_y = (y * (stride as i64)) as usize; - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px0); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px1); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px2); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px3); - let dst_ptr4 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px4); - let dst_ptr5 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px5); - - store_u8_u32::(dst_ptr0, prepared_px0); - store_u8_u32::(dst_ptr1, prepared_px1); - store_u8_u32::(dst_ptr2, prepared_px2); - store_u8_u32::(dst_ptr3, prepared_px3); - store_u8_u32::(dst_ptr4, prepared_px4); - store_u8_u32::(dst_ptr5, prepared_px5); - - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let mut d_stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - let mut d_stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - let mut d_stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - - d_stored0 = _mm256_slli_epi32::<1>(d_stored0); - d_stored1 = _mm256_slli_epi32::<1>(d_stored1); - d_stored2 = _mm256_slli_epi32::<1>(d_stored2); - - let a_stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + unsafe { + while xx + 6 <= width.min(end) { + let mut diffs0 = _mm256_setzero_si256(); + let mut diffs1 = _mm256_setzero_si256(); + let mut diffs2 = _mm256_setzero_si256(); + + let mut summs0 = _mm256_set1_epi32(initial_sum); + let mut summs1 = _mm256_set1_epi32(initial_sum); + let mut summs2 = _mm256_set1_epi32(initial_sum); + + let start_y = 0 - 2 * radius as i64; + + let current_px0 = (xx * CN as u32) as usize; + let current_px1 = ((xx + 1) * CN as u32) as usize; + let current_px2 = ((xx + 2) * CN as u32) as usize; + let current_px3 = ((xx + 3) * CN as u32) as usize; + let current_px4 = ((xx + 4) * CN as u32) as usize; + let current_px5 = ((xx + 5) * CN as u32) as usize; + + for y in start_y..height_wide { + if y >= 0 { + let ps01 = _mm256_cvtepi32_ps(summs0); + let ps23 = _mm256_cvtepi32_ps(summs1); + let ps45 = _mm256_cvtepi32_ps(summs2); + + let r01 = _mm256_mul_ps(ps01, v_weight); + let r23 = _mm256_mul_ps(ps23, v_weight); + let r45 = _mm256_mul_ps(ps45, v_weight); + + let e01 = _mm256_cvtps_epi32(r01); + let e23 = _mm256_cvtps_epi32(r23); + let e45 = _mm256_cvtps_epi32(r45); + + let prepared_px0 = _mm256_castsi256_si128(e01); + let prepared_px1 = _mm256_extracti128_si256::<1>(e01); + let prepared_px2 = _mm256_castsi256_si128(e23); + let prepared_px3 = _mm256_extracti128_si256::<1>(e23); + let prepared_px4 = _mm256_castsi256_si128(e45); + let prepared_px5 = _mm256_extracti128_si256::<1>(e45); + + let current_y = (y * (stride as i64)) as usize; + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px0); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px1); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px2); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px3); + let dst_ptr4 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px4); + let dst_ptr5 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px5); + + store_u8_u32::(dst_ptr0, prepared_px0); + store_u8_u32::(dst_ptr1, prepared_px1); + store_u8_u32::(dst_ptr2, prepared_px2); + store_u8_u32::(dst_ptr3, prepared_px3); + store_u8_u32::(dst_ptr4, prepared_px4); + store_u8_u32::(dst_ptr5, prepared_px5); + + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let mut d_stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + + d_stored0 = _mm256_slli_epi32::<1>(d_stored0); + d_stored1 = _mm256_slli_epi32::<1>(d_stored1); + d_stored2 = _mm256_slli_epi32::<1>(d_stored2); + + let a_stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm256_add_epi32(diffs0, _mm256_sub_epi32(a_stored0, d_stored0)); + diffs1 = _mm256_add_epi32(diffs1, _mm256_sub_epi32(a_stored1, d_stored1)); + diffs2 = _mm256_add_epi32(diffs2, _mm256_sub_epi32(a_stored2, d_stored2)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + + let mut stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + let mut stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + let mut stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + + stored0 = _mm256_slli_epi32::<1>(stored0); + stored1 = _mm256_slli_epi32::<1>(stored1); + stored2 = _mm256_slli_epi32::<1>(stored2); + + diffs0 = _mm256_sub_epi32(diffs0, stored0); + diffs1 = _mm256_sub_epi32(diffs1, stored1); + diffs2 = _mm256_sub_epi32(diffs2, stored2); + } + + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u8; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u8; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u8; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u8; + let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut u8; + let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut u8; + + let pixel_color0 = load_u8_s32_fast::(s_ptr0); + let pixel_color1 = load_u8_s32_fast::(s_ptr1); + let pixel_color2 = load_u8_s32_fast::(s_ptr2); + let pixel_color3 = load_u8_s32_fast::(s_ptr3); + let pixel_color4 = load_u8_s32_fast::(s_ptr4); + let pixel_color5 = load_u8_s32_fast::(s_ptr5); + + let arr_index = ((y + radius_64) & 1023) as usize; + + let px01 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color0), + pixel_color1, ); - let a_stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px23 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color2), + pixel_color3, ); - let a_stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px45 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color4), + pixel_color5, ); - diffs0 = _mm256_add_epi32(diffs0, _mm256_sub_epi32(a_stored0, d_stored0)); - diffs1 = _mm256_add_epi32(diffs1, _mm256_sub_epi32(a_stored1, d_stored1)); - diffs2 = _mm256_add_epi32(diffs2, _mm256_sub_epi32(a_stored2, d_stored2)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; + diffs0 = _mm256_add_epi32(diffs0, px01); + diffs1 = _mm256_add_epi32(diffs1, px23); + diffs2 = _mm256_add_epi32(diffs2, px45); - let mut stored0 = _mm256_load_si256( + _mm256_store_si256( buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px01, ); - let mut stored1 = _mm256_load_si256( + _mm256_store_si256( buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px23, ); - let mut stored2 = _mm256_load_si256( + _mm256_store_si256( buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px45, ); - stored0 = _mm256_slli_epi32::<1>(stored0); - stored1 = _mm256_slli_epi32::<1>(stored1); - stored2 = _mm256_slli_epi32::<1>(stored2); - - diffs0 = _mm256_sub_epi32(diffs0, stored0); - diffs1 = _mm256_sub_epi32(diffs1, stored1); - diffs2 = _mm256_sub_epi32(diffs2, stored2); + summs0 = _mm256_add_epi32(summs0, diffs0); + summs1 = _mm256_add_epi32(summs1, diffs1); + summs2 = _mm256_add_epi32(summs2, diffs2); } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u8; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u8; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u8; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u8; - let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut u8; - let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut u8; - - let pixel_color0 = load_u8_s32_fast::(s_ptr0); - let pixel_color1 = load_u8_s32_fast::(s_ptr1); - let pixel_color2 = load_u8_s32_fast::(s_ptr2); - let pixel_color3 = load_u8_s32_fast::(s_ptr3); - let pixel_color4 = load_u8_s32_fast::(s_ptr4); - let pixel_color5 = load_u8_s32_fast::(s_ptr5); - - let arr_index = ((y + radius_64) & 1023) as usize; - - let px01 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color0), pixel_color1); - let px23 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color2), pixel_color3); - let px45 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color4), pixel_color5); - - diffs0 = _mm256_add_epi32(diffs0, px01); - diffs1 = _mm256_add_epi32(diffs1, px23); - diffs2 = _mm256_add_epi32(diffs2, px45); - - _mm256_store_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px01, - ); - _mm256_store_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px23, - ); - _mm256_store_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px45, - ); - - summs0 = _mm256_add_epi32(summs0, diffs0); - summs1 = _mm256_add_epi32(summs1, diffs1); - summs2 = _mm256_add_epi32(summs2, diffs2); + xx += 6; } - xx += 6; - } - - for x in xx..width.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut summs = _mm_set1_epi32(initial_sum); + for x in xx..width.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut summs = _mm_set1_epi32(initial_sum); - let current_px = (x * CN as u32) as usize; + let current_px = (x * CN as u32) as usize; - let start_y = 0 - 2 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let pixel_f32 = - _mm_mul_ps(_mm_cvtepi32_ps(summs), _mm256_castps256_ps128(v_weight)); + let start_y = 0 - 2 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let pixel_f32 = + _mm_mul_ps(_mm_cvtepi32_ps(summs), _mm256_castps256_ps128(v_weight)); - let current_y = (y * (stride as i64)) as usize; + let current_y = (y * (stride as i64)) as usize; - let pixel_u32 = _mm_cvtps_epi32(pixel_f32); + let pixel_u32 = _mm_cvtps_epi32(pixel_f32); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - store_u8_u32::(dst_ptr, pixel_u32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + store_u8_u32::(dst_ptr, pixel_u32); - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let d_buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); - d_stored = _mm_slli_epi32::<1>(d_stored); + let d_buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); + d_stored = _mm_slli_epi32::<1>(d_stored); - let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; - let a_stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; + let a_stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let mut stored = _mm_load_si128(buf_ptr as *const __m128i); - stored = _mm_slli_epi32::<1>(stored); - diffs = _mm_sub_epi32(diffs, stored); - } + diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let mut stored = _mm_load_si128(buf_ptr as *const __m128i); + stored = _mm_slli_epi32::<1>(stored); + diffs = _mm_sub_epi32(diffs, stored); + } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; - let pixel_color = load_u8_s32_fast::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; + let pixel_color = load_u8_s32_fast::(s_ptr); - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); + diffs = _mm_add_epi32(diffs, pixel_color); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); - summs = _mm_add_epi32(summs, diffs); + summs = _mm_add_epi32(summs, diffs); + } } } } diff --git a/src/avx/fast_gaussian_f32.rs b/src/avx/fast_gaussian_f32.rs index f21c3c5e..a4303bef 100644 --- a/src/avx/fast_gaussian_f32.rs +++ b/src/avx/fast_gaussian_f32.rs @@ -25,19 +25,19 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::avx::fast_gaussian_next_f32::AvxSseF32x8; use crate::avx::utils::_mm256_opt_fnmlaf_ps; use crate::edge_mode::clamp_edge; use crate::sse::{_mm_opt_fnmlaf_ps, load_f32, store_f32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; -pub(crate) fn fg_horizontal_pass_avx_f32( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_horizontal_pass_avx_f32( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -47,21 +47,20 @@ pub(crate) fn fg_horizontal_pass_avx_f32( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undefined_slice); if std::arch::is_x86_feature_detected!("fma") { fg_horizontal_pass_avx_f32_fma::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } else { fg_horizontal_pass_avx_f32_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } } #[target_feature(enable = "avx2")] -unsafe fn fg_horizontal_pass_avx_f32_def( +fn fg_horizontal_pass_avx_f32_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -76,7 +75,7 @@ unsafe fn fg_horizontal_pass_avx_f32_def( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn fg_horizontal_pass_avx_f32_fma( +fn fg_horizontal_pass_avx_f32_fma( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -95,7 +94,7 @@ struct HorizontalExecutionUnit {} impl HorizontalExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, bytes: &UnsafeSlice, stride: u32, @@ -106,188 +105,205 @@ impl HorizontalExecutionUnit { end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseF32x8::default(); 1024 * 3]); - - let (bf0, rem) = full_buffer.split_at_mut(1024); - let (bf1, bf2) = rem.split_at_mut(1024); - - let radius_64 = radius as i64; - let width_wide = width as i64; - - let v_double = _mm256_set1_ps(2.); - let v_weight = _mm256_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let mut yy = start as usize; - - while yy + 6 < height.min(end) as usize { - let mut diffs0 = _mm256_setzero_ps(); - let mut diffs1 = _mm256_setzero_ps(); - let mut diffs2 = _mm256_setzero_ps(); - - let mut sums0 = _mm256_setzero_ps(); - let mut sums1 = _mm256_setzero_ps(); - let mut sums2 = _mm256_setzero_ps(); - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; - let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; - let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; - let current_y4 = ((yy as i64 + 4) * (stride as i64)) as usize; - let current_y5 = ((yy as i64 + 5) * (stride as i64)) as usize; - - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; - - let prepared_px0 = _mm256_mul_ps(sums0, v_weight); - let prepared_px1 = _mm256_mul_ps(sums1, v_weight); - let prepared_px2 = _mm256_mul_ps(sums2, v_weight); - - let dst_ptr0 = bytes.slice.as_ptr().add(current_y0 + current_px) as *mut f32; - let dst_ptr1 = bytes.slice.as_ptr().add(current_y1 + current_px) as *mut f32; - let dst_ptr2 = bytes.slice.as_ptr().add(current_y2 + current_px) as *mut f32; - let dst_ptr3 = bytes.slice.as_ptr().add(current_y3 + current_px) as *mut f32; - let dst_ptr4 = bytes.slice.as_ptr().add(current_y4 + current_px) as *mut f32; - let dst_ptr5 = bytes.slice.as_ptr().add(current_y5 + current_px) as *mut f32; - - store_f32::(dst_ptr0, _mm256_castps256_ps128(prepared_px0)); - store_f32::(dst_ptr1, _mm256_extractf128_ps::<1>(prepared_px0)); - store_f32::(dst_ptr2, _mm256_castps256_ps128(prepared_px1)); - store_f32::(dst_ptr3, _mm256_extractf128_ps::<1>(prepared_px1)); - store_f32::(dst_ptr4, _mm256_castps256_ps128(prepared_px2)); - store_f32::(dst_ptr5, _mm256_extractf128_ps::<1>(prepared_px2)); - - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let d_s0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_arr_index) as *mut f32); - let d_s1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_arr_index) as *mut f32); - let d_s2 = _mm256_load_ps(bf2.as_mut_ptr().add(d_arr_index) as *mut f32); - - let a_s0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let a_s1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let a_s2 = _mm256_load_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = - _mm256_add_ps(diffs0, _mm256_opt_fnmlaf_ps::(a_s0, d_s0, v_double)); - diffs1 = - _mm256_add_ps(diffs1, _mm256_opt_fnmlaf_ps::(a_s1, d_s1, v_double)); - diffs2 = - _mm256_add_ps(diffs2, _mm256_opt_fnmlaf_ps::(a_s2, d_s2, v_double)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let s0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let s1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let s2 = _mm256_load_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = _mm256_opt_fnmlaf_ps::(diffs0, s0, v_double); - diffs1 = _mm256_opt_fnmlaf_ps::(diffs1, s1, v_double); - diffs2 = _mm256_opt_fnmlaf_ps::(diffs2, s2, v_double); + unsafe { + let mut full_buffer = [AvxSseF32x8::default(); 1024 * 3]; + + let (bf0, rem) = full_buffer.split_at_mut(1024); + let (bf1, bf2) = rem.split_at_mut(1024); + + let radius_64 = radius as i64; + let width_wide = width as i64; + + let v_double = _mm256_set1_ps(2.); + let v_weight = _mm256_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let mut yy = start as usize; + + while yy + 6 <= height.min(end) as usize { + let mut diffs0 = _mm256_setzero_ps(); + let mut diffs1 = _mm256_setzero_ps(); + let mut diffs2 = _mm256_setzero_ps(); + + let mut sums0 = _mm256_setzero_ps(); + let mut sums1 = _mm256_setzero_ps(); + let mut sums2 = _mm256_setzero_ps(); + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; + let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; + let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + let current_y4 = ((yy as i64 + 4) * (stride as i64)) as usize; + let current_y5 = ((yy as i64 + 5) * (stride as i64)) as usize; + + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; + + let prepared_px0 = _mm256_mul_ps(sums0, v_weight); + let prepared_px1 = _mm256_mul_ps(sums1, v_weight); + let prepared_px2 = _mm256_mul_ps(sums2, v_weight); + + let dst_ptr0 = + bytes.slice.as_ptr().add(current_y0 + current_px) as *mut f32; + let dst_ptr1 = + bytes.slice.as_ptr().add(current_y1 + current_px) as *mut f32; + let dst_ptr2 = + bytes.slice.as_ptr().add(current_y2 + current_px) as *mut f32; + let dst_ptr3 = + bytes.slice.as_ptr().add(current_y3 + current_px) as *mut f32; + let dst_ptr4 = + bytes.slice.as_ptr().add(current_y4 + current_px) as *mut f32; + let dst_ptr5 = + bytes.slice.as_ptr().add(current_y5 + current_px) as *mut f32; + + store_f32::(dst_ptr0, _mm256_castps256_ps128(prepared_px0)); + store_f32::(dst_ptr1, _mm256_extractf128_ps::<1>(prepared_px0)); + store_f32::(dst_ptr2, _mm256_castps256_ps128(prepared_px1)); + store_f32::(dst_ptr3, _mm256_extractf128_ps::<1>(prepared_px1)); + store_f32::(dst_ptr4, _mm256_castps256_ps128(prepared_px2)); + store_f32::(dst_ptr5, _mm256_extractf128_ps::<1>(prepared_px2)); + + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let d_s0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_arr_index) as *mut f32); + let d_s1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_arr_index) as *mut f32); + let d_s2 = _mm256_load_ps(bf2.as_mut_ptr().add(d_arr_index) as *mut f32); + + let a_s0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let a_s1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let a_s2 = _mm256_load_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm256_add_ps( + diffs0, + _mm256_opt_fnmlaf_ps::(a_s0, d_s0, v_double), + ); + diffs1 = _mm256_add_ps( + diffs1, + _mm256_opt_fnmlaf_ps::(a_s1, d_s1, v_double), + ); + diffs2 = _mm256_add_ps( + diffs2, + _mm256_opt_fnmlaf_ps::(a_s2, d_s2, v_double), + ); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let s0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let s1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let s2 = _mm256_load_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm256_opt_fnmlaf_ps::(diffs0, s0, v_double); + diffs1 = _mm256_opt_fnmlaf_ps::(diffs1, s1, v_double); + diffs2 = _mm256_opt_fnmlaf_ps::(diffs2, s2, v_double); + } + + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut f32; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut f32; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut f32; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut f32; + let s_ptr4 = bytes.slice.as_ptr().add(current_y4 + next_row_px) as *mut f32; + let s_ptr5 = bytes.slice.as_ptr().add(current_y5 + next_row_px) as *mut f32; + + let px0 = load_f32::(s_ptr0); + let px1 = load_f32::(s_ptr1); + let px2 = load_f32::(s_ptr2); + let px3 = load_f32::(s_ptr3); + let px4 = load_f32::(s_ptr4); + let px5 = load_f32::(s_ptr5); + + let arr_index = ((x + radius_64) & 1023) as usize; + + let px01 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px0), px1); + let px23 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px2), px3); + let px45 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px4), px5); + + _mm256_store_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32, px01); + _mm256_store_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32, px23); + _mm256_store_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32, px45); + + diffs0 = _mm256_add_ps(diffs0, px01); + diffs1 = _mm256_add_ps(diffs1, px23); + diffs2 = _mm256_add_ps(diffs2, px45); + + sums0 = _mm256_add_ps(sums0, diffs0); + sums1 = _mm256_add_ps(sums1, diffs1); + sums2 = _mm256_add_ps(sums2, diffs2); } - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut f32; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut f32; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut f32; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut f32; - let s_ptr4 = bytes.slice.as_ptr().add(current_y4 + next_row_px) as *mut f32; - let s_ptr5 = bytes.slice.as_ptr().add(current_y5 + next_row_px) as *mut f32; - - let px0 = load_f32::(s_ptr0); - let px1 = load_f32::(s_ptr1); - let px2 = load_f32::(s_ptr2); - let px3 = load_f32::(s_ptr3); - let px4 = load_f32::(s_ptr4); - let px5 = load_f32::(s_ptr5); - - let arr_index = ((x + radius_64) & 1023) as usize; - - let px01 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px0), px1); - let px23 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px2), px3); - let px45 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px4), px5); - - _mm256_store_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32, px01); - _mm256_store_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32, px23); - _mm256_store_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32, px45); - - diffs0 = _mm256_add_ps(diffs0, px01); - diffs1 = _mm256_add_ps(diffs1, px23); - diffs2 = _mm256_add_ps(diffs2, px45); - - sums0 = _mm256_add_ps(sums0, diffs0); - sums1 = _mm256_add_ps(sums1, diffs1); - sums2 = _mm256_add_ps(sums2, diffs2); + yy += 6; } - yy += 6; - } - - for y in yy..height.min(end) as usize { - let mut diffs = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); + for y in yy..height.min(end) as usize { + let mut diffs = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); - let current_y = ((y as i64) * (stride as i64)) as usize; + let current_y = ((y as i64) * (stride as i64)) as usize; - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; - let pixel = _mm_mul_ps(summs, _mm256_castps256_ps128(v_weight)); + let pixel = _mm_mul_ps(summs, _mm256_castps256_ps128(v_weight)); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; - store_f32::(dst_ptr, pixel); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; + store_f32::(dst_ptr, pixel); - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let d_buf_ptr = bf0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let d_stored = _mm_loadu_ps(d_buf_ptr); + let d_buf_ptr = bf0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let d_stored = _mm_loadu_ps(d_buf_ptr); - let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; - let a_stored = _mm_loadu_ps(buf_ptr); + let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; + let a_stored = _mm_loadu_ps(buf_ptr); - diffs = _mm_add_ps( - diffs, - _mm_opt_fnmlaf_ps::( - a_stored, - d_stored, + diffs = _mm_add_ps( + diffs, + _mm_opt_fnmlaf_ps::( + a_stored, + d_stored, + _mm256_castps256_ps128(v_double), + ), + ); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); + diffs = _mm_opt_fnmlaf_ps::( + diffs, + stored, _mm256_castps256_ps128(v_double), - ), - ); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); - diffs = - _mm_opt_fnmlaf_ps::(diffs, stored, _mm256_castps256_ps128(v_double)); - } + ); + } - let next_row_y = y * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_y = y * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f32; - let pixel_color = load_f32::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f32; + let pixel_color = load_f32::(s_ptr); - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_ps(diffs, pixel_color); - summs = _mm_add_ps(summs, diffs); - _mm_storeu_ps(buf_ptr, pixel_color); + diffs = _mm_add_ps(diffs, pixel_color); + summs = _mm_add_ps(summs, diffs); + _mm_storeu_ps(buf_ptr, pixel_color); + } } } } } -pub(crate) fn fg_vertical_pass_avx_f32( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_vertical_pass_avx_f32( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -297,22 +313,20 @@ pub(crate) fn fg_vertical_pass_avx_f32( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undefined_slice); - if std::arch::is_x86_feature_detected!("fma") { fg_vertical_pass_avx_f32_fma::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } else { fg_vertical_pass_avx_f32_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } } #[target_feature(enable = "avx2")] -unsafe fn fg_vertical_pass_avx_f32_def( +fn fg_vertical_pass_avx_f32_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -327,7 +341,7 @@ unsafe fn fg_vertical_pass_avx_f32_def( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn fg_vertical_pass_avx_f32_fma( +fn fg_vertical_pass_avx_f32_fma( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -346,7 +360,7 @@ struct VerticalExecutionUnit {} impl VerticalExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, bytes: &UnsafeSlice, stride: u32, @@ -357,183 +371,200 @@ impl VerticalExecutionUnit { end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseF32x8::default(); 1024 * 3]); - - let (bf0, rem) = full_buffer.split_at_mut(1024); - let (bf1, bf2) = rem.split_at_mut(1024); - - let v_double = _mm256_set1_ps(2.); - let v_weight = _mm256_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let height_wide = height as i64; - - let radius_64 = radius as i64; - - let mut xx = start as usize; - - while xx + 6 < width.min(end) as usize { - let mut diffs0 = _mm256_setzero_ps(); - let mut diffs1 = _mm256_setzero_ps(); - let mut diffs2 = _mm256_setzero_ps(); - - let mut sums0 = _mm256_setzero_ps(); - let mut sums1 = _mm256_setzero_ps(); - let mut sums2 = _mm256_setzero_ps(); - - let current_px0 = xx * CN; - let current_px1 = (xx + 1) * CN; - let current_px2 = (xx + 2) * CN; - let current_px3 = (xx + 3) * CN; - let current_px4 = (xx + 3) * CN; - let current_px5 = (xx + 3) * CN; - - let start_y = 0 - 2 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_y = (y * (stride as i64)) as usize; - - let prepared_px0 = _mm256_mul_ps(sums0, v_weight); - let prepared_px1 = _mm256_mul_ps(sums1, v_weight); - let prepared_px2 = _mm256_mul_ps(sums2, v_weight); - - let dst_ptr0 = bytes.slice.as_ptr().add(current_y + current_px0) as *mut f32; - let dst_ptr1 = bytes.slice.as_ptr().add(current_y + current_px1) as *mut f32; - let dst_ptr2 = bytes.slice.as_ptr().add(current_y + current_px2) as *mut f32; - let dst_ptr3 = bytes.slice.as_ptr().add(current_y + current_px3) as *mut f32; - let dst_ptr4 = bytes.slice.as_ptr().add(current_y + current_px4) as *mut f32; - let dst_ptr5 = bytes.slice.as_ptr().add(current_y + current_px5) as *mut f32; - - store_f32::(dst_ptr0, _mm256_castps256_ps128(prepared_px0)); - store_f32::(dst_ptr1, _mm256_extractf128_ps::<1>(prepared_px0)); - store_f32::(dst_ptr2, _mm256_castps256_ps128(prepared_px1)); - store_f32::(dst_ptr3, _mm256_extractf128_ps::<1>(prepared_px1)); - store_f32::(dst_ptr4, _mm256_castps256_ps128(prepared_px2)); - store_f32::(dst_ptr5, _mm256_extractf128_ps::<1>(prepared_px2)); - - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let d_s0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_arr_index) as *mut f32); - let d_s1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_arr_index) as *mut f32); - let d_s2 = _mm256_load_ps(bf2.as_mut_ptr().add(d_arr_index) as *mut f32); - - let a_s0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let a_s1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let a_s2 = _mm256_load_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = - _mm256_add_ps(diffs0, _mm256_opt_fnmlaf_ps::(a_s0, d_s0, v_double)); - diffs1 = - _mm256_add_ps(diffs1, _mm256_opt_fnmlaf_ps::(a_s1, d_s1, v_double)); - diffs2 = - _mm256_add_ps(diffs2, _mm256_opt_fnmlaf_ps::(a_s2, d_s2, v_double)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let s0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let s1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let s2 = _mm256_load_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = _mm256_opt_fnmlaf_ps::(diffs0, s0, v_double); - diffs1 = _mm256_opt_fnmlaf_ps::(diffs1, s1, v_double); - diffs2 = _mm256_opt_fnmlaf_ps::(diffs2, s2, v_double); + unsafe { + let mut full_buffer = [AvxSseF32x8::default(); 1024 * 3]; + + let (bf0, rem) = full_buffer.split_at_mut(1024); + let (bf1, bf2) = rem.split_at_mut(1024); + + let v_double = _mm256_set1_ps(2.); + let v_weight = _mm256_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let height_wide = height as i64; + + let radius_64 = radius as i64; + + let mut xx = start as usize; + + while xx + 6 <= width.min(end) as usize { + let mut diffs0 = _mm256_setzero_ps(); + let mut diffs1 = _mm256_setzero_ps(); + let mut diffs2 = _mm256_setzero_ps(); + + let mut sums0 = _mm256_setzero_ps(); + let mut sums1 = _mm256_setzero_ps(); + let mut sums2 = _mm256_setzero_ps(); + + let current_px0 = xx * CN; + let current_px1 = (xx + 1) * CN; + let current_px2 = (xx + 2) * CN; + let current_px3 = (xx + 3) * CN; + let current_px4 = (xx + 3) * CN; + let current_px5 = (xx + 3) * CN; + + let start_y = 0 - 2 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_y = (y * (stride as i64)) as usize; + + let prepared_px0 = _mm256_mul_ps(sums0, v_weight); + let prepared_px1 = _mm256_mul_ps(sums1, v_weight); + let prepared_px2 = _mm256_mul_ps(sums2, v_weight); + + let dst_ptr0 = + bytes.slice.as_ptr().add(current_y + current_px0) as *mut f32; + let dst_ptr1 = + bytes.slice.as_ptr().add(current_y + current_px1) as *mut f32; + let dst_ptr2 = + bytes.slice.as_ptr().add(current_y + current_px2) as *mut f32; + let dst_ptr3 = + bytes.slice.as_ptr().add(current_y + current_px3) as *mut f32; + let dst_ptr4 = + bytes.slice.as_ptr().add(current_y + current_px4) as *mut f32; + let dst_ptr5 = + bytes.slice.as_ptr().add(current_y + current_px5) as *mut f32; + + store_f32::(dst_ptr0, _mm256_castps256_ps128(prepared_px0)); + store_f32::(dst_ptr1, _mm256_extractf128_ps::<1>(prepared_px0)); + store_f32::(dst_ptr2, _mm256_castps256_ps128(prepared_px1)); + store_f32::(dst_ptr3, _mm256_extractf128_ps::<1>(prepared_px1)); + store_f32::(dst_ptr4, _mm256_castps256_ps128(prepared_px2)); + store_f32::(dst_ptr5, _mm256_extractf128_ps::<1>(prepared_px2)); + + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let d_s0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_arr_index) as *mut f32); + let d_s1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_arr_index) as *mut f32); + let d_s2 = _mm256_load_ps(bf2.as_mut_ptr().add(d_arr_index) as *mut f32); + + let a_s0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let a_s1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let a_s2 = _mm256_load_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm256_add_ps( + diffs0, + _mm256_opt_fnmlaf_ps::(a_s0, d_s0, v_double), + ); + diffs1 = _mm256_add_ps( + diffs1, + _mm256_opt_fnmlaf_ps::(a_s1, d_s1, v_double), + ); + diffs2 = _mm256_add_ps( + diffs2, + _mm256_opt_fnmlaf_ps::(a_s2, d_s2, v_double), + ); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let s0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let s1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let s2 = _mm256_load_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm256_opt_fnmlaf_ps::(diffs0, s0, v_double); + diffs1 = _mm256_opt_fnmlaf_ps::(diffs1, s1, v_double); + diffs2 = _mm256_opt_fnmlaf_ps::(diffs2, s2, v_double); + } + + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut f32; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut f32; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut f32; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut f32; + let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut f32; + let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut f32; + + let px0 = load_f32::(s_ptr0); + let px1 = load_f32::(s_ptr1); + let px2 = load_f32::(s_ptr2); + let px3 = load_f32::(s_ptr3); + let px4 = load_f32::(s_ptr4); + let px5 = load_f32::(s_ptr5); + + let arr_index = ((y + radius_64) & 1023) as usize; + + let px01 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px0), px1); + let px23 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px2), px3); + let px45 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px4), px5); + + _mm256_store_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32, px01); + _mm256_store_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32, px23); + _mm256_store_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32, px45); + + diffs0 = _mm256_add_ps(diffs0, px01); + diffs1 = _mm256_add_ps(diffs1, px23); + diffs2 = _mm256_add_ps(diffs2, px45); + + sums0 = _mm256_add_ps(sums0, diffs0); + sums1 = _mm256_add_ps(sums1, diffs1); + sums2 = _mm256_add_ps(sums2, diffs2); } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut f32; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut f32; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut f32; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut f32; - let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut f32; - let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut f32; - - let px0 = load_f32::(s_ptr0); - let px1 = load_f32::(s_ptr1); - let px2 = load_f32::(s_ptr2); - let px3 = load_f32::(s_ptr3); - let px4 = load_f32::(s_ptr4); - let px5 = load_f32::(s_ptr5); - - let arr_index = ((y + radius_64) & 1023) as usize; - - let px01 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px0), px1); - let px23 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px2), px3); - let px45 = _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(px4), px5); - - _mm256_store_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32, px01); - _mm256_store_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32, px23); - _mm256_store_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32, px45); - - diffs0 = _mm256_add_ps(diffs0, px01); - diffs1 = _mm256_add_ps(diffs1, px23); - diffs2 = _mm256_add_ps(diffs2, px45); - - sums0 = _mm256_add_ps(sums0, diffs0); - sums1 = _mm256_add_ps(sums1, diffs1); - sums2 = _mm256_add_ps(sums2, diffs2); + xx += 6; } - xx += 6; - } - - for x in xx..width.min(end) as usize { - let mut diffs = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); + for x in xx..width.min(end) as usize { + let mut diffs = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); - let start_y = 0 - 2 * radius as i64; + let start_y = 0 - 2 * radius as i64; - let current_px = x * CN; + let current_px = x * CN; - for y in start_y..height_wide { - let current_y = (y * (stride as i64)) as usize; - - if y >= 0 { - let pixel = _mm_mul_ps(summs, _mm256_castps256_ps128(v_weight)); - - let bytes_offset = current_y + current_px; - - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; - store_f32::(dst_ptr, pixel); - - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let d_buf_ptr = bf0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let d_stored = _mm_loadu_ps(d_buf_ptr); - - let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; - let a_stored = _mm_loadu_ps(buf_ptr); + for y in start_y..height_wide { + let current_y = (y * (stride as i64)) as usize; - diffs = _mm_add_ps( - diffs, - _mm_opt_fnmlaf_ps::( - a_stored, - d_stored, + if y >= 0 { + let pixel = _mm_mul_ps(summs, _mm256_castps256_ps128(v_weight)); + + let bytes_offset = current_y + current_px; + + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; + store_f32::(dst_ptr, pixel); + + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let d_buf_ptr = bf0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let d_stored = _mm_loadu_ps(d_buf_ptr); + + let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; + let a_stored = _mm_loadu_ps(buf_ptr); + + diffs = _mm_add_ps( + diffs, + _mm_opt_fnmlaf_ps::( + a_stored, + d_stored, + _mm256_castps256_ps128(v_double), + ), + ); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); + diffs = _mm_opt_fnmlaf_ps::( + diffs, + stored, _mm256_castps256_ps128(v_double), - ), - ); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); - diffs = - _mm_opt_fnmlaf_ps::(diffs, stored, _mm256_castps256_ps128(v_double)); - } + ); + } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - let next_row_x = x * CN; + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + let next_row_x = x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f32; - let pixel_color = load_f32::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f32; + let pixel_color = load_f32::(s_ptr); - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_ps(diffs, pixel_color); - summs = _mm_add_ps(summs, diffs); - _mm_storeu_ps(buf_ptr, pixel_color); + diffs = _mm_add_ps(diffs, pixel_color); + summs = _mm_add_ps(summs, diffs); + _mm_storeu_ps(buf_ptr, pixel_color); + } } } } diff --git a/src/avx/fast_gaussian_next.rs b/src/avx/fast_gaussian_next.rs index c07658ee..4fa0cd10 100644 --- a/src/avx/fast_gaussian_next.rs +++ b/src/avx/fast_gaussian_next.rs @@ -25,15 +25,15 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::avx::utils::_mm256_mul_by_3_epi32; use crate::edge_mode::clamp_edge; use crate::sse::{_mm_mul_by_3_epi32, load_u8_s32_fast, store_u8_u32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::x86_64::*; -pub(crate) fn fgn_vertical_pass_avx_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fgn_vertical_pass_avx_u8( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -43,10 +43,8 @@ pub(crate) fn fgn_vertical_pass_avx_u8( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - let unit = VerticalExecutionUnit::::default(); - unit.pass(bytes, stride, width, height, radius, start, end, edge_mode); + unit.pass(slice, stride, width, height, radius, start, end, edge_mode); } } @@ -59,7 +57,7 @@ struct VerticalExecutionUnit {} impl VerticalExecutionUnit { #[target_feature(enable = "avx2")] - unsafe fn pass( + fn pass( &self, bytes: &UnsafeSlice, stride: u32, @@ -70,295 +68,305 @@ impl VerticalExecutionUnit { end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseI32x8::default(); 1024 * 3]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, buffer2) = rem.split_at_mut(1024); - - let height_wide = height as i64; - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm256_set1_ps(weight); - - let mut xx = start; - - while xx + 6 < width.min(end) { - let mut diffs0 = _mm256_setzero_si256(); - let mut diffs1 = _mm256_setzero_si256(); - let mut diffs2 = _mm256_setzero_si256(); - - let mut ders0 = _mm256_setzero_si256(); - let mut ders1 = _mm256_setzero_si256(); - let mut ders2 = _mm256_setzero_si256(); - - let mut summs0 = _mm256_setzero_si256(); - let mut summs1 = _mm256_setzero_si256(); - let mut summs2 = _mm256_setzero_si256(); - - let start_y = 0 - 3 * radius as i64; - - let current_px0 = (xx * CN as u32) as usize; - let current_px1 = ((xx + 1) * CN as u32) as usize; - let current_px2 = ((xx + 2) * CN as u32) as usize; - let current_px3 = ((xx + 3) * CN as u32) as usize; - let current_px4 = ((xx + 4) * CN as u32) as usize; - let current_px5 = ((xx + 5) * CN as u32) as usize; - - for y in start_y..height_wide { - if y >= 0 { - let ps01 = _mm256_cvtepi32_ps(summs0); - let ps23 = _mm256_cvtepi32_ps(summs1); - let ps45 = _mm256_cvtepi32_ps(summs2); - - let r01 = _mm256_mul_ps(ps01, v_weight); - let r23 = _mm256_mul_ps(ps23, v_weight); - let r45 = _mm256_mul_ps(ps45, v_weight); - - let e01 = _mm256_cvtps_epi32(r01); - let e23 = _mm256_cvtps_epi32(r23); - let e45 = _mm256_cvtps_epi32(r45); - - let prepared_px0 = _mm256_castsi256_si128(e01); - let prepared_px1 = _mm256_extracti128_si256::<1>(e01); - let prepared_px2 = _mm256_castsi256_si128(e23); - let prepared_px3 = _mm256_extracti128_si256::<1>(e23); - let prepared_px4 = _mm256_castsi256_si128(e45); - let prepared_px5 = _mm256_extracti128_si256::<1>(e45); - - let current_y = (y * (stride as i64)) as usize; - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px0); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px1); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px2); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px3); - let dst_ptr4 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px4); - let dst_ptr5 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px5); - - store_u8_u32::(dst_ptr0, prepared_px0); - store_u8_u32::(dst_ptr1, prepared_px1); - store_u8_u32::(dst_ptr2, prepared_px2); - store_u8_u32::(dst_ptr3, prepared_px3); - store_u8_u32::(dst_ptr4, prepared_px4); - store_u8_u32::(dst_ptr5, prepared_px5); - - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let stored0 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index) as *const _); - let stored1 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index) as *const _); - let stored2 = - _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index) as *const _); - - let stored_10 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_11 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_12 = - _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); - - let j0 = _mm256_sub_epi32(stored0, stored_10); - let j1 = _mm256_sub_epi32(stored1, stored_11); - let j2 = _mm256_sub_epi32(stored2, stored_12); - - let stored_20 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_21 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_22 = - _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); - - let k0 = _mm256_mul_by_3_epi32(j0); - let k1 = _mm256_mul_by_3_epi32(j1); - let k2 = _mm256_mul_by_3_epi32(j2); - - let new_diff0 = _mm256_sub_epi32(k0, stored_20); - let new_diff1 = _mm256_sub_epi32(k1, stored_21); - let new_diff2 = _mm256_sub_epi32(k2, stored_22); - - diffs0 = _mm256_add_epi32(diffs0, new_diff0); - diffs1 = _mm256_add_epi32(diffs1, new_diff1); - diffs2 = _mm256_add_epi32(diffs2, new_diff2); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; - - let stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + unsafe { + let mut full_buffer = [AvxSseI32x8::default(); 1024 * 3]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, buffer2) = rem.split_at_mut(1024); + + let height_wide = height as i64; + + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm256_set1_ps(weight); + + let mut xx = start; + + while xx + 6 <= width.min(end) { + let mut diffs0 = _mm256_setzero_si256(); + let mut diffs1 = _mm256_setzero_si256(); + let mut diffs2 = _mm256_setzero_si256(); + + let mut ders0 = _mm256_setzero_si256(); + let mut ders1 = _mm256_setzero_si256(); + let mut ders2 = _mm256_setzero_si256(); + + let mut summs0 = _mm256_setzero_si256(); + let mut summs1 = _mm256_setzero_si256(); + let mut summs2 = _mm256_setzero_si256(); + + let start_y = 0 - 3 * radius as i64; + + let current_px0 = (xx * CN as u32) as usize; + let current_px1 = ((xx + 1) * CN as u32) as usize; + let current_px2 = ((xx + 2) * CN as u32) as usize; + let current_px3 = ((xx + 3) * CN as u32) as usize; + let current_px4 = ((xx + 4) * CN as u32) as usize; + let current_px5 = ((xx + 5) * CN as u32) as usize; + + for y in start_y..height_wide { + if y >= 0 { + let ps01 = _mm256_cvtepi32_ps(summs0); + let ps23 = _mm256_cvtepi32_ps(summs1); + let ps45 = _mm256_cvtepi32_ps(summs2); + + let r01 = _mm256_mul_ps(ps01, v_weight); + let r23 = _mm256_mul_ps(ps23, v_weight); + let r45 = _mm256_mul_ps(ps45, v_weight); + + let e01 = _mm256_cvtps_epi32(r01); + let e23 = _mm256_cvtps_epi32(r23); + let e45 = _mm256_cvtps_epi32(r45); + + let prepared_px0 = _mm256_castsi256_si128(e01); + let prepared_px1 = _mm256_extracti128_si256::<1>(e01); + let prepared_px2 = _mm256_castsi256_si128(e23); + let prepared_px3 = _mm256_extracti128_si256::<1>(e23); + let prepared_px4 = _mm256_castsi256_si128(e45); + let prepared_px5 = _mm256_extracti128_si256::<1>(e45); + + let current_y = (y * (stride as i64)) as usize; + + let dst_ptr0 = + (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px0); + let dst_ptr1 = + (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px1); + let dst_ptr2 = + (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px2); + let dst_ptr3 = + (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px3); + let dst_ptr4 = + (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px4); + let dst_ptr5 = + (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px5); + + store_u8_u32::(dst_ptr0, prepared_px0); + store_u8_u32::(dst_ptr1, prepared_px1); + store_u8_u32::(dst_ptr2, prepared_px2); + store_u8_u32::(dst_ptr3, prepared_px3); + store_u8_u32::(dst_ptr4, prepared_px4); + store_u8_u32::(dst_ptr5, prepared_px5); + + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let stored0 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index) as *const _); + let stored1 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index) as *const _); + let stored2 = + _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index) as *const _); + + let stored_10 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_11 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_12 = + _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); + + let j0 = _mm256_sub_epi32(stored0, stored_10); + let j1 = _mm256_sub_epi32(stored1, stored_11); + let j2 = _mm256_sub_epi32(stored2, stored_12); + + let stored_20 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_21 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_22 = + _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); + + let k0 = _mm256_mul_by_3_epi32(j0); + let k1 = _mm256_mul_by_3_epi32(j1); + let k2 = _mm256_mul_by_3_epi32(j2); + + let new_diff0 = _mm256_sub_epi32(k0, stored_20); + let new_diff1 = _mm256_sub_epi32(k1, stored_21); + let new_diff2 = _mm256_sub_epi32(k2, stored_22); + + diffs0 = _mm256_add_epi32(diffs0, new_diff0); + diffs1 = _mm256_add_epi32(diffs1, new_diff1); + diffs2 = _mm256_add_epi32(diffs2, new_diff2); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + + let stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + let stored_10 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_11 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_21 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + + let new_diff0 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored0, stored_10)); + let new_diff1 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored1, stored_11)); + let new_diff2 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored2, stored_21)); + + diffs0 = _mm256_add_epi32(diffs0, new_diff0); + diffs1 = _mm256_add_epi32(diffs1, new_diff1); + diffs2 = _mm256_add_epi32(diffs2, new_diff2); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + + let stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm256_sub_epi32(diffs0, _mm256_mul_by_3_epi32(stored0)); + diffs1 = _mm256_sub_epi32(diffs1, _mm256_mul_by_3_epi32(stored1)); + diffs2 = _mm256_sub_epi32(diffs2, _mm256_mul_by_3_epi32(stored2)); + } + + let next_row_y = + clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u8; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u8; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u8; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u8; + let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut u8; + let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut u8; + + let pixel_color0 = load_u8_s32_fast::(s_ptr0); + let pixel_color1 = load_u8_s32_fast::(s_ptr1); + let pixel_color2 = load_u8_s32_fast::(s_ptr2); + let pixel_color3 = load_u8_s32_fast::(s_ptr3); + let pixel_color4 = load_u8_s32_fast::(s_ptr4); + let pixel_color5 = load_u8_s32_fast::(s_ptr5); + + let px01 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color0), + pixel_color1, ); - let stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px23 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color2), + pixel_color3, ); - let stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px45 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color4), + pixel_color5, ); - let stored_10 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_11 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_21 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); + let arr_index = ((y + 2 * radius_64) & 1023) as usize; - let new_diff0 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored0, stored_10)); - let new_diff1 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored1, stored_11)); - let new_diff2 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored2, stored_21)); + let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs0 = _mm256_add_epi32(diffs0, new_diff0); - diffs1 = _mm256_add_epi32(diffs1, new_diff1); - diffs2 = _mm256_add_epi32(diffs2, new_diff2); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; + diffs0 = _mm256_add_epi32(diffs0, px01); + diffs1 = _mm256_add_epi32(diffs1, px23); + diffs2 = _mm256_add_epi32(diffs2, px45); - let stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); - let stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); - let stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); + _mm256_store_si256(buf_ptr0 as *mut _, px01); + _mm256_store_si256(buf_ptr1 as *mut _, px23); + _mm256_store_si256(buf_ptr2 as *mut _, px45); - diffs0 = _mm256_sub_epi32(diffs0, _mm256_mul_by_3_epi32(stored0)); - diffs1 = _mm256_sub_epi32(diffs1, _mm256_mul_by_3_epi32(stored1)); - diffs2 = _mm256_sub_epi32(diffs2, _mm256_mul_by_3_epi32(stored2)); - } + ders0 = _mm256_add_epi32(ders0, diffs0); + ders1 = _mm256_add_epi32(ders1, diffs1); + ders2 = _mm256_add_epi32(ders2, diffs2); - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u8; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u8; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u8; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u8; - let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut u8; - let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut u8; - - let pixel_color0 = load_u8_s32_fast::(s_ptr0); - let pixel_color1 = load_u8_s32_fast::(s_ptr1); - let pixel_color2 = load_u8_s32_fast::(s_ptr2); - let pixel_color3 = load_u8_s32_fast::(s_ptr3); - let pixel_color4 = load_u8_s32_fast::(s_ptr4); - let pixel_color5 = load_u8_s32_fast::(s_ptr5); - - let px01 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color0), - pixel_color1, - ); - let px23 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color2), - pixel_color3, - ); - let px45 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color4), - pixel_color5, - ); - - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - - let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); - - diffs0 = _mm256_add_epi32(diffs0, px01); - diffs1 = _mm256_add_epi32(diffs1, px23); - diffs2 = _mm256_add_epi32(diffs2, px45); - - _mm256_store_si256(buf_ptr0 as *mut _, px01); - _mm256_store_si256(buf_ptr1 as *mut _, px23); - _mm256_store_si256(buf_ptr2 as *mut _, px45); - - ders0 = _mm256_add_epi32(ders0, diffs0); - ders1 = _mm256_add_epi32(ders1, diffs1); - ders2 = _mm256_add_epi32(ders2, diffs2); - - summs0 = _mm256_add_epi32(summs0, ders0); - summs1 = _mm256_add_epi32(summs1, ders1); - summs2 = _mm256_add_epi32(summs2, ders2); + summs0 = _mm256_add_epi32(summs0, ders0); + summs1 = _mm256_add_epi32(summs1, ders1); + summs2 = _mm256_add_epi32(summs2, ders2); + } + xx += 6; } - xx += 6; - } - for x in xx..width.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut ders = _mm_setzero_si128(); - let mut summs = _mm_setzero_si128(); + for x in xx..width.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut ders = _mm_setzero_si128(); + let mut summs = _mm_setzero_si128(); - let start_y = 0 - 3 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_px = (x * CN as u32) as usize; - let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps( - _mm_cvtepi32_ps(summs), - _mm256_castps256_ps128(v_weight), - )); + let start_y = 0 - 3 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_px = (x * CN as u32) as usize; + let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps( + _mm_cvtepi32_ps(summs), + _mm256_castps256_ps128(v_weight), + )); - let current_y = (y * (stride as i64)) as usize; + let current_y = (y * (stride as i64)) as usize; - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - store_u8_u32::(dst_ptr, prepared_px_s32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + store_u8_u32::(dst_ptr, prepared_px_s32); - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.as_mut_ptr().add(d_arr_index_1); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.as_mut_ptr().add(d_arr_index_1); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let buf_ptr_2 = buffer0.as_mut_ptr().add(d_arr_index_2); - let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); + let buf_ptr_2 = buffer0.as_mut_ptr().add(d_arr_index_2); + let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); - let new_diff = _mm_sub_epi32( - _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), - stored_2, - ); - diffs = _mm_add_epi32(diffs, new_diff); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let new_diff = _mm_sub_epi32( + _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), + stored_2, + ); + diffs = _mm_add_epi32(diffs, new_diff); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr(); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr(); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); + let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); - diffs = _mm_add_epi32(diffs, new_diff); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); - } + diffs = _mm_add_epi32(diffs, new_diff); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); + diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); + } - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let next_row_y = + clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; - let pixel_color = load_u8_s32_fast::(s_ptr); + let pixel_color = load_u8_s32_fast::(s_ptr); - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((y + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); - ders = _mm_add_epi32(ders, diffs); - summs = _mm_add_epi32(summs, ders); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + diffs = _mm_add_epi32(diffs, pixel_color); + ders = _mm_add_epi32(ders, diffs); + summs = _mm_add_epi32(summs, ders); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } @@ -386,7 +394,7 @@ struct HorizontalExecutionUnit {} impl HorizontalExecutionUnit { #[target_feature(enable = "avx2")] - unsafe fn pass( + fn pass( &self, bytes: &UnsafeSlice, stride: u32, @@ -397,292 +405,300 @@ impl HorizontalExecutionUnit { end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseI32x8::default(); 1024 * 3]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, buffer2) = rem.split_at_mut(1024); - - let width_wide = width as i64; - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm256_set1_ps(weight); - - let mut yy = start; - - while yy + 6 < height.min(end) { - let mut diffs0 = _mm256_setzero_si256(); - let mut diffs1 = _mm256_setzero_si256(); - let mut diffs2 = _mm256_setzero_si256(); - - let mut ders0 = _mm256_setzero_si256(); - let mut ders1 = _mm256_setzero_si256(); - let mut ders2 = _mm256_setzero_si256(); - - let mut summs0 = _mm256_setzero_si256(); - let mut summs1 = _mm256_setzero_si256(); - let mut summs2 = _mm256_setzero_si256(); - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; - let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; - let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; - let current_y4 = ((yy as i64 + 4) * (stride as i64)) as usize; - let current_y5 = ((yy as i64 + 5) * (stride as i64)) as usize; - - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; - - let ps01 = _mm256_cvtepi32_ps(summs0); - let ps23 = _mm256_cvtepi32_ps(summs1); - let ps45 = _mm256_cvtepi32_ps(summs2); - - let r01 = _mm256_mul_ps(ps01, v_weight); - let r23 = _mm256_mul_ps(ps23, v_weight); - let r45 = _mm256_mul_ps(ps45, v_weight); - - let e01 = _mm256_cvtps_epi32(r01); - let e23 = _mm256_cvtps_epi32(r23); - let e45 = _mm256_cvtps_epi32(r45); - - let prepared_px0 = _mm256_castsi256_si128(e01); - let prepared_px1 = _mm256_extracti128_si256::<1>(e01); - let prepared_px2 = _mm256_castsi256_si128(e23); - let prepared_px3 = _mm256_extracti128_si256::<1>(e23); - let prepared_px4 = _mm256_castsi256_si128(e45); - let prepared_px5 = _mm256_extracti128_si256::<1>(e45); - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y0 + current_px); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y1 + current_px); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y2 + current_px); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y3 + current_px); - let dst_ptr4 = (bytes.slice.as_ptr() as *mut u8).add(current_y4 + current_px); - let dst_ptr5 = (bytes.slice.as_ptr() as *mut u8).add(current_y5 + current_px); - - store_u8_u32::(dst_ptr0, prepared_px0); - store_u8_u32::(dst_ptr1, prepared_px1); - store_u8_u32::(dst_ptr2, prepared_px2); - store_u8_u32::(dst_ptr3, prepared_px3); - store_u8_u32::(dst_ptr4, prepared_px4); - store_u8_u32::(dst_ptr5, prepared_px5); - - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let stored0 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index) as *const _); - let stored1 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index) as *const _); - let stored2 = - _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index) as *const _); - - let stored_10 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_11 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_12 = - _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); - - let j0 = _mm256_sub_epi32(stored0, stored_10); - let j1 = _mm256_sub_epi32(stored1, stored_11); - let j2 = _mm256_sub_epi32(stored2, stored_12); - - let stored_20 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_21 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_22 = - _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); - - let k0 = _mm256_mul_by_3_epi32(j0); - let k1 = _mm256_mul_by_3_epi32(j1); - let k2 = _mm256_mul_by_3_epi32(j2); - - let new_diff0 = _mm256_sub_epi32(k0, stored_20); - let new_diff1 = _mm256_sub_epi32(k1, stored_21); - let new_diff2 = _mm256_sub_epi32(k2, stored_22); - - diffs0 = _mm256_add_epi32(diffs0, new_diff0); - diffs1 = _mm256_add_epi32(diffs1, new_diff1); - diffs2 = _mm256_add_epi32(diffs2, new_diff2); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - - let stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + unsafe { + let mut full_buffer = [AvxSseI32x8::default(); 1024 * 3]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, buffer2) = rem.split_at_mut(1024); + + let width_wide = width as i64; + + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm256_set1_ps(weight); + + let mut yy = start; + + while yy + 6 <= height.min(end) { + let mut diffs0 = _mm256_setzero_si256(); + let mut diffs1 = _mm256_setzero_si256(); + let mut diffs2 = _mm256_setzero_si256(); + + let mut ders0 = _mm256_setzero_si256(); + let mut ders1 = _mm256_setzero_si256(); + let mut ders2 = _mm256_setzero_si256(); + + let mut summs0 = _mm256_setzero_si256(); + let mut summs1 = _mm256_setzero_si256(); + let mut summs2 = _mm256_setzero_si256(); + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; + let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; + let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + let current_y4 = ((yy as i64 + 4) * (stride as i64)) as usize; + let current_y5 = ((yy as i64 + 5) * (stride as i64)) as usize; + + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; + + let ps01 = _mm256_cvtepi32_ps(summs0); + let ps23 = _mm256_cvtepi32_ps(summs1); + let ps45 = _mm256_cvtepi32_ps(summs2); + + let r01 = _mm256_mul_ps(ps01, v_weight); + let r23 = _mm256_mul_ps(ps23, v_weight); + let r45 = _mm256_mul_ps(ps45, v_weight); + + let e01 = _mm256_cvtps_epi32(r01); + let e23 = _mm256_cvtps_epi32(r23); + let e45 = _mm256_cvtps_epi32(r45); + + let prepared_px0 = _mm256_castsi256_si128(e01); + let prepared_px1 = _mm256_extracti128_si256::<1>(e01); + let prepared_px2 = _mm256_castsi256_si128(e23); + let prepared_px3 = _mm256_extracti128_si256::<1>(e23); + let prepared_px4 = _mm256_castsi256_si128(e45); + let prepared_px5 = _mm256_extracti128_si256::<1>(e45); + + let dst_ptr0 = + (bytes.slice.as_ptr() as *mut u8).add(current_y0 + current_px); + let dst_ptr1 = + (bytes.slice.as_ptr() as *mut u8).add(current_y1 + current_px); + let dst_ptr2 = + (bytes.slice.as_ptr() as *mut u8).add(current_y2 + current_px); + let dst_ptr3 = + (bytes.slice.as_ptr() as *mut u8).add(current_y3 + current_px); + let dst_ptr4 = + (bytes.slice.as_ptr() as *mut u8).add(current_y4 + current_px); + let dst_ptr5 = + (bytes.slice.as_ptr() as *mut u8).add(current_y5 + current_px); + + store_u8_u32::(dst_ptr0, prepared_px0); + store_u8_u32::(dst_ptr1, prepared_px1); + store_u8_u32::(dst_ptr2, prepared_px2); + store_u8_u32::(dst_ptr3, prepared_px3); + store_u8_u32::(dst_ptr4, prepared_px4); + store_u8_u32::(dst_ptr5, prepared_px5); + + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let stored0 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index) as *const _); + let stored1 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index) as *const _); + let stored2 = + _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index) as *const _); + + let stored_10 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_11 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_12 = + _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); + + let j0 = _mm256_sub_epi32(stored0, stored_10); + let j1 = _mm256_sub_epi32(stored1, stored_11); + let j2 = _mm256_sub_epi32(stored2, stored_12); + + let stored_20 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_21 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_22 = + _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); + + let k0 = _mm256_mul_by_3_epi32(j0); + let k1 = _mm256_mul_by_3_epi32(j1); + let k2 = _mm256_mul_by_3_epi32(j2); + + let new_diff0 = _mm256_sub_epi32(k0, stored_20); + let new_diff1 = _mm256_sub_epi32(k1, stored_21); + let new_diff2 = _mm256_sub_epi32(k2, stored_22); + + diffs0 = _mm256_add_epi32(diffs0, new_diff0); + diffs1 = _mm256_add_epi32(diffs1, new_diff1); + diffs2 = _mm256_add_epi32(diffs2, new_diff2); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + + let stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + let stored_10 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_11 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_12 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + + let new_diff0 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored0, stored_10)); + let new_diff1 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored1, stored_11)); + let new_diff2 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored2, stored_12)); + + diffs0 = _mm256_add_epi32(diffs0, new_diff0); + diffs1 = _mm256_add_epi32(diffs1, new_diff1); + diffs2 = _mm256_add_epi32(diffs2, new_diff2); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm256_sub_epi32(diffs0, _mm256_mul_by_3_epi32(stored0)); + diffs1 = _mm256_sub_epi32(diffs1, _mm256_mul_by_3_epi32(stored1)); + diffs2 = _mm256_sub_epi32(diffs2, _mm256_mul_by_3_epi32(stored2)); + } + + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u8; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u8; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u8; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u8; + let s_ptr4 = bytes.slice.as_ptr().add(current_y4 + next_row_px) as *mut u8; + let s_ptr5 = bytes.slice.as_ptr().add(current_y5 + next_row_px) as *mut u8; + + let pixel_color0 = load_u8_s32_fast::(s_ptr0); + let pixel_color1 = load_u8_s32_fast::(s_ptr1); + let pixel_color2 = load_u8_s32_fast::(s_ptr2); + let pixel_color3 = load_u8_s32_fast::(s_ptr3); + let pixel_color4 = load_u8_s32_fast::(s_ptr4); + let pixel_color5 = load_u8_s32_fast::(s_ptr5); + + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); + + let px01 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color0), + pixel_color1, ); - let stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px23 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color2), + pixel_color3, ); - let stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px45 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color4), + pixel_color5, ); - let stored_10 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_11 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_12 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); + diffs0 = _mm256_add_epi32(diffs0, px01); + diffs1 = _mm256_add_epi32(diffs1, px23); + diffs2 = _mm256_add_epi32(diffs2, px45); - let new_diff0 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored0, stored_10)); - let new_diff1 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored1, stored_11)); - let new_diff2 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored2, stored_12)); - - diffs0 = _mm256_add_epi32(diffs0, new_diff0); - diffs1 = _mm256_add_epi32(diffs1, new_diff1); - diffs2 = _mm256_add_epi32(diffs2, new_diff2); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); - let stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); - let stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); + _mm256_store_si256(buf_ptr0 as *mut _, px01); + _mm256_store_si256(buf_ptr1 as *mut _, px23); + _mm256_store_si256(buf_ptr2 as *mut _, px45); - diffs0 = _mm256_sub_epi32(diffs0, _mm256_mul_by_3_epi32(stored0)); - diffs1 = _mm256_sub_epi32(diffs1, _mm256_mul_by_3_epi32(stored1)); - diffs2 = _mm256_sub_epi32(diffs2, _mm256_mul_by_3_epi32(stored2)); + ders0 = _mm256_add_epi32(ders0, diffs0); + ders1 = _mm256_add_epi32(ders1, diffs1); + ders2 = _mm256_add_epi32(ders2, diffs2); + + summs0 = _mm256_add_epi32(summs0, ders0); + summs1 = _mm256_add_epi32(summs1, ders1); + summs2 = _mm256_add_epi32(summs2, ders2); } - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u8; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u8; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u8; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u8; - let s_ptr4 = bytes.slice.as_ptr().add(current_y4 + next_row_px) as *mut u8; - let s_ptr5 = bytes.slice.as_ptr().add(current_y5 + next_row_px) as *mut u8; - - let pixel_color0 = load_u8_s32_fast::(s_ptr0); - let pixel_color1 = load_u8_s32_fast::(s_ptr1); - let pixel_color2 = load_u8_s32_fast::(s_ptr2); - let pixel_color3 = load_u8_s32_fast::(s_ptr3); - let pixel_color4 = load_u8_s32_fast::(s_ptr4); - let pixel_color5 = load_u8_s32_fast::(s_ptr5); - - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); - - let px01 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color0), - pixel_color1, - ); - let px23 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color2), - pixel_color3, - ); - let px45 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color4), - pixel_color5, - ); - - diffs0 = _mm256_add_epi32(diffs0, px01); - diffs1 = _mm256_add_epi32(diffs1, px23); - diffs2 = _mm256_add_epi32(diffs2, px45); - - _mm256_store_si256(buf_ptr0 as *mut _, px01); - _mm256_store_si256(buf_ptr1 as *mut _, px23); - _mm256_store_si256(buf_ptr2 as *mut _, px45); - - ders0 = _mm256_add_epi32(ders0, diffs0); - ders1 = _mm256_add_epi32(ders1, diffs1); - ders2 = _mm256_add_epi32(ders2, diffs2); - - summs0 = _mm256_add_epi32(summs0, ders0); - summs1 = _mm256_add_epi32(summs1, ders1); - summs2 = _mm256_add_epi32(summs2, ders2); + yy += 6; } - yy += 6; - } + for y in yy..height.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut ders = _mm_setzero_si128(); + let mut summs = _mm_setzero_si128(); - for y in yy..height.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut ders = _mm_setzero_si128(); - let mut summs = _mm_setzero_si128(); + let current_y = ((y as i64) * (stride as i64)) as usize; - let current_y = ((y as i64) * (stride as i64)) as usize; + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps( + _mm_cvtepi32_ps(summs), + _mm256_castps256_ps128(v_weight), + )); - let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps( - _mm_cvtepi32_ps(summs), - _mm256_castps256_ps128(v_weight), - )); + let bytes_offset = current_y + current_px; - let bytes_offset = current_y + current_px; + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + store_u8_u32::(dst_ptr, prepared_px_s32); - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - store_u8_u32::(dst_ptr, prepared_px_s32); + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr_1 = buffer0.get_unchecked_mut(d_arr_index_1).0.as_mut_ptr(); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let buf_ptr_1 = buffer0.get_unchecked_mut(d_arr_index_1).0.as_mut_ptr(); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_2 = buffer0.get_unchecked_mut(d_arr_index_2).0.as_mut_ptr(); + let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); - let buf_ptr_2 = buffer0.get_unchecked_mut(d_arr_index_2).0.as_mut_ptr(); - let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); + let new_diff = _mm_sub_epi32( + _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), + stored_2, + ); + diffs = _mm_add_epi32(diffs, new_diff); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let new_diff = _mm_sub_epi32( - _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), - stored_2, - ); - diffs = _mm_add_epi32(diffs, new_diff); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; - let stored = _mm_load_si128(buf_ptr as *const __m128i); - - let buf_ptr_1 = buffer0.as_mut_ptr().add(arr_index_1); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - - let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); - - diffs = _mm_add_epi32(diffs, new_diff); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.as_mut_ptr().add(arr_index); - let stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); - } + let buf_ptr_1 = buffer0.as_mut_ptr().add(arr_index_1); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; + let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; + diffs = _mm_add_epi32(diffs, new_diff); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.as_mut_ptr().add(arr_index); + let stored = _mm_load_si128(buf_ptr as *const __m128i); + diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); + } - let pixel_color = load_u8_s32_fast::(s_ptr); + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; - diffs = _mm_add_epi32(diffs, pixel_color); - ders = _mm_add_epi32(ders, diffs); - summs = _mm_add_epi32(summs, ders); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + let pixel_color = load_u8_s32_fast::(s_ptr); + + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + + diffs = _mm_add_epi32(diffs, pixel_color); + ders = _mm_add_epi32(ders, diffs); + summs = _mm_add_epi32(summs, ders); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } diff --git a/src/avx/fast_gaussian_next_f32.rs b/src/avx/fast_gaussian_next_f32.rs index bd82428f..050a37de 100644 --- a/src/avx/fast_gaussian_next_f32.rs +++ b/src/avx/fast_gaussian_next_f32.rs @@ -25,15 +25,15 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::avx::utils::{_mm256_opt_fmlaf_ps, _mm256_opt_fnmlaf_ps, _mm256_opt_fnmlsf_ps}; use crate::edge_mode::clamp_edge; use crate::sse::{_mm_opt_fmlaf_ps, _mm_opt_fnmlaf_ps, _mm_opt_fnmlsf_ps, load_f32, store_f32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::x86_64::*; -pub(crate) fn fgn_vertical_pass_avx_f32( - undefined_slice: &UnsafeSlice, +pub(crate) fn fgn_vertical_pass_avx_f32( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -43,14 +43,13 @@ pub(crate) fn fgn_vertical_pass_avx_f32( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undefined_slice); if std::arch::is_x86_feature_detected!("fma") { fgn_vertical_pass_avx_f32_fma::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } else { fgn_vertical_pass_avx_f32_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } @@ -61,7 +60,7 @@ pub(crate) fn fgn_vertical_pass_avx_f32( pub(crate) struct AvxSseF32x8(pub(crate) [f32; 8]); #[target_feature(enable = "avx2")] -unsafe fn fgn_vertical_pass_avx_f32_def( +fn fgn_vertical_pass_avx_f32_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -76,7 +75,7 @@ unsafe fn fgn_vertical_pass_avx_f32_def( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn fgn_vertical_pass_avx_f32_fma( +fn fgn_vertical_pass_avx_f32_fma( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -96,7 +95,7 @@ struct VerticalGaussianExecutorF32 {} impl VerticalGaussianExecutorF32 { #[inline(always)] - unsafe fn vertical_pass( + fn vertical_pass( &self, bytes: &UnsafeSlice, stride: u32, @@ -107,234 +106,254 @@ impl VerticalGaussianExecutorF32 { end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseF32x8::default(); 1024 * 3]); - - let (bf0, rem) = full_buffer.split_at_mut(1024); - let (bf1, bf2) = rem.split_at_mut(1024); - - let height_wide = height as i64; - - let threes = _mm256_set1_ps(3.); - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm256_set1_ps(weight); - - let mut xx = start as usize; - - while xx + 6 < width.min(end) as usize { - let mut diffs0 = _mm256_setzero_ps(); - let mut diffs1 = _mm256_setzero_ps(); - let mut diffs2 = _mm256_setzero_ps(); - - let mut ders0 = _mm256_setzero_ps(); - let mut ders1 = _mm256_setzero_ps(); - let mut ders2 = _mm256_setzero_ps(); - - let mut summs0 = _mm256_setzero_ps(); - let mut summs1 = _mm256_setzero_ps(); - let mut summs2 = _mm256_setzero_ps(); - - let current_px0 = xx * CN; - let current_px1 = (xx + 1) * CN; - let current_px2 = (xx + 2) * CN; - let current_px3 = (xx + 3) * CN; - let current_px4 = (xx + 4) * CN; - let current_px5 = (xx + 5) * CN; - - let start_y = 0 - 3 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_y = (y * (stride as i64)) as usize; - - let r01 = _mm256_mul_ps(summs0, v_weight); - let r23 = _mm256_mul_ps(summs1, v_weight); - let r45 = _mm256_mul_ps(summs2, v_weight); - - let dst_ptr0 = bytes.slice.as_ptr().add(current_y + current_px0) as *mut f32; - let dst_ptr1 = bytes.slice.as_ptr().add(current_y + current_px1) as *mut f32; - let dst_ptr2 = bytes.slice.as_ptr().add(current_y + current_px2) as *mut f32; - let dst_ptr3 = bytes.slice.as_ptr().add(current_y + current_px3) as *mut f32; - let dst_ptr4 = bytes.slice.as_ptr().add(current_y + current_px4) as *mut f32; - let dst_ptr5 = bytes.slice.as_ptr().add(current_y + current_px5) as *mut f32; - - store_f32::(dst_ptr0, _mm256_castps256_ps128(r01)); - store_f32::(dst_ptr1, _mm256_extractf128_ps::<1>(r01)); - store_f32::(dst_ptr2, _mm256_castps256_ps128(r23)); - store_f32::(dst_ptr3, _mm256_extractf128_ps::<1>(r23)); - store_f32::(dst_ptr4, _mm256_castps256_ps128(r45)); - store_f32::(dst_ptr5, _mm256_extractf128_ps::<1>(r45)); - - let d_a_1 = ((y + radius_64) & 1023) as usize; - let d_a_2 = ((y - radius_64) & 1023) as usize; - let d_i = (y & 1023) as usize; - - let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_i) as *mut f32); - let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_i) as *mut f32); - let sd2 = _mm256_load_ps(bf2.as_mut_ptr().add(d_i) as *mut f32); - - let sd_1_0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_a_1) as *mut f32); - let sd_1_1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_a_1) as *mut f32); - let sd_1_2 = _mm256_load_ps(bf2.as_mut_ptr().add(d_a_1) as *mut f32); - - let j0 = _mm256_sub_ps(sd0, sd_1_0); - let j1 = _mm256_sub_ps(sd1, sd_1_1); - let j2 = _mm256_sub_ps(sd2, sd_1_2); - - let sd_2_0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_a_2) as *mut f32); - let sd_2_1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_a_2) as *mut f32); - let sd_2_2 = _mm256_load_ps(bf2.as_mut_ptr().add(d_a_2) as *mut f32); - - let new_diff0 = _mm256_opt_fnmlsf_ps::(sd_2_0, j0, threes); - let new_diff1 = _mm256_opt_fnmlsf_ps::(sd_2_1, j1, threes); - let new_diff2 = _mm256_opt_fnmlsf_ps::(sd_2_2, j2, threes); - - diffs0 = _mm256_add_ps(diffs0, new_diff0); - diffs1 = _mm256_add_ps(diffs1, new_diff1); - diffs2 = _mm256_add_ps(diffs2, new_diff2); - } else if y + radius_64 >= 0 { - let a_i = (y & 1023) as usize; - let a_i_1 = ((y + radius_64) & 1023) as usize; - let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(a_i) as *mut f32); - let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(a_i) as *mut f32); - let sd2 = _mm256_load_ps(bf2.as_mut_ptr().add(a_i) as *mut f32); - - let sd_1_0 = _mm256_load_ps(bf0.as_mut_ptr().add(a_i_1) as *mut f32); - let sd_1_1 = _mm256_load_ps(bf1.as_mut_ptr().add(a_i_1) as *mut f32); - let sd_1_2 = _mm256_load_ps(bf2.as_mut_ptr().add(a_i_1) as *mut f32); - - diffs0 = _mm256_opt_fmlaf_ps::(diffs0, _mm256_sub_ps(sd0, sd_1_0), threes); - diffs1 = _mm256_opt_fmlaf_ps::(diffs1, _mm256_sub_ps(sd1, sd_1_1), threes); - diffs2 = _mm256_opt_fmlaf_ps::(diffs2, _mm256_sub_ps(sd2, sd_1_2), threes); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let sd2 = _mm256_load_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = _mm256_opt_fnmlaf_ps::(diffs0, sd0, threes); - diffs1 = _mm256_opt_fnmlaf_ps::(diffs1, sd1, threes); - diffs2 = _mm256_opt_fnmlaf_ps::(diffs2, sd2, threes); - } - - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut f32; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut f32; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut f32; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut f32; - let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut f32; - let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut f32; - - let pixel_color0 = load_f32::(s_ptr0); - let pixel_color1 = load_f32::(s_ptr1); - let pixel_color2 = load_f32::(s_ptr2); - let pixel_color3 = load_f32::(s_ptr3); - let pixel_color4 = load_f32::(s_ptr4); - let pixel_color5 = load_f32::(s_ptr5); - - let a_i = ((y + 2 * radius_64) & 1023) as usize; - - let px01 = - _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(pixel_color0), pixel_color1); - let px23 = - _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(pixel_color2), pixel_color3); - let px45 = - _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(pixel_color4), pixel_color5); - - _mm256_store_ps(bf0.as_mut_ptr().add(a_i) as *mut f32, px01); - _mm256_store_ps(bf1.as_mut_ptr().add(a_i) as *mut f32, px23); - _mm256_store_ps(bf2.as_mut_ptr().add(a_i) as *mut f32, px45); - - diffs0 = _mm256_add_ps(diffs0, px01); - diffs1 = _mm256_add_ps(diffs1, px23); - diffs2 = _mm256_add_ps(diffs2, px45); - - ders0 = _mm256_add_ps(ders0, diffs0); - ders1 = _mm256_add_ps(ders1, diffs1); - ders2 = _mm256_add_ps(ders2, diffs2); - - summs0 = _mm256_add_ps(summs0, ders0); - summs1 = _mm256_add_ps(summs1, ders1); - summs2 = _mm256_add_ps(summs2, ders2); - } - - xx += 6; - } - - for x in xx..width.min(end) as usize { - let mut diffs = _mm_setzero_ps(); - let mut ders = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); - - let current_px = x * CN; - - let start_y = 0 - 3 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_y = (y * (stride as i64)) as usize; - let bytes_offset = current_y + current_px; + unsafe { + let mut full_buffer = [AvxSseF32x8::default(); 1024 * 3]; + + let (bf0, rem) = full_buffer.split_at_mut(1024); + let (bf1, bf2) = rem.split_at_mut(1024); + + let height_wide = height as i64; + + let threes = _mm256_set1_ps(3.); + + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm256_set1_ps(weight); + + let mut xx = start as usize; + + while xx + 6 <= width.min(end) as usize { + let mut diffs0 = _mm256_setzero_ps(); + let mut diffs1 = _mm256_setzero_ps(); + let mut diffs2 = _mm256_setzero_ps(); + + let mut ders0 = _mm256_setzero_ps(); + let mut ders1 = _mm256_setzero_ps(); + let mut ders2 = _mm256_setzero_ps(); + + let mut summs0 = _mm256_setzero_ps(); + let mut summs1 = _mm256_setzero_ps(); + let mut summs2 = _mm256_setzero_ps(); + + let current_px0 = xx * CN; + let current_px1 = (xx + 1) * CN; + let current_px2 = (xx + 2) * CN; + let current_px3 = (xx + 3) * CN; + let current_px4 = (xx + 4) * CN; + let current_px5 = (xx + 5) * CN; + + let start_y = 0 - 3 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_y = (y * (stride as i64)) as usize; + + let r01 = _mm256_mul_ps(summs0, v_weight); + let r23 = _mm256_mul_ps(summs1, v_weight); + let r45 = _mm256_mul_ps(summs2, v_weight); + + let dst_ptr0 = + bytes.slice.as_ptr().add(current_y + current_px0) as *mut f32; + let dst_ptr1 = + bytes.slice.as_ptr().add(current_y + current_px1) as *mut f32; + let dst_ptr2 = + bytes.slice.as_ptr().add(current_y + current_px2) as *mut f32; + let dst_ptr3 = + bytes.slice.as_ptr().add(current_y + current_px3) as *mut f32; + let dst_ptr4 = + bytes.slice.as_ptr().add(current_y + current_px4) as *mut f32; + let dst_ptr5 = + bytes.slice.as_ptr().add(current_y + current_px5) as *mut f32; + + store_f32::(dst_ptr0, _mm256_castps256_ps128(r01)); + store_f32::(dst_ptr1, _mm256_extractf128_ps::<1>(r01)); + store_f32::(dst_ptr2, _mm256_castps256_ps128(r23)); + store_f32::(dst_ptr3, _mm256_extractf128_ps::<1>(r23)); + store_f32::(dst_ptr4, _mm256_castps256_ps128(r45)); + store_f32::(dst_ptr5, _mm256_extractf128_ps::<1>(r45)); + + let d_a_1 = ((y + radius_64) & 1023) as usize; + let d_a_2 = ((y - radius_64) & 1023) as usize; + let d_i = (y & 1023) as usize; + + let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_i) as *mut f32); + let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_i) as *mut f32); + let sd2 = _mm256_load_ps(bf2.as_mut_ptr().add(d_i) as *mut f32); + + let sd_1_0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_a_1) as *mut f32); + let sd_1_1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_a_1) as *mut f32); + let sd_1_2 = _mm256_load_ps(bf2.as_mut_ptr().add(d_a_1) as *mut f32); + + let j0 = _mm256_sub_ps(sd0, sd_1_0); + let j1 = _mm256_sub_ps(sd1, sd_1_1); + let j2 = _mm256_sub_ps(sd2, sd_1_2); + + let sd_2_0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_a_2) as *mut f32); + let sd_2_1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_a_2) as *mut f32); + let sd_2_2 = _mm256_load_ps(bf2.as_mut_ptr().add(d_a_2) as *mut f32); + + let new_diff0 = _mm256_opt_fnmlsf_ps::(sd_2_0, j0, threes); + let new_diff1 = _mm256_opt_fnmlsf_ps::(sd_2_1, j1, threes); + let new_diff2 = _mm256_opt_fnmlsf_ps::(sd_2_2, j2, threes); + + diffs0 = _mm256_add_ps(diffs0, new_diff0); + diffs1 = _mm256_add_ps(diffs1, new_diff1); + diffs2 = _mm256_add_ps(diffs2, new_diff2); + } else if y + radius_64 >= 0 { + let a_i = (y & 1023) as usize; + let a_i_1 = ((y + radius_64) & 1023) as usize; + let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(a_i) as *mut f32); + let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(a_i) as *mut f32); + let sd2 = _mm256_load_ps(bf2.as_mut_ptr().add(a_i) as *mut f32); + + let sd_1_0 = _mm256_load_ps(bf0.as_mut_ptr().add(a_i_1) as *mut f32); + let sd_1_1 = _mm256_load_ps(bf1.as_mut_ptr().add(a_i_1) as *mut f32); + let sd_1_2 = _mm256_load_ps(bf2.as_mut_ptr().add(a_i_1) as *mut f32); + + diffs0 = + _mm256_opt_fmlaf_ps::(diffs0, _mm256_sub_ps(sd0, sd_1_0), threes); + diffs1 = + _mm256_opt_fmlaf_ps::(diffs1, _mm256_sub_ps(sd1, sd_1_1), threes); + diffs2 = + _mm256_opt_fmlaf_ps::(diffs2, _mm256_sub_ps(sd2, sd_1_2), threes); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let sd2 = _mm256_load_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm256_opt_fnmlaf_ps::(diffs0, sd0, threes); + diffs1 = _mm256_opt_fnmlaf_ps::(diffs1, sd1, threes); + diffs2 = _mm256_opt_fnmlaf_ps::(diffs2, sd2, threes); + } + + let next_row_y = + clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut f32; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut f32; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut f32; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut f32; + let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut f32; + let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut f32; + + let pixel_color0 = load_f32::(s_ptr0); + let pixel_color1 = load_f32::(s_ptr1); + let pixel_color2 = load_f32::(s_ptr2); + let pixel_color3 = load_f32::(s_ptr3); + let pixel_color4 = load_f32::(s_ptr4); + let pixel_color5 = load_f32::(s_ptr5); + + let a_i = ((y + 2 * radius_64) & 1023) as usize; + + let px01 = _mm256_insertf128_ps::<1>( + _mm256_castps128_ps256(pixel_color0), + pixel_color1, + ); + let px23 = _mm256_insertf128_ps::<1>( + _mm256_castps128_ps256(pixel_color2), + pixel_color3, + ); + let px45 = _mm256_insertf128_ps::<1>( + _mm256_castps128_ps256(pixel_color4), + pixel_color5, + ); - let pixel = _mm_mul_ps(summs, _mm256_castps256_ps128(v_weight)); - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; - store_f32::(dst_ptr, pixel); + _mm256_store_ps(bf0.as_mut_ptr().add(a_i) as *mut f32, px01); + _mm256_store_ps(bf1.as_mut_ptr().add(a_i) as *mut f32, px23); + _mm256_store_ps(bf2.as_mut_ptr().add(a_i) as *mut f32, px45); - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + diffs0 = _mm256_add_ps(diffs0, px01); + diffs1 = _mm256_add_ps(diffs1, px23); + diffs2 = _mm256_add_ps(diffs2, px45); - let buf_ptr = bf0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let stored = _mm_load_ps(buf_ptr); + ders0 = _mm256_add_ps(ders0, diffs0); + ders1 = _mm256_add_ps(ders1, diffs1); + ders2 = _mm256_add_ps(ders2, diffs2); - let buf_ptr_1 = bf0.as_mut_ptr().add(d_arr_index_1) as *mut f32; - let stored_1 = _mm_load_ps(buf_ptr_1); + summs0 = _mm256_add_ps(summs0, ders0); + summs1 = _mm256_add_ps(summs1, ders1); + summs2 = _mm256_add_ps(summs2, ders2); + } - let buf_ptr_2 = bf0.as_mut_ptr().add(d_arr_index_2) as *mut f32; - let stored_2 = _mm_load_ps(buf_ptr_2); + xx += 6; + } - let new_diff = _mm_opt_fnmlsf_ps::( - stored_2, - _mm_sub_ps(stored, stored_1), - _mm256_castps256_ps128(threes), - ); - diffs = _mm_add_ps(diffs, new_diff); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; + for x in xx..width.min(end) as usize { + let mut diffs = _mm_setzero_ps(); + let mut ders = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); + + let current_px = x * CN; + + let start_y = 0 - 3 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_y = (y * (stride as i64)) as usize; + let bytes_offset = current_y + current_px; + + let pixel = _mm_mul_ps(summs, _mm256_castps256_ps128(v_weight)); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; + store_f32::(dst_ptr, pixel); + + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let buf_ptr = bf0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let stored = _mm_load_ps(buf_ptr); + + let buf_ptr_1 = bf0.as_mut_ptr().add(d_arr_index_1) as *mut f32; + let stored_1 = _mm_load_ps(buf_ptr_1); + + let buf_ptr_2 = bf0.as_mut_ptr().add(d_arr_index_2) as *mut f32; + let stored_2 = _mm_load_ps(buf_ptr_2); + + let new_diff = _mm_opt_fnmlsf_ps::( + stored_2, + _mm_sub_ps(stored, stored_1), + _mm256_castps256_ps128(threes), + ); + diffs = _mm_add_ps(diffs, new_diff); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_load_ps(buf_ptr); + + let buf_ptr_1 = bf0.get_unchecked_mut(arr_index_1).0.as_mut_ptr(); + let stored_1 = _mm_load_ps(buf_ptr_1); + + diffs = _mm_opt_fmlaf_ps::( + diffs, + _mm_sub_ps(stored, stored_1), + _mm256_castps256_ps128(threes), + ); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_load_ps(buf_ptr); + diffs = + _mm_opt_fnmlaf_ps::(diffs, stored, _mm256_castps256_ps128(threes)); + } + + let next_row_y = + clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + let next_row_x = x * CN; + + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f32; + + let pixel_color = load_f32::(s_ptr); + + let arr_index = ((y + 2 * radius_64) & 1023) as usize; let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_load_ps(buf_ptr); - - let buf_ptr_1 = bf0.get_unchecked_mut(arr_index_1).0.as_mut_ptr(); - let stored_1 = _mm_load_ps(buf_ptr_1); - diffs = _mm_opt_fmlaf_ps::( - diffs, - _mm_sub_ps(stored, stored_1), - _mm256_castps256_ps128(threes), - ); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_load_ps(buf_ptr); - diffs = _mm_opt_fnmlaf_ps::(diffs, stored, _mm256_castps256_ps128(threes)); + diffs = _mm_add_ps(diffs, pixel_color); + ders = _mm_add_ps(ders, diffs); + summs = _mm_add_ps(summs, ders); + _mm_store_ps(buf_ptr, pixel_color); } - - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - let next_row_x = x * CN; - - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f32; - - let pixel_color = load_f32::(s_ptr); - - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - - diffs = _mm_add_ps(diffs, pixel_color); - ders = _mm_add_ps(ders, diffs); - summs = _mm_add_ps(summs, ders); - _mm_store_ps(buf_ptr, pixel_color); } } } @@ -365,7 +384,7 @@ pub(crate) fn fgn_horizontal_pass_avx_f32( } #[target_feature(enable = "avx2")] -unsafe fn fgn_horizontal_pass_avx_f32_def( +fn fgn_horizontal_pass_avx_f32_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -380,7 +399,7 @@ unsafe fn fgn_horizontal_pass_avx_f32_def( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn fgn_horizontal_pass_avx_f32_fma( +fn fgn_horizontal_pass_avx_f32_fma( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -399,7 +418,7 @@ struct HorizontalAvxF32Executor {} impl HorizontalAvxF32Executor { #[inline(always)] - unsafe fn horizontal_pass( + fn horizontal_pass( &self, bytes: &UnsafeSlice, stride: u32, @@ -410,202 +429,215 @@ impl HorizontalAvxF32Executor { end: u32, edge_mode: EdgeMode, ) { - let mut bf0 = Box::new([AvxSseF32x8::default(); 1024]); - let mut bf1 = Box::new([AvxSseF32x8::default(); 1024]); + unsafe { + let mut full_scratch = [AvxSseF32x8::default(); 2048]; + let (bf0, bf1) = full_scratch.split_at_mut(1024); - let width_wide = width as i64; + let width_wide = width as i64; - let threes = _mm256_set1_ps(3.); + let threes = _mm256_set1_ps(3.); - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm256_set1_ps(weight); + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm256_set1_ps(weight); - let mut yy = start as usize; + let mut yy = start as usize; - while yy + 4 < height.min(end) as usize { - let mut diffs0 = _mm256_setzero_ps(); - let mut diffs1 = _mm256_setzero_ps(); + while yy + 4 <= height.min(end) as usize { + let mut diffs0 = _mm256_setzero_ps(); + let mut diffs1 = _mm256_setzero_ps(); - let mut ders0 = _mm256_setzero_ps(); - let mut ders1 = _mm256_setzero_ps(); + let mut ders0 = _mm256_setzero_ps(); + let mut ders1 = _mm256_setzero_ps(); - let mut summs0 = _mm256_setzero_ps(); - let mut summs1 = _mm256_setzero_ps(); + let mut summs0 = _mm256_setzero_ps(); + let mut summs1 = _mm256_setzero_ps(); - let start_x = 0 - 3 * radius_64; + let start_x = 0 - 3 * radius_64; - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = (((yy + 1) as i64) * (stride as i64)) as usize; - let current_y2 = (((yy + 2) as i64) * (stride as i64)) as usize; - let current_y3 = (((yy + 3) as i64) * (stride as i64)) as usize; + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = (((yy + 1) as i64) * (stride as i64)) as usize; + let current_y2 = (((yy + 2) as i64) * (stride as i64)) as usize; + let current_y3 = (((yy + 3) as i64) * (stride as i64)) as usize; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; - let r01 = _mm256_mul_ps(summs0, v_weight); - let r23 = _mm256_mul_ps(summs1, v_weight); + let r01 = _mm256_mul_ps(summs0, v_weight); + let r23 = _mm256_mul_ps(summs1, v_weight); - let dst_ptr0 = bytes.slice.as_ptr().add(current_y0 + current_px) as *mut f32; - let dst_ptr1 = bytes.slice.as_ptr().add(current_y1 + current_px) as *mut f32; - let dst_ptr2 = bytes.slice.as_ptr().add(current_y2 + current_px) as *mut f32; - let dst_ptr3 = bytes.slice.as_ptr().add(current_y3 + current_px) as *mut f32; + let dst_ptr0 = + bytes.slice.as_ptr().add(current_y0 + current_px) as *mut f32; + let dst_ptr1 = + bytes.slice.as_ptr().add(current_y1 + current_px) as *mut f32; + let dst_ptr2 = + bytes.slice.as_ptr().add(current_y2 + current_px) as *mut f32; + let dst_ptr3 = + bytes.slice.as_ptr().add(current_y3 + current_px) as *mut f32; - store_f32::(dst_ptr0, _mm256_castps256_ps128(r01)); - store_f32::(dst_ptr1, _mm256_extractf128_ps::<1>(r01)); - store_f32::(dst_ptr2, _mm256_castps256_ps128(r23)); - store_f32::(dst_ptr3, _mm256_extractf128_ps::<1>(r23)); + store_f32::(dst_ptr0, _mm256_castps256_ps128(r01)); + store_f32::(dst_ptr1, _mm256_extractf128_ps::<1>(r01)); + store_f32::(dst_ptr2, _mm256_castps256_ps128(r23)); + store_f32::(dst_ptr3, _mm256_extractf128_ps::<1>(r23)); - let d_a_1 = ((x + radius_64) & 1023) as usize; - let d_a_2 = ((x - radius_64) & 1023) as usize; - let d_i = (x & 1023) as usize; + let d_a_1 = ((x + radius_64) & 1023) as usize; + let d_a_2 = ((x - radius_64) & 1023) as usize; + let d_i = (x & 1023) as usize; - let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_i) as *mut f32); - let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_i) as *mut f32); + let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_i) as *mut f32); + let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_i) as *mut f32); - let sd_1_0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_a_1) as *mut f32); - let sd_1_1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_a_1) as *mut f32); + let sd_1_0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_a_1) as *mut f32); + let sd_1_1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_a_1) as *mut f32); - let j0 = _mm256_sub_ps(sd0, sd_1_0); - let j1 = _mm256_sub_ps(sd1, sd_1_1); + let j0 = _mm256_sub_ps(sd0, sd_1_0); + let j1 = _mm256_sub_ps(sd1, sd_1_1); - let sd_2_0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_a_2) as *mut f32); - let sd_2_1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_a_2) as *mut f32); + let sd_2_0 = _mm256_load_ps(bf0.as_mut_ptr().add(d_a_2) as *mut f32); + let sd_2_1 = _mm256_load_ps(bf1.as_mut_ptr().add(d_a_2) as *mut f32); - let new_diff0 = _mm256_opt_fnmlsf_ps::(sd_2_0, j0, threes); - let new_diff1 = _mm256_opt_fnmlsf_ps::(sd_2_1, j1, threes); + let new_diff0 = _mm256_opt_fnmlsf_ps::(sd_2_0, j0, threes); + let new_diff1 = _mm256_opt_fnmlsf_ps::(sd_2_1, j1, threes); - diffs0 = _mm256_add_ps(diffs0, new_diff0); - diffs1 = _mm256_add_ps(diffs1, new_diff1); - } else if x + radius_64 >= 0 { - let a_i = (x & 1023) as usize; - let a_i_1 = ((x + radius_64) & 1023) as usize; - let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(a_i) as *mut f32); - let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(a_i) as *mut f32); + diffs0 = _mm256_add_ps(diffs0, new_diff0); + diffs1 = _mm256_add_ps(diffs1, new_diff1); + } else if x + radius_64 >= 0 { + let a_i = (x & 1023) as usize; + let a_i_1 = ((x + radius_64) & 1023) as usize; + let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(a_i) as *mut f32); + let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(a_i) as *mut f32); - let sd_1_0 = _mm256_load_ps(bf0.as_mut_ptr().add(a_i_1) as *mut f32); - let sd_1_1 = _mm256_load_ps(bf1.as_mut_ptr().add(a_i_1) as *mut f32); + let sd_1_0 = _mm256_load_ps(bf0.as_mut_ptr().add(a_i_1) as *mut f32); + let sd_1_1 = _mm256_load_ps(bf1.as_mut_ptr().add(a_i_1) as *mut f32); - diffs0 = _mm256_opt_fmlaf_ps::(diffs0, _mm256_sub_ps(sd0, sd_1_0), threes); - diffs1 = _mm256_opt_fmlaf_ps::(diffs1, _mm256_sub_ps(sd1, sd_1_1), threes); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + diffs0 = + _mm256_opt_fmlaf_ps::(diffs0, _mm256_sub_ps(sd0, sd_1_0), threes); + diffs1 = + _mm256_opt_fmlaf_ps::(diffs1, _mm256_sub_ps(sd1, sd_1_1), threes); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let sd0 = _mm256_load_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let sd1 = _mm256_load_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - diffs0 = _mm256_opt_fnmlaf_ps::(diffs0, sd0, threes); - diffs1 = _mm256_opt_fnmlaf_ps::(diffs1, sd1, threes); - } + diffs0 = _mm256_opt_fnmlaf_ps::(diffs0, sd0, threes); + diffs1 = _mm256_opt_fnmlaf_ps::(diffs1, sd1, threes); + } - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut f32; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut f32; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut f32; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut f32; + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut f32; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut f32; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut f32; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut f32; - let pixel_color0 = load_f32::(s_ptr0); - let pixel_color1 = load_f32::(s_ptr1); - let pixel_color2 = load_f32::(s_ptr2); - let pixel_color3 = load_f32::(s_ptr3); + let pixel_color0 = load_f32::(s_ptr0); + let pixel_color1 = load_f32::(s_ptr1); + let pixel_color2 = load_f32::(s_ptr2); + let pixel_color3 = load_f32::(s_ptr3); - let a_i = ((x + 2 * radius_64) & 1023) as usize; + let a_i = ((x + 2 * radius_64) & 1023) as usize; + + let px01 = _mm256_insertf128_ps::<1>( + _mm256_castps128_ps256(pixel_color0), + pixel_color1, + ); + let px23 = _mm256_insertf128_ps::<1>( + _mm256_castps128_ps256(pixel_color2), + pixel_color3, + ); - let px01 = - _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(pixel_color0), pixel_color1); - let px23 = - _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(pixel_color2), pixel_color3); + _mm256_store_ps(bf0.as_mut_ptr().add(a_i) as *mut f32, px01); + _mm256_store_ps(bf1.as_mut_ptr().add(a_i) as *mut f32, px23); - _mm256_store_ps(bf0.as_mut_ptr().add(a_i) as *mut f32, px01); - _mm256_store_ps(bf1.as_mut_ptr().add(a_i) as *mut f32, px23); + diffs0 = _mm256_add_ps(diffs0, px01); + diffs1 = _mm256_add_ps(diffs1, px23); - diffs0 = _mm256_add_ps(diffs0, px01); - diffs1 = _mm256_add_ps(diffs1, px23); + ders0 = _mm256_add_ps(ders0, diffs0); + ders1 = _mm256_add_ps(ders1, diffs1); - ders0 = _mm256_add_ps(ders0, diffs0); - ders1 = _mm256_add_ps(ders1, diffs1); + summs0 = _mm256_add_ps(summs0, ders0); + summs1 = _mm256_add_ps(summs1, ders1); + } - summs0 = _mm256_add_ps(summs0, ders0); - summs1 = _mm256_add_ps(summs1, ders1); + yy += 4; } - yy += 4; - } - - for y in yy..height.min(end) as usize { - let mut diffs = _mm_setzero_ps(); - let mut ders = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); - - let current_y = ((y as i64) * (stride as i64)) as usize; - - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; - - let bytes_offset = current_y + current_px; - - let pixel = _mm_mul_ps(summs, _mm256_castps256_ps128(v_weight)); - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; - store_f32::(dst_ptr, pixel); - - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let buf_ptr = bf0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let stored = _mm_load_ps(buf_ptr); - - let buf_ptr_1 = bf0.get_unchecked_mut(d_arr_index_1).0.as_mut_ptr(); - let stored_1 = _mm_load_ps(buf_ptr_1); - - let buf_ptr_2 = bf0.get_unchecked_mut(d_arr_index_2).0.as_mut_ptr(); - let stored_2 = _mm_load_ps(buf_ptr_2); + for y in yy..height.min(end) as usize { + let mut diffs = _mm_setzero_ps(); + let mut ders = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); + + let current_y = ((y as i64) * (stride as i64)) as usize; + + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; + + let bytes_offset = current_y + current_px; + + let pixel = _mm_mul_ps(summs, _mm256_castps256_ps128(v_weight)); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; + store_f32::(dst_ptr, pixel); + + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let buf_ptr = bf0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let stored = _mm_load_ps(buf_ptr); + + let buf_ptr_1 = bf0.get_unchecked_mut(d_arr_index_1).0.as_mut_ptr(); + let stored_1 = _mm_load_ps(buf_ptr_1); + + let buf_ptr_2 = bf0.get_unchecked_mut(d_arr_index_2).0.as_mut_ptr(); + let stored_2 = _mm_load_ps(buf_ptr_2); + + let new_diff = _mm_opt_fnmlsf_ps::( + stored_2, + _mm_sub_ps(stored, stored_1), + _mm256_castps256_ps128(threes), + ); + diffs = _mm_add_ps(diffs, new_diff); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; + let stored = _mm_load_ps(buf_ptr); + + let buf_ptr_1 = bf0.as_mut_ptr().add(arr_index_1); + let stored_1 = _mm_load_ps(buf_ptr_1 as *const f32); + + diffs = _mm_opt_fmlaf_ps::( + diffs, + _mm_sub_ps(stored, stored_1), + _mm256_castps256_ps128(threes), + ); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = bf0.as_mut_ptr().add(arr_index); + let stored = _mm_load_ps(buf_ptr as *const f32); + diffs = + _mm_opt_fnmlaf_ps::(diffs, stored, _mm256_castps256_ps128(threes)); + } + + let next_row_y = y * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f32; + + let pixel_color = load_f32::(s_ptr); + + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let new_diff = _mm_opt_fnmlsf_ps::( - stored_2, - _mm_sub_ps(stored, stored_1), - _mm256_castps256_ps128(threes), - ); - diffs = _mm_add_ps(diffs, new_diff); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; - let stored = _mm_load_ps(buf_ptr); - - let buf_ptr_1 = bf0.as_mut_ptr().add(arr_index_1); - let stored_1 = _mm_load_ps(buf_ptr_1 as *const f32); - - diffs = _mm_opt_fmlaf_ps::( - diffs, - _mm_sub_ps(stored, stored_1), - _mm256_castps256_ps128(threes), - ); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = bf0.as_mut_ptr().add(arr_index); - let stored = _mm_load_ps(buf_ptr as *const f32); - diffs = _mm_opt_fnmlaf_ps::(diffs, stored, _mm256_castps256_ps128(threes)); + diffs = _mm_add_ps(diffs, pixel_color); + ders = _mm_add_ps(ders, diffs); + summs = _mm_add_ps(summs, ders); + _mm_store_ps(buf_ptr, pixel_color); } - - let next_row_y = y * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f32; - - let pixel_color = load_f32::(s_ptr); - - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - - diffs = _mm_add_ps(diffs, pixel_color); - ders = _mm_add_ps(ders, diffs); - summs = _mm_add_ps(summs, ders); - _mm_store_ps(buf_ptr, pixel_color); } } } diff --git a/src/avx/fast_gaussian_next_u16.rs b/src/avx/fast_gaussian_next_u16.rs index 9bae140a..d86efa8e 100644 --- a/src/avx/fast_gaussian_next_u16.rs +++ b/src/avx/fast_gaussian_next_u16.rs @@ -25,13 +25,13 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::avx::fast_gaussian_next::AvxSseI32x8; use crate::avx::utils::_mm256_mul_by_3_epi32; use crate::edge_mode::clamp_edge; use crate::sse::utils::load_u16_s32_fast; use crate::sse::{_mm_mul_by_3_epi32, store_u16_u32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::x86_64::*; pub(crate) fn fgn_vertical_pass_avx_u16( @@ -55,7 +55,7 @@ struct VerticalExecutionUnit {} impl VerticalExecutionUnit { #[target_feature(enable = "avx2")] - unsafe fn pass( + fn pass( &self, bytes: &UnsafeSlice, stride: u32, @@ -66,297 +66,307 @@ impl VerticalExecutionUnit { end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseI32x8::default(); 1024 * 3]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, buffer2) = rem.split_at_mut(1024); - - let height_wide = height as i64; - - let radius_64 = radius as i64; - - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm256_set1_ps(weight); - - let mut xx = start; - - while xx + 6 < width.min(end) { - let mut diffs0 = _mm256_setzero_si256(); - let mut diffs1 = _mm256_setzero_si256(); - let mut diffs2 = _mm256_setzero_si256(); - - let mut ders0 = _mm256_setzero_si256(); - let mut ders1 = _mm256_setzero_si256(); - let mut ders2 = _mm256_setzero_si256(); - - let mut summs0 = _mm256_setzero_si256(); - let mut summs1 = _mm256_setzero_si256(); - let mut summs2 = _mm256_setzero_si256(); - - let start_y = 0 - 3 * radius as i64; - - let current_px0 = (xx * CN as u32) as usize; - let current_px1 = ((xx + 1) * CN as u32) as usize; - let current_px2 = ((xx + 2) * CN as u32) as usize; - let current_px3 = ((xx + 3) * CN as u32) as usize; - let current_px4 = ((xx + 4) * CN as u32) as usize; - let current_px5 = ((xx + 5) * CN as u32) as usize; - - for y in start_y..height_wide { - if y >= 0 { - let ps01 = _mm256_cvtepi32_ps(summs0); - let ps23 = _mm256_cvtepi32_ps(summs1); - let ps45 = _mm256_cvtepi32_ps(summs2); - - let r01 = _mm256_mul_ps(ps01, v_weight); - let r23 = _mm256_mul_ps(ps23, v_weight); - let r45 = _mm256_mul_ps(ps45, v_weight); - - let e01 = _mm256_cvtps_epi32(r01); - let e23 = _mm256_cvtps_epi32(r23); - let e45 = _mm256_cvtps_epi32(r45); - - let prepared_px0 = _mm256_castsi256_si128(e01); - let prepared_px1 = _mm256_extracti128_si256::<1>(e01); - let prepared_px2 = _mm256_castsi256_si128(e23); - let prepared_px3 = _mm256_extracti128_si256::<1>(e23); - let prepared_px4 = _mm256_castsi256_si128(e45); - let prepared_px5 = _mm256_extracti128_si256::<1>(e45); - - let current_y = (y * (stride as i64)) as usize; - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px0); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px1); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px2); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px3); - let dst_ptr4 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px4); - let dst_ptr5 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px5); - - store_u16_u32::(dst_ptr0, prepared_px0); - store_u16_u32::(dst_ptr1, prepared_px1); - store_u16_u32::(dst_ptr2, prepared_px2); - store_u16_u32::(dst_ptr3, prepared_px3); - store_u16_u32::(dst_ptr4, prepared_px4); - store_u16_u32::(dst_ptr5, prepared_px5); - - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let stored0 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index) as *const _); - let stored1 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index) as *const _); - let stored2 = - _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index) as *const _); - - let stored_10 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_11 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_21 = - _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); - - let j0 = _mm256_sub_epi32(stored0, stored_10); - let j1 = _mm256_sub_epi32(stored1, stored_11); - let j2 = _mm256_sub_epi32(stored2, stored_21); - - let stored_20 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_21 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_22 = - _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); - - let k0 = _mm256_mul_by_3_epi32(j0); - let k1 = _mm256_mul_by_3_epi32(j1); - let k2 = _mm256_mul_by_3_epi32(j2); - - let new_diff0 = _mm256_sub_epi32(k0, stored_20); - let new_diff1 = _mm256_sub_epi32(k1, stored_21); - let new_diff2 = _mm256_sub_epi32(k2, stored_22); - - diffs0 = _mm256_add_epi32(diffs0, new_diff0); - diffs1 = _mm256_add_epi32(diffs1, new_diff1); - diffs2 = _mm256_add_epi32(diffs2, new_diff2); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; - - let stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + unsafe { + let mut full_buffer = [AvxSseI32x8::default(); 1024 * 3]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, buffer2) = rem.split_at_mut(1024); + + let height_wide = height as i64; + + let radius_64 = radius as i64; + + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm256_set1_ps(weight); + + let mut xx = start; + + while xx + 6 <= width.min(end) { + let mut diffs0 = _mm256_setzero_si256(); + let mut diffs1 = _mm256_setzero_si256(); + let mut diffs2 = _mm256_setzero_si256(); + + let mut ders0 = _mm256_setzero_si256(); + let mut ders1 = _mm256_setzero_si256(); + let mut ders2 = _mm256_setzero_si256(); + + let mut summs0 = _mm256_setzero_si256(); + let mut summs1 = _mm256_setzero_si256(); + let mut summs2 = _mm256_setzero_si256(); + + let start_y = 0 - 3 * radius as i64; + + let current_px0 = (xx * CN as u32) as usize; + let current_px1 = ((xx + 1) * CN as u32) as usize; + let current_px2 = ((xx + 2) * CN as u32) as usize; + let current_px3 = ((xx + 3) * CN as u32) as usize; + let current_px4 = ((xx + 4) * CN as u32) as usize; + let current_px5 = ((xx + 5) * CN as u32) as usize; + + for y in start_y..height_wide { + if y >= 0 { + let ps01 = _mm256_cvtepi32_ps(summs0); + let ps23 = _mm256_cvtepi32_ps(summs1); + let ps45 = _mm256_cvtepi32_ps(summs2); + + let r01 = _mm256_mul_ps(ps01, v_weight); + let r23 = _mm256_mul_ps(ps23, v_weight); + let r45 = _mm256_mul_ps(ps45, v_weight); + + let e01 = _mm256_cvtps_epi32(r01); + let e23 = _mm256_cvtps_epi32(r23); + let e45 = _mm256_cvtps_epi32(r45); + + let prepared_px0 = _mm256_castsi256_si128(e01); + let prepared_px1 = _mm256_extracti128_si256::<1>(e01); + let prepared_px2 = _mm256_castsi256_si128(e23); + let prepared_px3 = _mm256_extracti128_si256::<1>(e23); + let prepared_px4 = _mm256_castsi256_si128(e45); + let prepared_px5 = _mm256_extracti128_si256::<1>(e45); + + let current_y = (y * (stride as i64)) as usize; + + let dst_ptr0 = + (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px0); + let dst_ptr1 = + (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px1); + let dst_ptr2 = + (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px2); + let dst_ptr3 = + (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px3); + let dst_ptr4 = + (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px4); + let dst_ptr5 = + (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px5); + + store_u16_u32::(dst_ptr0, prepared_px0); + store_u16_u32::(dst_ptr1, prepared_px1); + store_u16_u32::(dst_ptr2, prepared_px2); + store_u16_u32::(dst_ptr3, prepared_px3); + store_u16_u32::(dst_ptr4, prepared_px4); + store_u16_u32::(dst_ptr5, prepared_px5); + + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let stored0 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index) as *const _); + let stored1 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index) as *const _); + let stored2 = + _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index) as *const _); + + let stored_10 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_11 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_21 = + _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); + + let j0 = _mm256_sub_epi32(stored0, stored_10); + let j1 = _mm256_sub_epi32(stored1, stored_11); + let j2 = _mm256_sub_epi32(stored2, stored_21); + + let stored_20 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_21 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_22 = + _mm256_load_si256(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); + + let k0 = _mm256_mul_by_3_epi32(j0); + let k1 = _mm256_mul_by_3_epi32(j1); + let k2 = _mm256_mul_by_3_epi32(j2); + + let new_diff0 = _mm256_sub_epi32(k0, stored_20); + let new_diff1 = _mm256_sub_epi32(k1, stored_21); + let new_diff2 = _mm256_sub_epi32(k2, stored_22); + + diffs0 = _mm256_add_epi32(diffs0, new_diff0); + diffs1 = _mm256_add_epi32(diffs1, new_diff1); + diffs2 = _mm256_add_epi32(diffs2, new_diff2); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + + let stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + let stored_10 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_11 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_21 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + + let new_diff0 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored0, stored_10)); + let new_diff1 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored1, stored_11)); + let new_diff2 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored2, stored_21)); + + diffs0 = _mm256_add_epi32(diffs0, new_diff0); + diffs1 = _mm256_add_epi32(diffs1, new_diff1); + diffs2 = _mm256_add_epi32(diffs2, new_diff2); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + + let stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm256_sub_epi32(diffs0, _mm256_mul_by_3_epi32(stored0)); + diffs1 = _mm256_sub_epi32(diffs1, _mm256_mul_by_3_epi32(stored1)); + diffs2 = _mm256_sub_epi32(diffs2, _mm256_mul_by_3_epi32(stored2)); + } + + let next_row_y = + clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u16; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u16; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u16; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u16; + let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut u16; + let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut u16; + + let pixel_color0 = load_u16_s32_fast::(s_ptr0); + let pixel_color1 = load_u16_s32_fast::(s_ptr1); + let pixel_color2 = load_u16_s32_fast::(s_ptr2); + let pixel_color3 = load_u16_s32_fast::(s_ptr3); + let pixel_color4 = load_u16_s32_fast::(s_ptr4); + let pixel_color5 = load_u16_s32_fast::(s_ptr5); + + let arr_index = ((y + 2 * radius_64) & 1023) as usize; + + let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); + + let px01 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color0), + pixel_color1, ); - let stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px23 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color2), + pixel_color3, ); - let stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px45 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color4), + pixel_color5, ); - let stored_10 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_11 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_21 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); + diffs0 = _mm256_add_epi32(diffs0, px01); + diffs1 = _mm256_add_epi32(diffs1, px23); + diffs2 = _mm256_add_epi32(diffs2, px45); - let new_diff0 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored0, stored_10)); - let new_diff1 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored1, stored_11)); - let new_diff2 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored2, stored_21)); + _mm256_store_si256(buf_ptr0 as *mut _, px01); + _mm256_store_si256(buf_ptr1 as *mut _, px23); + _mm256_store_si256(buf_ptr2 as *mut _, px45); - diffs0 = _mm256_add_epi32(diffs0, new_diff0); - diffs1 = _mm256_add_epi32(diffs1, new_diff1); - diffs2 = _mm256_add_epi32(diffs2, new_diff2); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; + ders0 = _mm256_add_epi32(ders0, diffs0); + ders1 = _mm256_add_epi32(ders1, diffs1); + ders2 = _mm256_add_epi32(ders2, diffs2); - let stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); - let stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); - let stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); - - diffs0 = _mm256_sub_epi32(diffs0, _mm256_mul_by_3_epi32(stored0)); - diffs1 = _mm256_sub_epi32(diffs1, _mm256_mul_by_3_epi32(stored1)); - diffs2 = _mm256_sub_epi32(diffs2, _mm256_mul_by_3_epi32(stored2)); + summs0 = _mm256_add_epi32(summs0, ders0); + summs1 = _mm256_add_epi32(summs1, ders1); + summs2 = _mm256_add_epi32(summs2, ders2); } - - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u16; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u16; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u16; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u16; - let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut u16; - let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut u16; - - let pixel_color0 = load_u16_s32_fast::(s_ptr0); - let pixel_color1 = load_u16_s32_fast::(s_ptr1); - let pixel_color2 = load_u16_s32_fast::(s_ptr2); - let pixel_color3 = load_u16_s32_fast::(s_ptr3); - let pixel_color4 = load_u16_s32_fast::(s_ptr4); - let pixel_color5 = load_u16_s32_fast::(s_ptr5); - - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - - let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); - - let px01 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color0), - pixel_color1, - ); - let px23 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color2), - pixel_color3, - ); - let px45 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color4), - pixel_color5, - ); - - diffs0 = _mm256_add_epi32(diffs0, px01); - diffs1 = _mm256_add_epi32(diffs1, px23); - diffs2 = _mm256_add_epi32(diffs2, px45); - - _mm256_store_si256(buf_ptr0 as *mut _, px01); - _mm256_store_si256(buf_ptr1 as *mut _, px23); - _mm256_store_si256(buf_ptr2 as *mut _, px45); - - ders0 = _mm256_add_epi32(ders0, diffs0); - ders1 = _mm256_add_epi32(ders1, diffs1); - ders2 = _mm256_add_epi32(ders2, diffs2); - - summs0 = _mm256_add_epi32(summs0, ders0); - summs1 = _mm256_add_epi32(summs1, ders1); - summs2 = _mm256_add_epi32(summs2, ders2); + xx += 6; } - xx += 6; - } - for x in xx..width.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut ders = _mm_setzero_si128(); - let mut summs = _mm_setzero_si128(); + for x in xx..width.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut ders = _mm_setzero_si128(); + let mut summs = _mm_setzero_si128(); - let start_y = 0 - 3 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_px = (x * CN as u32) as usize; + let start_y = 0 - 3 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_px = (x * CN as u32) as usize; - let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps( - _mm_cvtepi32_ps(summs), - _mm256_castps256_ps128(v_weight), - )); + let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps( + _mm_cvtepi32_ps(summs), + _mm256_castps256_ps128(v_weight), + )); - let current_y = (y * (stride as i64)) as usize; + let current_y = (y * (stride as i64)) as usize; - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); - store_u16_u32::(dst_ptr, prepared_px_s32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); + store_u16_u32::(dst_ptr, prepared_px_s32); - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.as_mut_ptr().add(d_arr_index_1); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.as_mut_ptr().add(d_arr_index_1); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let buf_ptr_2 = buffer0.as_mut_ptr().add(d_arr_index_2); - let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); + let buf_ptr_2 = buffer0.as_mut_ptr().add(d_arr_index_2); + let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); - let new_diff = _mm_sub_epi32( - _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), - stored_2, - ); - diffs = _mm_add_epi32(diffs, new_diff); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let new_diff = _mm_sub_epi32( + _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), + stored_2, + ); + diffs = _mm_add_epi32(diffs, new_diff); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr(); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr(); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); + let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); - diffs = _mm_add_epi32(diffs, new_diff); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); - } + diffs = _mm_add_epi32(diffs, new_diff); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); + diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); + } - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let next_row_y = + clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u16; + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u16; - let pixel_color = load_u16_s32_fast::(s_ptr); + let pixel_color = load_u16_s32_fast::(s_ptr); - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((y + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); - ders = _mm_add_epi32(ders, diffs); - summs = _mm_add_epi32(summs, ders); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + diffs = _mm_add_epi32(diffs, pixel_color); + ders = _mm_add_epi32(ders, diffs); + summs = _mm_add_epi32(summs, ders); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } @@ -383,7 +393,7 @@ struct HorizontalExecutionUnit {} impl HorizontalExecutionUnit { #[target_feature(enable = "avx2")] - unsafe fn pass( + fn pass( &self, bytes: &UnsafeSlice, stride: u32, @@ -394,243 +404,249 @@ impl HorizontalExecutionUnit { end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseI32x8::default(); 1024 * 2]); - - let (buffer0, buffer1) = full_buffer.split_at_mut(1024); - - let width_wide = width as i64; - - let radius_64 = radius as i64; - - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm256_set1_ps(weight); - - let mut yy = start; - - while yy + 6 < height.min(end) { - let mut diffs0 = _mm256_setzero_si256(); - let mut diffs1 = _mm256_setzero_si256(); - - let mut ders0 = _mm256_setzero_si256(); - let mut ders1 = _mm256_setzero_si256(); - - let mut summs0 = _mm256_setzero_si256(); - let mut summs1 = _mm256_setzero_si256(); - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; - let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; - let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + unsafe { + let mut full_buffer = [AvxSseI32x8::default(); 1024 * 2]; - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + let (buffer0, buffer1) = full_buffer.split_at_mut(1024); - let ps01 = _mm256_cvtepi32_ps(summs0); - let ps23 = _mm256_cvtepi32_ps(summs1); + let width_wide = width as i64; - let r01 = _mm256_mul_ps(ps01, v_weight); - let r23 = _mm256_mul_ps(ps23, v_weight); + let radius_64 = radius as i64; - let e01 = _mm256_cvtps_epi32(r01); - let e23 = _mm256_cvtps_epi32(r23); + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm256_set1_ps(weight); - let prepared_px0 = _mm256_castsi256_si128(e01); - let prepared_px1 = _mm256_extracti128_si256::<1>(e01); - let prepared_px2 = _mm256_castsi256_si128(e23); - let prepared_px3 = _mm256_extracti128_si256::<1>(e23); + let mut yy = start; - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y0 + current_px); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y1 + current_px); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y2 + current_px); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y3 + current_px); - - store_u16_u32::(dst_ptr0, prepared_px0); - store_u16_u32::(dst_ptr1, prepared_px1); - store_u16_u32::(dst_ptr2, prepared_px2); - store_u16_u32::(dst_ptr3, prepared_px3); - - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let stored0 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index) as *const _); - let stored1 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index) as *const _); - - let stored_10 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_11 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); - - let j0 = _mm256_sub_epi32(stored0, stored_10); - let j1 = _mm256_sub_epi32(stored1, stored_11); - - let stored_20 = - _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_21 = - _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); - - let k0 = _mm256_mul_by_3_epi32(j0); - let k1 = _mm256_mul_by_3_epi32(j1); - - let new_diff0 = _mm256_sub_epi32(k0, stored_20); - let new_diff1 = _mm256_sub_epi32(k1, stored_21); - - diffs0 = _mm256_add_epi32(diffs0, new_diff0); - diffs1 = _mm256_add_epi32(diffs1, new_diff1); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - - let stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + while yy + 6 <= height.min(end) { + let mut diffs0 = _mm256_setzero_si256(); + let mut diffs1 = _mm256_setzero_si256(); + + let mut ders0 = _mm256_setzero_si256(); + let mut ders1 = _mm256_setzero_si256(); + + let mut summs0 = _mm256_setzero_si256(); + let mut summs1 = _mm256_setzero_si256(); + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; + let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; + let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; + + let ps01 = _mm256_cvtepi32_ps(summs0); + let ps23 = _mm256_cvtepi32_ps(summs1); + + let r01 = _mm256_mul_ps(ps01, v_weight); + let r23 = _mm256_mul_ps(ps23, v_weight); + + let e01 = _mm256_cvtps_epi32(r01); + let e23 = _mm256_cvtps_epi32(r23); + + let prepared_px0 = _mm256_castsi256_si128(e01); + let prepared_px1 = _mm256_extracti128_si256::<1>(e01); + let prepared_px2 = _mm256_castsi256_si128(e23); + let prepared_px3 = _mm256_extracti128_si256::<1>(e23); + + let dst_ptr0 = + (bytes.slice.as_ptr() as *mut u16).add(current_y0 + current_px); + let dst_ptr1 = + (bytes.slice.as_ptr() as *mut u16).add(current_y1 + current_px); + let dst_ptr2 = + (bytes.slice.as_ptr() as *mut u16).add(current_y2 + current_px); + let dst_ptr3 = + (bytes.slice.as_ptr() as *mut u16).add(current_y3 + current_px); + + store_u16_u32::(dst_ptr0, prepared_px0); + store_u16_u32::(dst_ptr1, prepared_px1); + store_u16_u32::(dst_ptr2, prepared_px2); + store_u16_u32::(dst_ptr3, prepared_px3); + + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let stored0 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index) as *const _); + let stored1 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index) as *const _); + + let stored_10 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_11 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); + + let j0 = _mm256_sub_epi32(stored0, stored_10); + let j1 = _mm256_sub_epi32(stored1, stored_11); + + let stored_20 = + _mm256_load_si256(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_21 = + _mm256_load_si256(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); + + let k0 = _mm256_mul_by_3_epi32(j0); + let k1 = _mm256_mul_by_3_epi32(j1); + + let new_diff0 = _mm256_sub_epi32(k0, stored_20); + let new_diff1 = _mm256_sub_epi32(k1, stored_21); + + diffs0 = _mm256_add_epi32(diffs0, new_diff0); + diffs1 = _mm256_add_epi32(diffs1, new_diff1); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + + let stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + let stored_10 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_11 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + + let new_diff0 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored0, stored_10)); + let new_diff1 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored1, stored_11)); + + diffs0 = _mm256_add_epi32(diffs0, new_diff0); + diffs1 = _mm256_add_epi32(diffs1, new_diff1); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm256_sub_epi32(diffs0, _mm256_mul_by_3_epi32(stored0)); + diffs1 = _mm256_sub_epi32(diffs1, _mm256_mul_by_3_epi32(stored1)); + } + + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u16; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u16; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u16; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u16; + + let pixel_color0 = load_u16_s32_fast::(s_ptr0); + let pixel_color1 = load_u16_s32_fast::(s_ptr1); + let pixel_color2 = load_u16_s32_fast::(s_ptr2); + let pixel_color3 = load_u16_s32_fast::(s_ptr3); + + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); + + let px01 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color0), + pixel_color1, ); - let stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px23 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color2), + pixel_color3, ); - let stored_10 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_11 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); + diffs0 = _mm256_add_epi32(diffs0, px01); + diffs1 = _mm256_add_epi32(diffs1, px23); - let new_diff0 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored0, stored_10)); - let new_diff1 = _mm256_mul_by_3_epi32(_mm256_sub_epi32(stored1, stored_11)); + _mm256_store_si256(buf_ptr0 as *mut _, px01); + _mm256_store_si256(buf_ptr1 as *mut _, px23); - diffs0 = _mm256_add_epi32(diffs0, new_diff0); - diffs1 = _mm256_add_epi32(diffs1, new_diff1); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); - let stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, - ); + ders0 = _mm256_add_epi32(ders0, diffs0); + ders1 = _mm256_add_epi32(ders1, diffs1); - diffs0 = _mm256_sub_epi32(diffs0, _mm256_mul_by_3_epi32(stored0)); - diffs1 = _mm256_sub_epi32(diffs1, _mm256_mul_by_3_epi32(stored1)); + summs0 = _mm256_add_epi32(summs0, ders0); + summs1 = _mm256_add_epi32(summs1, ders1); } - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u16; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u16; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u16; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u16; + yy += 4; + } - let pixel_color0 = load_u16_s32_fast::(s_ptr0); - let pixel_color1 = load_u16_s32_fast::(s_ptr1); - let pixel_color2 = load_u16_s32_fast::(s_ptr2); - let pixel_color3 = load_u16_s32_fast::(s_ptr3); + for y in yy..height.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut ders = _mm_setzero_si128(); + let mut summs = _mm_setzero_si128(); - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let current_y = ((y as i64) * (stride as i64)) as usize; - let px01 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color0), - pixel_color1, - ); - let px23 = _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(pixel_color2), - pixel_color3, - ); + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; - diffs0 = _mm256_add_epi32(diffs0, px01); - diffs1 = _mm256_add_epi32(diffs1, px23); + let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps( + _mm_cvtepi32_ps(summs), + _mm256_castps256_ps128(v_weight), + )); - _mm256_store_si256(buf_ptr0 as *mut _, px01); - _mm256_store_si256(buf_ptr1 as *mut _, px23); + let bytes_offset = current_y + current_px; - ders0 = _mm256_add_epi32(ders0, diffs0); - ders1 = _mm256_add_epi32(ders1, diffs1); + let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); + store_u16_u32::(dst_ptr, prepared_px_s32); - summs0 = _mm256_add_epi32(summs0, ders0); - summs1 = _mm256_add_epi32(summs1, ders1); - } + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - yy += 4; - } + let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - for y in yy..height.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut ders = _mm_setzero_si128(); - let mut summs = _mm_setzero_si128(); + let buf_ptr_1 = buffer0.get_unchecked_mut(d_arr_index_1).0.as_mut_ptr(); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let current_y = ((y as i64) * (stride as i64)) as usize; + let buf_ptr_2 = buffer0.get_unchecked_mut(d_arr_index_2).0.as_mut_ptr(); + let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + let new_diff = _mm_sub_epi32( + _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), + stored_2, + ); + diffs = _mm_add_epi32(diffs, new_diff); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps( - _mm_cvtepi32_ps(summs), - _mm256_castps256_ps128(v_weight), - )); + let buf_ptr_1 = buffer0.as_mut_ptr().add(arr_index_1); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let bytes_offset = current_y + current_px; + let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); - let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); - store_u16_u32::(dst_ptr, prepared_px_s32); + diffs = _mm_add_epi32(diffs, new_diff); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.as_mut_ptr().add(arr_index); + let stored = _mm_load_si128(buf_ptr as *const __m128i); + diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); + } - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; - let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u16; - let buf_ptr_1 = buffer0.get_unchecked_mut(d_arr_index_1).0.as_mut_ptr(); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let pixel_color = load_u16_s32_fast::(s_ptr); - let buf_ptr_2 = buffer0.get_unchecked_mut(d_arr_index_2).0.as_mut_ptr(); - let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let new_diff = _mm_sub_epi32( - _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), - stored_2, - ); - diffs = _mm_add_epi32(diffs, new_diff); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; - let stored = _mm_load_si128(buf_ptr as *const __m128i); - - let buf_ptr_1 = buffer0.as_mut_ptr().add(arr_index_1); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - - let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); - - diffs = _mm_add_epi32(diffs, new_diff); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.as_mut_ptr().add(arr_index); - let stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); + diffs = _mm_add_epi32(diffs, pixel_color); + ders = _mm_add_epi32(ders, diffs); + summs = _mm_add_epi32(summs, ders); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); } - - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u16; - - let pixel_color = load_u16_s32_fast::(s_ptr); - - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - - diffs = _mm_add_epi32(diffs, pixel_color); - ders = _mm_add_epi32(ders, diffs); - summs = _mm_add_epi32(summs, ders); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); } } } diff --git a/src/avx/fast_gaussian_u16.rs b/src/avx/fast_gaussian_u16.rs index acd331fb..2ee4b663 100644 --- a/src/avx/fast_gaussian_u16.rs +++ b/src/avx/fast_gaussian_u16.rs @@ -25,12 +25,12 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::avx::fast_gaussian_next::AvxSseI32x8; use crate::edge_mode::clamp_edge; use crate::sse::store_u16_u32; use crate::sse::utils::load_u16_s32_fast; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::x86_64::*; pub(crate) fn fg_horizontal_pass_avx_u16( @@ -51,7 +51,7 @@ pub(crate) fn fg_horizontal_pass_avx_u16( } #[target_feature(enable = "avx2")] -unsafe fn fg_horizontal_pass_avx_u16_def( +fn fg_horizontal_pass_avx_u16_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -61,229 +61,237 @@ unsafe fn fg_horizontal_pass_avx_u16_def( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseI32x8::default(); 1024 * 3]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, buffer2) = rem.split_at_mut(1024); - - let initial_sum = ((radius * radius) >> 1) as i32; - - let radius_64 = radius as i64; - let width_wide = width as i64; - - let v_weight = _mm256_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let mut yy = start; - - while yy + 6 < height.min(end) { - let mut diffs0 = _mm256_setzero_si256(); - let mut diffs1 = _mm256_setzero_si256(); - let mut diffs2 = _mm256_setzero_si256(); - - let mut summs0 = _mm256_set1_epi32(initial_sum); - let mut summs1 = _mm256_set1_epi32(initial_sum); - let mut summs2 = _mm256_set1_epi32(initial_sum); - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; - let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; - let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; - let current_y4 = ((yy as i64 + 4) * (stride as i64)) as usize; - let current_y5 = ((yy as i64 + 5) * (stride as i64)) as usize; - - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = (x as u32 * CN as u32) as usize; - - let ps01 = _mm256_cvtepi32_ps(summs0); - let ps23 = _mm256_cvtepi32_ps(summs1); - let ps45 = _mm256_cvtepi32_ps(summs2); - - let r01 = _mm256_mul_ps(ps01, v_weight); - let r23 = _mm256_mul_ps(ps23, v_weight); - let r45 = _mm256_mul_ps(ps45, v_weight); - - let e01 = _mm256_cvtps_epi32(r01); - let e23 = _mm256_cvtps_epi32(r23); - let e45 = _mm256_cvtps_epi32(r45); - - let prepared_px0 = _mm256_castsi256_si128(e01); - let prepared_px1 = _mm256_extracti128_si256::<1>(e01); - let prepared_px2 = _mm256_castsi256_si128(e23); - let prepared_px3 = _mm256_extracti128_si256::<1>(e23); - let prepared_px4 = _mm256_castsi256_si128(e45); - let prepared_px5 = _mm256_extracti128_si256::<1>(e45); - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y0 + current_px); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y1 + current_px); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y2 + current_px); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y3 + current_px); - let dst_ptr4 = (bytes.slice.as_ptr() as *mut u16).add(current_y4 + current_px); - let dst_ptr5 = (bytes.slice.as_ptr() as *mut u16).add(current_y5 + current_px); - - store_u16_u32::(dst_ptr0, prepared_px0); - store_u16_u32::(dst_ptr1, prepared_px1); - store_u16_u32::(dst_ptr2, prepared_px2); - store_u16_u32::(dst_ptr3, prepared_px3); - store_u16_u32::(dst_ptr4, prepared_px4); - store_u16_u32::(dst_ptr5, prepared_px5); - - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let mut d_stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - let mut d_stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - let mut d_stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - - d_stored0 = _mm256_slli_epi32::<1>(d_stored0); - d_stored1 = _mm256_slli_epi32::<1>(d_stored1); - d_stored2 = _mm256_slli_epi32::<1>(d_stored2); - - let a_stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + unsafe { + let mut full_buffer = [AvxSseI32x8::default(); 1024 * 3]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, buffer2) = rem.split_at_mut(1024); + + let initial_sum = ((radius * radius) >> 1) as i32; + + let radius_64 = radius as i64; + let width_wide = width as i64; + + let v_weight = _mm256_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let mut yy = start; + + while yy + 6 <= height.min(end) { + let mut diffs0 = _mm256_setzero_si256(); + let mut diffs1 = _mm256_setzero_si256(); + let mut diffs2 = _mm256_setzero_si256(); + + let mut summs0 = _mm256_set1_epi32(initial_sum); + let mut summs1 = _mm256_set1_epi32(initial_sum); + let mut summs2 = _mm256_set1_epi32(initial_sum); + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; + let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; + let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + let current_y4 = ((yy as i64 + 4) * (stride as i64)) as usize; + let current_y5 = ((yy as i64 + 5) * (stride as i64)) as usize; + + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = (x as u32 * CN as u32) as usize; + + let ps01 = _mm256_cvtepi32_ps(summs0); + let ps23 = _mm256_cvtepi32_ps(summs1); + let ps45 = _mm256_cvtepi32_ps(summs2); + + let r01 = _mm256_mul_ps(ps01, v_weight); + let r23 = _mm256_mul_ps(ps23, v_weight); + let r45 = _mm256_mul_ps(ps45, v_weight); + + let e01 = _mm256_cvtps_epi32(r01); + let e23 = _mm256_cvtps_epi32(r23); + let e45 = _mm256_cvtps_epi32(r45); + + let prepared_px0 = _mm256_castsi256_si128(e01); + let prepared_px1 = _mm256_extracti128_si256::<1>(e01); + let prepared_px2 = _mm256_castsi256_si128(e23); + let prepared_px3 = _mm256_extracti128_si256::<1>(e23); + let prepared_px4 = _mm256_castsi256_si128(e45); + let prepared_px5 = _mm256_extracti128_si256::<1>(e45); + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y0 + current_px); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y1 + current_px); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y2 + current_px); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y3 + current_px); + let dst_ptr4 = (bytes.slice.as_ptr() as *mut u16).add(current_y4 + current_px); + let dst_ptr5 = (bytes.slice.as_ptr() as *mut u16).add(current_y5 + current_px); + + store_u16_u32::(dst_ptr0, prepared_px0); + store_u16_u32::(dst_ptr1, prepared_px1); + store_u16_u32::(dst_ptr2, prepared_px2); + store_u16_u32::(dst_ptr3, prepared_px3); + store_u16_u32::(dst_ptr4, prepared_px4); + store_u16_u32::(dst_ptr5, prepared_px5); + + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let mut d_stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + + d_stored0 = _mm256_slli_epi32::<1>(d_stored0); + d_stored1 = _mm256_slli_epi32::<1>(d_stored1); + d_stored2 = _mm256_slli_epi32::<1>(d_stored2); + + let a_stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm256_add_epi32(diffs0, _mm256_sub_epi32(a_stored0, d_stored0)); + diffs1 = _mm256_add_epi32(diffs1, _mm256_sub_epi32(a_stored1, d_stored1)); + diffs2 = _mm256_add_epi32(diffs2, _mm256_sub_epi32(a_stored2, d_stored2)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let mut stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let mut stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let mut stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + stored0 = _mm256_slli_epi32::<1>(stored0); + stored1 = _mm256_slli_epi32::<1>(stored1); + stored2 = _mm256_slli_epi32::<1>(stored2); + + diffs0 = _mm256_sub_epi32(diffs0, stored0); + diffs1 = _mm256_sub_epi32(diffs1, stored1); + diffs2 = _mm256_sub_epi32(diffs2, stored2); + } + + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u16; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u16; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u16; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u16; + let s_ptr4 = bytes.slice.as_ptr().add(current_y4 + next_row_px) as *mut u16; + let s_ptr5 = bytes.slice.as_ptr().add(current_y5 + next_row_px) as *mut u16; + + let pixel_color0 = load_u16_s32_fast::(s_ptr0); + let pixel_color1 = load_u16_s32_fast::(s_ptr1); + let pixel_color2 = load_u16_s32_fast::(s_ptr2); + let pixel_color3 = load_u16_s32_fast::(s_ptr3); + let pixel_color4 = load_u16_s32_fast::(s_ptr4); + let pixel_color5 = load_u16_s32_fast::(s_ptr5); + + let arr_index = ((x + radius_64) & 1023) as usize; + + let px01 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color0), + pixel_color1, ); - let a_stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px23 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color2), + pixel_color3, ); - let a_stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px45 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color4), + pixel_color5, ); - diffs0 = _mm256_add_epi32(diffs0, _mm256_sub_epi32(a_stored0, d_stored0)); - diffs1 = _mm256_add_epi32(diffs1, _mm256_sub_epi32(a_stored1, d_stored1)); - diffs2 = _mm256_add_epi32(diffs2, _mm256_sub_epi32(a_stored2, d_stored2)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let mut stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + _mm256_store_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px01, ); - let mut stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + _mm256_store_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px23, ); - let mut stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + _mm256_store_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px45, ); - stored0 = _mm256_slli_epi32::<1>(stored0); - stored1 = _mm256_slli_epi32::<1>(stored1); - stored2 = _mm256_slli_epi32::<1>(stored2); + diffs0 = _mm256_add_epi32(diffs0, px01); + diffs1 = _mm256_add_epi32(diffs1, px23); + diffs2 = _mm256_add_epi32(diffs2, px45); - diffs0 = _mm256_sub_epi32(diffs0, stored0); - diffs1 = _mm256_sub_epi32(diffs1, stored1); - diffs2 = _mm256_sub_epi32(diffs2, stored2); + summs0 = _mm256_add_epi32(summs0, diffs0); + summs1 = _mm256_add_epi32(summs1, diffs1); + summs2 = _mm256_add_epi32(summs2, diffs2); } - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u16; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u16; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u16; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u16; - let s_ptr4 = bytes.slice.as_ptr().add(current_y4 + next_row_px) as *mut u16; - let s_ptr5 = bytes.slice.as_ptr().add(current_y5 + next_row_px) as *mut u16; - - let pixel_color0 = load_u16_s32_fast::(s_ptr0); - let pixel_color1 = load_u16_s32_fast::(s_ptr1); - let pixel_color2 = load_u16_s32_fast::(s_ptr2); - let pixel_color3 = load_u16_s32_fast::(s_ptr3); - let pixel_color4 = load_u16_s32_fast::(s_ptr4); - let pixel_color5 = load_u16_s32_fast::(s_ptr5); - - let arr_index = ((x + radius_64) & 1023) as usize; - - let px01 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color0), pixel_color1); - let px23 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color2), pixel_color3); - let px45 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color4), pixel_color5); - - _mm256_store_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px01, - ); - _mm256_store_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px23, - ); - _mm256_store_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px45, - ); - - diffs0 = _mm256_add_epi32(diffs0, px01); - diffs1 = _mm256_add_epi32(diffs1, px23); - diffs2 = _mm256_add_epi32(diffs2, px45); - - summs0 = _mm256_add_epi32(summs0, diffs0); - summs1 = _mm256_add_epi32(summs1, diffs1); - summs2 = _mm256_add_epi32(summs2, diffs2); + yy += 6; } - yy += 6; - } + for y in yy..height.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut summs = _mm_set1_epi32(initial_sum); - for y in yy..height.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut summs = _mm_set1_epi32(initial_sum); + let current_y = ((y as i64) * (stride as i64)) as usize; - let current_y = ((y as i64) * (stride as i64)) as usize; + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = (x as u32 * CN as u32) as usize; - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = (x as u32 * CN as u32) as usize; + let pixel_f32 = + _mm_mul_ps(_mm_cvtepi32_ps(summs), _mm256_castps256_ps128(v_weight)); + let pixel_u32 = _mm_cvtps_epi32(pixel_f32); - let pixel_f32 = - _mm_mul_ps(_mm_cvtepi32_ps(summs), _mm256_castps256_ps128(v_weight)); - let pixel_u32 = _mm_cvtps_epi32(pixel_f32); + let bytes_offset = current_y + current_px; - let bytes_offset = current_y + current_px; + let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); + store_u16_u32::(dst_ptr, pixel_u32); - let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); - store_u16_u32::(dst_ptr, pixel_u32); + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let d_buf_ptr = buffer0.as_mut_ptr().add(d_arr_index) as *const i32; + let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); + d_stored = _mm_slli_epi32::<1>(d_stored); - let d_buf_ptr = buffer0.as_mut_ptr().add(d_arr_index) as *const i32; - let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); - d_stored = _mm_slli_epi32::<1>(d_stored); + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let a_stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let a_stored = _mm_load_si128(buf_ptr as *const __m128i); + diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let mut stored = _mm_load_si128(buf_ptr as *const __m128i); + stored = _mm_slli_epi32::<1>(stored); + diffs = _mm_sub_epi32(diffs, stored); + } - diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let mut stored = _mm_load_si128(buf_ptr as *const __m128i); - stored = _mm_slli_epi32::<1>(stored); - diffs = _mm_sub_epi32(diffs, stored); - } - - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u16; - let pixel_color = load_u16_s32_fast::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u16; + let pixel_color = load_u16_s32_fast::(s_ptr); - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); - summs = _mm_add_epi32(summs, diffs); + diffs = _mm_add_epi32(diffs, pixel_color); + summs = _mm_add_epi32(summs, diffs); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } @@ -306,7 +314,7 @@ pub(crate) fn fg_vertical_pass_avx_u16( } #[target_feature(enable = "avx2")] -unsafe fn fg_vertical_pass_avx_u16_def( +fn fg_vertical_pass_avx_u16_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -316,234 +324,242 @@ unsafe fn fg_vertical_pass_avx_u16_def( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([AvxSseI32x8::default(); 1024 * 3]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, buffer2) = rem.split_at_mut(1024); - - let initial_sum = ((radius * radius) >> 1) as i32; - - let height_wide = height as i64; - - let radius_64 = radius as i64; - - let v_weight = _mm256_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let mut xx = start; - - while xx + 6 < width.min(end) { - let mut diffs0 = _mm256_setzero_si256(); - let mut diffs1 = _mm256_setzero_si256(); - let mut diffs2 = _mm256_setzero_si256(); - - let mut summs0 = _mm256_set1_epi32(initial_sum); - let mut summs1 = _mm256_set1_epi32(initial_sum); - let mut summs2 = _mm256_set1_epi32(initial_sum); - - let start_y = 0 - 2 * radius as i64; - - let current_px0 = (xx * CN as u32) as usize; - let current_px1 = ((xx + 1) * CN as u32) as usize; - let current_px2 = ((xx + 2) * CN as u32) as usize; - let current_px3 = ((xx + 3) * CN as u32) as usize; - let current_px4 = ((xx + 4) * CN as u32) as usize; - let current_px5 = ((xx + 5) * CN as u32) as usize; - - for y in start_y..height_wide { - if y >= 0 { - let ps01 = _mm256_cvtepi32_ps(summs0); - let ps23 = _mm256_cvtepi32_ps(summs1); - let ps45 = _mm256_cvtepi32_ps(summs2); - - let r01 = _mm256_mul_ps(ps01, v_weight); - let r23 = _mm256_mul_ps(ps23, v_weight); - let r45 = _mm256_mul_ps(ps45, v_weight); - - let e01 = _mm256_cvtps_epi32(r01); - let e23 = _mm256_cvtps_epi32(r23); - let e45 = _mm256_cvtps_epi32(r45); - - let prepared_px0 = _mm256_castsi256_si128(e01); - let prepared_px1 = _mm256_extracti128_si256::<1>(e01); - let prepared_px2 = _mm256_castsi256_si128(e23); - let prepared_px3 = _mm256_extracti128_si256::<1>(e23); - let prepared_px4 = _mm256_castsi256_si128(e45); - let prepared_px5 = _mm256_extracti128_si256::<1>(e45); - - let current_y = (y * (stride as i64)) as usize; - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px0); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px1); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px2); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px3); - let dst_ptr4 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px4); - let dst_ptr5 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px5); - - store_u16_u32::(dst_ptr0, prepared_px0); - store_u16_u32::(dst_ptr1, prepared_px1); - store_u16_u32::(dst_ptr2, prepared_px2); - store_u16_u32::(dst_ptr3, prepared_px3); - store_u16_u32::(dst_ptr4, prepared_px4); - store_u16_u32::(dst_ptr5, prepared_px5); - - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let mut d_stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - let mut d_stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - let mut d_stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, - ); - - d_stored0 = _mm256_slli_epi32::<1>(d_stored0); - d_stored1 = _mm256_slli_epi32::<1>(d_stored1); - d_stored2 = _mm256_slli_epi32::<1>(d_stored2); - - let a_stored0 = _mm256_load_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + unsafe { + let mut full_buffer = [AvxSseI32x8::default(); 1024 * 3]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, buffer2) = rem.split_at_mut(1024); + + let initial_sum = ((radius * radius) >> 1) as i32; + + let height_wide = height as i64; + + let radius_64 = radius as i64; + + let v_weight = _mm256_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let mut xx = start; + + while xx + 6 <= width.min(end) { + let mut diffs0 = _mm256_setzero_si256(); + let mut diffs1 = _mm256_setzero_si256(); + let mut diffs2 = _mm256_setzero_si256(); + + let mut summs0 = _mm256_set1_epi32(initial_sum); + let mut summs1 = _mm256_set1_epi32(initial_sum); + let mut summs2 = _mm256_set1_epi32(initial_sum); + + let start_y = 0 - 2 * radius as i64; + + let current_px0 = (xx * CN as u32) as usize; + let current_px1 = ((xx + 1) * CN as u32) as usize; + let current_px2 = ((xx + 2) * CN as u32) as usize; + let current_px3 = ((xx + 3) * CN as u32) as usize; + let current_px4 = ((xx + 4) * CN as u32) as usize; + let current_px5 = ((xx + 5) * CN as u32) as usize; + + for y in start_y..height_wide { + if y >= 0 { + let ps01 = _mm256_cvtepi32_ps(summs0); + let ps23 = _mm256_cvtepi32_ps(summs1); + let ps45 = _mm256_cvtepi32_ps(summs2); + + let r01 = _mm256_mul_ps(ps01, v_weight); + let r23 = _mm256_mul_ps(ps23, v_weight); + let r45 = _mm256_mul_ps(ps45, v_weight); + + let e01 = _mm256_cvtps_epi32(r01); + let e23 = _mm256_cvtps_epi32(r23); + let e45 = _mm256_cvtps_epi32(r45); + + let prepared_px0 = _mm256_castsi256_si128(e01); + let prepared_px1 = _mm256_extracti128_si256::<1>(e01); + let prepared_px2 = _mm256_castsi256_si128(e23); + let prepared_px3 = _mm256_extracti128_si256::<1>(e23); + let prepared_px4 = _mm256_castsi256_si128(e45); + let prepared_px5 = _mm256_extracti128_si256::<1>(e45); + + let current_y = (y * (stride as i64)) as usize; + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px0); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px1); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px2); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px3); + let dst_ptr4 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px4); + let dst_ptr5 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px5); + + store_u16_u32::(dst_ptr0, prepared_px0); + store_u16_u32::(dst_ptr1, prepared_px1); + store_u16_u32::(dst_ptr2, prepared_px2); + store_u16_u32::(dst_ptr3, prepared_px3); + store_u16_u32::(dst_ptr4, prepared_px4); + store_u16_u32::(dst_ptr5, prepared_px5); + + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let mut d_stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + + d_stored0 = _mm256_slli_epi32::<1>(d_stored0); + d_stored1 = _mm256_slli_epi32::<1>(d_stored1); + d_stored2 = _mm256_slli_epi32::<1>(d_stored2); + + let a_stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm256_add_epi32(diffs0, _mm256_sub_epi32(a_stored0, d_stored0)); + diffs1 = _mm256_add_epi32(diffs1, _mm256_sub_epi32(a_stored1, d_stored1)); + diffs2 = _mm256_add_epi32(diffs2, _mm256_sub_epi32(a_stored2, d_stored2)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + + let mut stored0 = _mm256_load_si256( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + let mut stored1 = _mm256_load_si256( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + let mut stored2 = _mm256_load_si256( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + + stored0 = _mm256_slli_epi32::<1>(stored0); + stored1 = _mm256_slli_epi32::<1>(stored1); + stored2 = _mm256_slli_epi32::<1>(stored2); + + diffs0 = _mm256_sub_epi32(diffs0, stored0); + diffs1 = _mm256_sub_epi32(diffs1, stored1); + diffs2 = _mm256_sub_epi32(diffs2, stored2); + } + + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u16; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u16; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u16; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u16; + let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut u16; + let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut u16; + + let pixel_color0 = load_u16_s32_fast::(s_ptr0); + let pixel_color1 = load_u16_s32_fast::(s_ptr1); + let pixel_color2 = load_u16_s32_fast::(s_ptr2); + let pixel_color3 = load_u16_s32_fast::(s_ptr3); + let pixel_color4 = load_u16_s32_fast::(s_ptr4); + let pixel_color5 = load_u16_s32_fast::(s_ptr5); + + let arr_index = ((y + radius_64) & 1023) as usize; + + let px01 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color0), + pixel_color1, ); - let a_stored1 = _mm256_load_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px23 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color2), + pixel_color3, ); - let a_stored2 = _mm256_load_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + let px45 = _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(pixel_color4), + pixel_color5, ); - diffs0 = _mm256_add_epi32(diffs0, _mm256_sub_epi32(a_stored0, d_stored0)); - diffs1 = _mm256_add_epi32(diffs1, _mm256_sub_epi32(a_stored1, d_stored1)); - diffs2 = _mm256_add_epi32(diffs2, _mm256_sub_epi32(a_stored2, d_stored2)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; + diffs0 = _mm256_add_epi32(diffs0, px01); + diffs1 = _mm256_add_epi32(diffs1, px23); + diffs2 = _mm256_add_epi32(diffs2, px45); - let mut stored0 = _mm256_load_si256( + _mm256_store_si256( buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px01, ); - let mut stored1 = _mm256_load_si256( + _mm256_store_si256( buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px23, ); - let mut stored2 = _mm256_load_si256( + _mm256_store_si256( buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + px45, ); - stored0 = _mm256_slli_epi32::<1>(stored0); - stored1 = _mm256_slli_epi32::<1>(stored1); - stored2 = _mm256_slli_epi32::<1>(stored2); - - diffs0 = _mm256_sub_epi32(diffs0, stored0); - diffs1 = _mm256_sub_epi32(diffs1, stored1); - diffs2 = _mm256_sub_epi32(diffs2, stored2); + summs0 = _mm256_add_epi32(summs0, diffs0); + summs1 = _mm256_add_epi32(summs1, diffs1); + summs2 = _mm256_add_epi32(summs2, diffs2); } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u16; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u16; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u16; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u16; - let s_ptr4 = bytes.slice.as_ptr().add(next_row_y + current_px4) as *mut u16; - let s_ptr5 = bytes.slice.as_ptr().add(next_row_y + current_px5) as *mut u16; - - let pixel_color0 = load_u16_s32_fast::(s_ptr0); - let pixel_color1 = load_u16_s32_fast::(s_ptr1); - let pixel_color2 = load_u16_s32_fast::(s_ptr2); - let pixel_color3 = load_u16_s32_fast::(s_ptr3); - let pixel_color4 = load_u16_s32_fast::(s_ptr4); - let pixel_color5 = load_u16_s32_fast::(s_ptr5); - - let arr_index = ((y + radius_64) & 1023) as usize; - - let px01 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color0), pixel_color1); - let px23 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color2), pixel_color3); - let px45 = - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(pixel_color4), pixel_color5); - - diffs0 = _mm256_add_epi32(diffs0, px01); - diffs1 = _mm256_add_epi32(diffs1, px23); - diffs2 = _mm256_add_epi32(diffs2, px45); - - _mm256_store_si256( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px01, - ); - _mm256_store_si256( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px23, - ); - _mm256_store_si256( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - px45, - ); - - summs0 = _mm256_add_epi32(summs0, diffs0); - summs1 = _mm256_add_epi32(summs1, diffs1); - summs2 = _mm256_add_epi32(summs2, diffs2); + xx += 6; } - xx += 6; - } - - for x in xx..width.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut summs = _mm_set1_epi32(initial_sum); + for x in xx..width.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut summs = _mm_set1_epi32(initial_sum); - let current_px = (x * CN as u32) as usize; + let current_px = (x * CN as u32) as usize; - let start_y = 0 - 2 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let pixel_f32 = - _mm_mul_ps(_mm_cvtepi32_ps(summs), _mm256_castps256_ps128(v_weight)); + let start_y = 0 - 2 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let pixel_f32 = + _mm_mul_ps(_mm_cvtepi32_ps(summs), _mm256_castps256_ps128(v_weight)); - let current_y = (y * (stride as i64)) as usize; + let current_y = (y * (stride as i64)) as usize; - let pixel_u32 = _mm_cvtps_epi32(pixel_f32); + let pixel_u32 = _mm_cvtps_epi32(pixel_f32); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); - store_u16_u32::(dst_ptr, pixel_u32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); + store_u16_u32::(dst_ptr, pixel_u32); - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let d_buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); - d_stored = _mm_slli_epi32::<1>(d_stored); + let d_buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); + d_stored = _mm_slli_epi32::<1>(d_stored); - let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; - let a_stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; + let a_stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let mut stored = _mm_load_si128(buf_ptr as *const __m128i); - stored = _mm_slli_epi32::<1>(stored); - diffs = _mm_sub_epi32(diffs, stored); - } + diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let mut stored = _mm_load_si128(buf_ptr as *const __m128i); + stored = _mm_slli_epi32::<1>(stored); + diffs = _mm_sub_epi32(diffs, stored); + } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u16; - let pixel_color = load_u16_s32_fast::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u16; + let pixel_color = load_u16_s32_fast::(s_ptr); - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); + diffs = _mm_add_epi32(diffs, pixel_color); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); - summs = _mm_add_epi32(summs, diffs); + summs = _mm_add_epi32(summs, diffs); + } } } } diff --git a/src/avx/pack.rs b/src/avx/pack.rs index 816567d0..408d153a 100644 --- a/src/avx/pack.rs +++ b/src/avx/pack.rs @@ -39,331 +39,355 @@ pub(crate) const fn shuffle(z: u32, y: u32, x: u32, w: u32) -> i32 { } #[inline(always)] -pub(crate) unsafe fn _mm256_pack_u16(s_1: __m256i, s_2: __m256i) -> __m256i { - let packed = _mm256_packus_epi16(s_1, s_2); - const MASK: i32 = shuffle(3, 1, 2, 0); - _mm256_permute4x64_epi64::(packed) +pub(crate) fn _mm256_pack_u16(s_1: __m256i, s_2: __m256i) -> __m256i { + unsafe { + let packed = _mm256_packus_epi16(s_1, s_2); + const MASK: i32 = shuffle(3, 1, 2, 0); + _mm256_permute4x64_epi64::(packed) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgba_epi8( +pub(crate) fn _mm256_deinterleave_rgba_epi8( rgba0: __m256i, rgba1: __m256i, rgba2: __m256i, rgba3: __m256i, ) -> (__m256i, __m256i, __m256i, __m256i) { - #[rustfmt::skip] - let sh = _mm256_setr_epi8( - 0, 4, 8, 12, 1, 5, - 9, 13, 2, 6, 10, 14, - 3, 7, 11, 15, 0, 4, - 8, 12, 1, 5, 9, 13, - 2, 6, 10, 14, 3, 7, - 11, 15, - ); - - let p0 = _mm256_shuffle_epi8(rgba0, sh); - let p1 = _mm256_shuffle_epi8(rgba1, sh); - let p2 = _mm256_shuffle_epi8(rgba2, sh); - let p3 = _mm256_shuffle_epi8(rgba3, sh); - - let p01l = _mm256_unpacklo_epi32(p0, p1); - let p01h = _mm256_unpackhi_epi32(p0, p1); - let p23l = _mm256_unpacklo_epi32(p2, p3); - let p23h = _mm256_unpackhi_epi32(p2, p3); - - let pll = _mm256_permute2x128_si256::<32>(p01l, p23l); - let plh = _mm256_permute2x128_si256::<49>(p01l, p23l); - let phl = _mm256_permute2x128_si256::<32>(p01h, p23h); - let phh = _mm256_permute2x128_si256::<49>(p01h, p23h); - - let b0 = _mm256_unpacklo_epi32(pll, plh); - let g0 = _mm256_unpackhi_epi32(pll, plh); - let r0 = _mm256_unpacklo_epi32(phl, phh); - let a0 = _mm256_unpackhi_epi32(phl, phh); - - (b0, g0, r0, a0) + unsafe { + #[rustfmt::skip] + let sh = _mm256_setr_epi8( + 0, 4, 8, 12, 1, 5, + 9, 13, 2, 6, 10, 14, + 3, 7, 11, 15, 0, 4, + 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, + 11, 15, + ); + + let p0 = _mm256_shuffle_epi8(rgba0, sh); + let p1 = _mm256_shuffle_epi8(rgba1, sh); + let p2 = _mm256_shuffle_epi8(rgba2, sh); + let p3 = _mm256_shuffle_epi8(rgba3, sh); + + let p01l = _mm256_unpacklo_epi32(p0, p1); + let p01h = _mm256_unpackhi_epi32(p0, p1); + let p23l = _mm256_unpacklo_epi32(p2, p3); + let p23h = _mm256_unpackhi_epi32(p2, p3); + + let pll = _mm256_permute2x128_si256::<32>(p01l, p23l); + let plh = _mm256_permute2x128_si256::<49>(p01l, p23l); + let phl = _mm256_permute2x128_si256::<32>(p01h, p23h); + let phh = _mm256_permute2x128_si256::<49>(p01h, p23h); + + let b0 = _mm256_unpacklo_epi32(pll, plh); + let g0 = _mm256_unpackhi_epi32(pll, plh); + let r0 = _mm256_unpacklo_epi32(phl, phh); + let a0 = _mm256_unpackhi_epi32(phl, phh); + + (b0, g0, r0, a0) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgba_epi8( +pub(crate) fn _mm256_interleave_rgba_epi8( vals: (__m256i, __m256i, __m256i, __m256i), ) -> (__m256i, __m256i, __m256i, __m256i) { - let bg0 = _mm256_unpacklo_epi8(vals.0, vals.1); - let bg1 = _mm256_unpackhi_epi8(vals.0, vals.1); - let ra0 = _mm256_unpacklo_epi8(vals.2, vals.3); - let ra1 = _mm256_unpackhi_epi8(vals.2, vals.3); - - let rgba0_ = _mm256_unpacklo_epi16(bg0, ra0); - let rgba1_ = _mm256_unpackhi_epi16(bg0, ra0); - let rgba2_ = _mm256_unpacklo_epi16(bg1, ra1); - let rgba3_ = _mm256_unpackhi_epi16(bg1, ra1); - - let rgba0 = _mm256_permute2x128_si256::<32>(rgba0_, rgba1_); - let rgba2 = _mm256_permute2x128_si256::<49>(rgba0_, rgba1_); - let rgba1 = _mm256_permute2x128_si256::<32>(rgba2_, rgba3_); - let rgba3 = _mm256_permute2x128_si256::<49>(rgba2_, rgba3_); - (rgba0, rgba1, rgba2, rgba3) + unsafe { + let bg0 = _mm256_unpacklo_epi8(vals.0, vals.1); + let bg1 = _mm256_unpackhi_epi8(vals.0, vals.1); + let ra0 = _mm256_unpacklo_epi8(vals.2, vals.3); + let ra1 = _mm256_unpackhi_epi8(vals.2, vals.3); + + let rgba0_ = _mm256_unpacklo_epi16(bg0, ra0); + let rgba1_ = _mm256_unpackhi_epi16(bg0, ra0); + let rgba2_ = _mm256_unpacklo_epi16(bg1, ra1); + let rgba3_ = _mm256_unpackhi_epi16(bg1, ra1); + + let rgba0 = _mm256_permute2x128_si256::<32>(rgba0_, rgba1_); + let rgba2 = _mm256_permute2x128_si256::<49>(rgba0_, rgba1_); + let rgba1 = _mm256_permute2x128_si256::<32>(rgba2_, rgba3_); + let rgba3 = _mm256_permute2x128_si256::<49>(rgba2_, rgba3_); + (rgba0, rgba1, rgba2, rgba3) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgb( +pub(crate) fn _mm256_interleave_rgb( r: __m256i, g: __m256i, b: __m256i, ) -> (__m256i, __m256i, __m256i) { - let sh_b = _mm256_setr_epi8( - 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, - 9, 4, 15, 10, 5, - ); - let sh_g = _mm256_setr_epi8( - 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, - 14, 9, 4, 15, 10, - ); - let sh_r = _mm256_setr_epi8( - 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, - 3, 14, 9, 4, 15, - ); - - let b0 = _mm256_shuffle_epi8(r, sh_b); - let g0 = _mm256_shuffle_epi8(g, sh_g); - let r0 = _mm256_shuffle_epi8(b, sh_r); - - let m0 = _mm256_setr_epi8( - 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, - 0, 0, -1, 0, 0, - ); - let m1 = _mm256_setr_epi8( - 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, - -1, 0, 0, -1, 0, - ); - - let p0 = _mm256_blendv_epi8(_mm256_blendv_epi8(b0, g0, m0), r0, m1); - let p1 = _mm256_blendv_epi8(_mm256_blendv_epi8(g0, r0, m0), b0, m1); - let p2 = _mm256_blendv_epi8(_mm256_blendv_epi8(r0, b0, m0), g0, m1); - - let bgr0 = _mm256_permute2x128_si256::<32>(p0, p1); - let bgr1 = _mm256_permute2x128_si256::<48>(p2, p0); - let bgr2 = _mm256_permute2x128_si256::<49>(p1, p2); - - (bgr0, bgr1, bgr2) + unsafe { + let sh_b = _mm256_setr_epi8( + 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, + 14, 9, 4, 15, 10, 5, + ); + let sh_g = _mm256_setr_epi8( + 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, + 3, 14, 9, 4, 15, 10, + ); + let sh_r = _mm256_setr_epi8( + 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, + 8, 3, 14, 9, 4, 15, + ); + + let b0 = _mm256_shuffle_epi8(r, sh_b); + let g0 = _mm256_shuffle_epi8(g, sh_g); + let r0 = _mm256_shuffle_epi8(b, sh_r); + + let m0 = _mm256_setr_epi8( + 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, + -1, 0, 0, -1, 0, 0, + ); + let m1 = _mm256_setr_epi8( + 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, + 0, -1, 0, 0, -1, 0, + ); + + let p0 = _mm256_blendv_epi8(_mm256_blendv_epi8(b0, g0, m0), r0, m1); + let p1 = _mm256_blendv_epi8(_mm256_blendv_epi8(g0, r0, m0), b0, m1); + let p2 = _mm256_blendv_epi8(_mm256_blendv_epi8(r0, b0, m0), g0, m1); + + let bgr0 = _mm256_permute2x128_si256::<32>(p0, p1); + let bgr1 = _mm256_permute2x128_si256::<48>(p2, p0); + let bgr2 = _mm256_permute2x128_si256::<49>(p1, p2); + + (bgr0, bgr1, bgr2) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgb( +pub(crate) fn _mm256_deinterleave_rgb( rgb0: __m256i, rgb1: __m256i, rgb2: __m256i, ) -> (__m256i, __m256i, __m256i) { - let s02_low = _mm256_permute2x128_si256::<32>(rgb0, rgb2); - let s02_high = _mm256_permute2x128_si256::<49>(rgb0, rgb2); - - #[rustfmt::skip] - let m0 = _mm256_setr_epi8( - 0, 0, -1, 0, 0, - -1, 0, 0, -1, 0, - 0, -1, 0, 0, -1, - 0, 0, -1, 0, 0, - -1, 0, 0, -1, 0, - 0, -1, 0, 0, -1, - 0, 0, - ); - - #[rustfmt::skip] - let m1 = _mm256_setr_epi8( - 0, -1, 0, 0, -1, - 0, 0, -1, 0, 0, - -1, 0, 0, -1, 0, - 0, -1, 0, 0, -1, - 0, 0, -1, 0, 0, - -1, 0, 0, -1, 0, - 0, -1, - ); - - let b0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_low, s02_high, m0), rgb1, m1); - let g0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_high, s02_low, m1), rgb1, m0); - let r0 = _mm256_blendv_epi8(_mm256_blendv_epi8(rgb1, s02_low, m0), s02_high, m1); - - #[rustfmt::skip] - let sh_b = _mm256_setr_epi8( - 0, 3, 6, 9, 12, - 15, 2, 5, 8, 11, - 14, 1, 4, 7, 10, - 13, 0, 3, 6, 9, - 12, 15, 2, 5, 8, - 11, 14, 1, 4, 7, - 10, 13, - ); - - #[rustfmt::skip] - let sh_g = _mm256_setr_epi8( - 1, 4, 7, 10, 13, - 0, 3, 6, 9, 12, - 15, 2, 5, 8, 11, - 14, 1, 4, 7, 10, - 13, 0, 3, 6, 9, - 12, 15, 2, 5, 8, - 11, 14, - ); - - #[rustfmt::skip] - let sh_r = _mm256_setr_epi8( - 2, 5, 8, 11, 14, - 1, 4, 7, 10, 13, - 0, 3, 6, 9, 12, - 15, 2, 5, 8, 11, - 14, 1, 4, 7, 10, - 13, 0, 3, 6, 9, - 12, 15, - ); - let b0 = _mm256_shuffle_epi8(b0, sh_b); - let g0 = _mm256_shuffle_epi8(g0, sh_g); - let r0 = _mm256_shuffle_epi8(r0, sh_r); - (b0, g0, r0) + unsafe { + let s02_low = _mm256_permute2x128_si256::<32>(rgb0, rgb2); + let s02_high = _mm256_permute2x128_si256::<49>(rgb0, rgb2); + + #[rustfmt::skip] + let m0 = _mm256_setr_epi8( + 0, 0, -1, 0, 0, + -1, 0, 0, -1, 0, + 0, -1, 0, 0, -1, + 0, 0, -1, 0, 0, + -1, 0, 0, -1, 0, + 0, -1, 0, 0, -1, + 0, 0, + ); + + #[rustfmt::skip] + let m1 = _mm256_setr_epi8( + 0, -1, 0, 0, -1, + 0, 0, -1, 0, 0, + -1, 0, 0, -1, 0, + 0, -1, 0, 0, -1, + 0, 0, -1, 0, 0, + -1, 0, 0, -1, 0, + 0, -1, + ); + + let b0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_low, s02_high, m0), rgb1, m1); + let g0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_high, s02_low, m1), rgb1, m0); + let r0 = _mm256_blendv_epi8(_mm256_blendv_epi8(rgb1, s02_low, m0), s02_high, m1); + + #[rustfmt::skip] + let sh_b = _mm256_setr_epi8( + 0, 3, 6, 9, 12, + 15, 2, 5, 8, 11, + 14, 1, 4, 7, 10, + 13, 0, 3, 6, 9, + 12, 15, 2, 5, 8, + 11, 14, 1, 4, 7, + 10, 13, + ); + + #[rustfmt::skip] + let sh_g = _mm256_setr_epi8( + 1, 4, 7, 10, 13, + 0, 3, 6, 9, 12, + 15, 2, 5, 8, 11, + 14, 1, 4, 7, 10, + 13, 0, 3, 6, 9, + 12, 15, 2, 5, 8, + 11, 14, + ); + + #[rustfmt::skip] + let sh_r = _mm256_setr_epi8( + 2, 5, 8, 11, 14, + 1, 4, 7, 10, 13, + 0, 3, 6, 9, 12, + 15, 2, 5, 8, 11, + 14, 1, 4, 7, 10, + 13, 0, 3, 6, 9, + 12, 15, + ); + let b0 = _mm256_shuffle_epi8(b0, sh_b); + let g0 = _mm256_shuffle_epi8(g0, sh_g); + let r0 = _mm256_shuffle_epi8(r0, sh_r); + (b0, g0, r0) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgba_epi32( +pub(crate) fn _mm256_deinterleave_rgba_epi32( vals: (__m256i, __m256i, __m256i, __m256i), ) -> (__m256i, __m256i, __m256i, __m256i) { - let p01l = _mm256_unpacklo_epi32(vals.0, vals.1); - let p01h = _mm256_unpackhi_epi32(vals.0, vals.1); - let p23l = _mm256_unpacklo_epi32(vals.2, vals.3); - let p23h = _mm256_unpackhi_epi32(vals.2, vals.3); - - let pll = _mm256_permute2x128_si256::<32>(p01l, p23l); - let plh = _mm256_permute2x128_si256::<49>(p01l, p23l); - let phl = _mm256_permute2x128_si256::<32>(p01h, p23h); - let phh = _mm256_permute2x128_si256::<49>(p01h, p23h); - - let v0 = _mm256_unpacklo_epi32(pll, plh); - let v1 = _mm256_unpackhi_epi32(pll, plh); - let v2 = _mm256_unpacklo_epi32(phl, phh); - let v3 = _mm256_unpackhi_epi32(phl, phh); - (v0, v1, v2, v3) + unsafe { + let p01l = _mm256_unpacklo_epi32(vals.0, vals.1); + let p01h = _mm256_unpackhi_epi32(vals.0, vals.1); + let p23l = _mm256_unpacklo_epi32(vals.2, vals.3); + let p23h = _mm256_unpackhi_epi32(vals.2, vals.3); + + let pll = _mm256_permute2x128_si256::<32>(p01l, p23l); + let plh = _mm256_permute2x128_si256::<49>(p01l, p23l); + let phl = _mm256_permute2x128_si256::<32>(p01h, p23h); + let phh = _mm256_permute2x128_si256::<49>(p01h, p23h); + + let v0 = _mm256_unpacklo_epi32(pll, plh); + let v1 = _mm256_unpackhi_epi32(pll, plh); + let v2 = _mm256_unpacklo_epi32(phl, phh); + let v3 = _mm256_unpackhi_epi32(phl, phh); + (v0, v1, v2, v3) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgb_epi32( +pub(crate) fn _mm256_deinterleave_rgb_epi32( vals: (__m256i, __m256i, __m256i), ) -> (__m256i, __m256i, __m256i) { - let s02_low = _mm256_permute2x128_si256::<32>(vals.0, vals.2); - let s02_high = _mm256_permute2x128_si256::<49>(vals.0, vals.2); - - let b0 = _mm256_blend_epi32::<0x92>(_mm256_blend_epi32::<0x24>(s02_low, s02_high), vals.1); - let g0 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(s02_high, s02_low), vals.1); - let r0 = _mm256_blend_epi32::<0x92>(_mm256_blend_epi32::<0x24>(vals.1, s02_low), s02_high); - - let v0 = _mm256_shuffle_epi32::<0x6c>(b0); - let v1 = _mm256_shuffle_epi32::<0xb1>(g0); - let v2 = _mm256_shuffle_epi32::<0xc6>(r0); - (v0, v1, v2) + unsafe { + let s02_low = _mm256_permute2x128_si256::<32>(vals.0, vals.2); + let s02_high = _mm256_permute2x128_si256::<49>(vals.0, vals.2); + + let b0 = _mm256_blend_epi32::<0x92>(_mm256_blend_epi32::<0x24>(s02_low, s02_high), vals.1); + let g0 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(s02_high, s02_low), vals.1); + let r0 = _mm256_blend_epi32::<0x92>(_mm256_blend_epi32::<0x24>(vals.1, s02_low), s02_high); + + let v0 = _mm256_shuffle_epi32::<0x6c>(b0); + let v1 = _mm256_shuffle_epi32::<0xb1>(g0); + let v2 = _mm256_shuffle_epi32::<0xc6>(r0); + (v0, v1, v2) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgba_ps( +pub(crate) fn _mm256_deinterleave_rgba_ps( vals: (__m256, __m256, __m256, __m256), ) -> (__m256, __m256, __m256, __m256) { - let (v0, v1, v2, v3) = _mm256_deinterleave_rgba_epi32(( - _mm256_castps_si256(vals.0), - _mm256_castps_si256(vals.1), - _mm256_castps_si256(vals.2), - _mm256_castps_si256(vals.3), - )); - ( - _mm256_castsi256_ps(v0), - _mm256_castsi256_ps(v1), - _mm256_castsi256_ps(v2), - _mm256_castsi256_ps(v3), - ) + unsafe { + let (v0, v1, v2, v3) = _mm256_deinterleave_rgba_epi32(( + _mm256_castps_si256(vals.0), + _mm256_castps_si256(vals.1), + _mm256_castps_si256(vals.2), + _mm256_castps_si256(vals.3), + )); + ( + _mm256_castsi256_ps(v0), + _mm256_castsi256_ps(v1), + _mm256_castsi256_ps(v2), + _mm256_castsi256_ps(v3), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgb_ps( +pub(crate) fn _mm256_deinterleave_rgb_ps( vals: (__m256, __m256, __m256), ) -> (__m256, __m256, __m256) { - let (v0, v1, v2) = _mm256_deinterleave_rgb_epi32(( - _mm256_castps_si256(vals.0), - _mm256_castps_si256(vals.1), - _mm256_castps_si256(vals.2), - )); - ( - _mm256_castsi256_ps(v0), - _mm256_castsi256_ps(v1), - _mm256_castsi256_ps(v2), - ) + unsafe { + let (v0, v1, v2) = _mm256_deinterleave_rgb_epi32(( + _mm256_castps_si256(vals.0), + _mm256_castps_si256(vals.1), + _mm256_castps_si256(vals.2), + )); + ( + _mm256_castsi256_ps(v0), + _mm256_castsi256_ps(v1), + _mm256_castsi256_ps(v2), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgb_epi32( +pub(crate) fn _mm256_interleave_rgb_epi32( vals: (__m256i, __m256i, __m256i), ) -> (__m256i, __m256i, __m256i) { - let b0 = _mm256_shuffle_epi32::<0x6c>(vals.0); - let g0 = _mm256_shuffle_epi32::<0xb1>(vals.1); - let r0 = _mm256_shuffle_epi32::<0xc6>(vals.2); - - let p0 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(b0, g0), r0); - let p1 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(g0, r0), b0); - let p2 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(r0, b0), g0); - - let v0 = _mm256_permute2x128_si256::<32>(p0, p1); - let v1 = p2; - let v2 = _mm256_permute2x128_si256::<49>(p0, p1); - (v0, v1, v2) + unsafe { + let b0 = _mm256_shuffle_epi32::<0x6c>(vals.0); + let g0 = _mm256_shuffle_epi32::<0xb1>(vals.1); + let r0 = _mm256_shuffle_epi32::<0xc6>(vals.2); + + let p0 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(b0, g0), r0); + let p1 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(g0, r0), b0); + let p2 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(r0, b0), g0); + + let v0 = _mm256_permute2x128_si256::<32>(p0, p1); + let v1 = p2; + let v2 = _mm256_permute2x128_si256::<49>(p0, p1); + (v0, v1, v2) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgb_ps( - vals: (__m256, __m256, __m256), -) -> (__m256, __m256, __m256) { - let (v0, v1, v2) = _mm256_interleave_rgb_epi32(( - _mm256_castps_si256(vals.0), - _mm256_castps_si256(vals.1), - _mm256_castps_si256(vals.2), - )); - ( - _mm256_castsi256_ps(v0), - _mm256_castsi256_ps(v1), - _mm256_castsi256_ps(v2), - ) +pub(crate) fn _mm256_interleave_rgb_ps(vals: (__m256, __m256, __m256)) -> (__m256, __m256, __m256) { + unsafe { + let (v0, v1, v2) = _mm256_interleave_rgb_epi32(( + _mm256_castps_si256(vals.0), + _mm256_castps_si256(vals.1), + _mm256_castps_si256(vals.2), + )); + ( + _mm256_castsi256_ps(v0), + _mm256_castsi256_ps(v1), + _mm256_castsi256_ps(v2), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgba_epi32( +pub(crate) fn _mm256_interleave_rgba_epi32( vals: (__m256i, __m256i, __m256i, __m256i), ) -> (__m256i, __m256i, __m256i, __m256i) { - let bg0 = _mm256_unpacklo_epi32(vals.0, vals.1); - let bg1 = _mm256_unpackhi_epi32(vals.0, vals.1); - let ra0 = _mm256_unpacklo_epi32(vals.2, vals.3); - let ra1 = _mm256_unpackhi_epi32(vals.2, vals.3); - - let bgra0_ = _mm256_unpacklo_epi64(bg0, ra0); - let bgra1_ = _mm256_unpackhi_epi64(bg0, ra0); - let bgra2_ = _mm256_unpacklo_epi64(bg1, ra1); - let bgra3_ = _mm256_unpackhi_epi64(bg1, ra1); - - let v0 = _mm256_permute2x128_si256::<32>(bgra0_, bgra1_); - let v1 = _mm256_permute2x128_si256::<32>(bgra2_, bgra3_); - let v2 = _mm256_permute2x128_si256::<49>(bgra0_, bgra1_); - let v3 = _mm256_permute2x128_si256::<49>(bgra2_, bgra3_); - - (v0, v1, v2, v3) + unsafe { + let bg0 = _mm256_unpacklo_epi32(vals.0, vals.1); + let bg1 = _mm256_unpackhi_epi32(vals.0, vals.1); + let ra0 = _mm256_unpacklo_epi32(vals.2, vals.3); + let ra1 = _mm256_unpackhi_epi32(vals.2, vals.3); + + let bgra0_ = _mm256_unpacklo_epi64(bg0, ra0); + let bgra1_ = _mm256_unpackhi_epi64(bg0, ra0); + let bgra2_ = _mm256_unpacklo_epi64(bg1, ra1); + let bgra3_ = _mm256_unpackhi_epi64(bg1, ra1); + + let v0 = _mm256_permute2x128_si256::<32>(bgra0_, bgra1_); + let v1 = _mm256_permute2x128_si256::<32>(bgra2_, bgra3_); + let v2 = _mm256_permute2x128_si256::<49>(bgra0_, bgra1_); + let v3 = _mm256_permute2x128_si256::<49>(bgra2_, bgra3_); + + (v0, v1, v2, v3) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgba_ps( +pub(crate) fn _mm256_interleave_rgba_ps( vals: (__m256, __m256, __m256, __m256), ) -> (__m256, __m256, __m256, __m256) { - let (v0, v1, v2, v3) = _mm256_interleave_rgba_epi32(( - _mm256_castps_si256(vals.0), - _mm256_castps_si256(vals.1), - _mm256_castps_si256(vals.2), - _mm256_castps_si256(vals.3), - )); - ( - _mm256_castsi256_ps(v0), - _mm256_castsi256_ps(v1), - _mm256_castsi256_ps(v2), - _mm256_castsi256_ps(v3), - ) + unsafe { + let (v0, v1, v2, v3) = _mm256_interleave_rgba_epi32(( + _mm256_castps_si256(vals.0), + _mm256_castps_si256(vals.1), + _mm256_castps_si256(vals.2), + _mm256_castps_si256(vals.3), + )); + ( + _mm256_castsi256_ps(v0), + _mm256_castsi256_ps(v1), + _mm256_castsi256_ps(v2), + _mm256_castsi256_ps(v3), + ) + } } diff --git a/src/avx/utils.rs b/src/avx/utils.rs index 46ee87c4..e4927dd9 100644 --- a/src/avx/utils.rs +++ b/src/avx/utils.rs @@ -32,55 +32,49 @@ use std::arch::x86::*; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm256_mul_round_ps(a: __m256, b: __m256) -> __m256 { - _mm256_fmadd_ps(a, b, _mm256_set1_ps(0.5f32)) +pub(crate) fn _mm256_mul_round_ps(a: __m256, b: __m256) -> __m256 { + unsafe { _mm256_fmadd_ps(a, b, _mm256_set1_ps(0.5f32)) } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_round_ps(a: __m128, b: __m128) -> __m128 { - _mm_fmadd_ps(a, b, _mm_set1_ps(0.5f32)) +pub(crate) fn _mm_mul_round_ps(a: __m128, b: __m128) -> __m128 { + unsafe { _mm_fmadd_ps(a, b, _mm_set1_ps(0.5f32)) } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_by_3_epi32(v: __m256i) -> __m256i { - _mm256_add_epi32(_mm256_slli_epi32::<1>(v), v) +pub(crate) fn _mm256_mul_by_3_epi32(v: __m256i) -> __m256i { + unsafe { _mm256_add_epi32(_mm256_slli_epi32::<1>(v), v) } } #[inline(always)] -pub(crate) unsafe fn _mm256_opt_fnmlaf_ps( - a: __m256, - b: __m256, - c: __m256, -) -> __m256 { - if FMA { - _mm256_fnmadd_ps(b, c, a) - } else { - _mm256_sub_ps(a, _mm256_mul_ps(b, c)) +pub(crate) fn _mm256_opt_fnmlaf_ps(a: __m256, b: __m256, c: __m256) -> __m256 { + unsafe { + if FMA { + _mm256_fnmadd_ps(b, c, a) + } else { + _mm256_sub_ps(a, _mm256_mul_ps(b, c)) + } } } #[inline(always)] -pub(crate) unsafe fn _mm256_opt_fmlaf_ps( - a: __m256, - b: __m256, - c: __m256, -) -> __m256 { - if FMA { - _mm256_fmadd_ps(b, c, a) - } else { - _mm256_add_ps(a, _mm256_mul_ps(b, c)) +pub(crate) fn _mm256_opt_fmlaf_ps(a: __m256, b: __m256, c: __m256) -> __m256 { + unsafe { + if FMA { + _mm256_fmadd_ps(b, c, a) + } else { + _mm256_add_ps(a, _mm256_mul_ps(b, c)) + } } } #[inline(always)] -pub(crate) unsafe fn _mm256_opt_fnmlsf_ps( - a: __m256, - b: __m256, - c: __m256, -) -> __m256 { - if FMA { - _mm256_fmsub_ps(b, c, a) - } else { - _mm256_sub_ps(_mm256_mul_ps(b, c), a) +pub(crate) fn _mm256_opt_fnmlsf_ps(a: __m256, b: __m256, c: __m256) -> __m256 { + unsafe { + if FMA { + _mm256_fmsub_ps(b, c, a) + } else { + _mm256_sub_ps(_mm256_mul_ps(b, c), a) + } } } diff --git a/src/avx/v_load.rs b/src/avx/v_load.rs index 9d9b5b00..bfe2b9c2 100644 --- a/src/avx/v_load.rs +++ b/src/avx/v_load.rs @@ -38,78 +38,96 @@ use std::arch::x86_64::*; #[inline(always)] pub(crate) unsafe fn _mm256_load_pack_x4(ptr: *const u8) -> (__m256i, __m256i, __m256i, __m256i) { - let row0 = _mm256_loadu_si256(ptr as *const __m256i); - let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); - let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); - let row3 = _mm256_loadu_si256(ptr.add(96) as *const __m256i); - (row0, row1, row2, row3) + unsafe { + let row0 = _mm256_loadu_si256(ptr as *const __m256i); + let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); + let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); + let row3 = _mm256_loadu_si256(ptr.add(96) as *const __m256i); + (row0, row1, row2, row3) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_pack_x3(ptr: *const u8) -> (__m256i, __m256i, __m256i) { - let row0 = _mm256_loadu_si256(ptr as *const __m256i); - let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); - let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); - (row0, row1, row2) + unsafe { + let row0 = _mm256_loadu_si256(ptr as *const __m256i); + let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); + let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); + (row0, row1, row2) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_pack_x2(ptr: *const u8) -> (__m256i, __m256i) { - let row0 = _mm256_loadu_si256(ptr as *const __m256i); - let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); - (row0, row1) + unsafe { + let row0 = _mm256_loadu_si256(ptr as *const __m256i); + let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); + (row0, row1) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_deinterleave_rgb(ptr: *const u8) -> (__m256i, __m256i, __m256i) { - let row0 = _mm256_loadu_si256(ptr as *const __m256i); - let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); - let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); - _mm256_deinterleave_rgb(row0, row1, row2) + unsafe { + let row0 = _mm256_loadu_si256(ptr as *const __m256i); + let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); + let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); + _mm256_deinterleave_rgb(row0, row1, row2) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_deinterleave_rgba( ptr: *const u8, ) -> (__m256i, __m256i, __m256i, __m256i) { - let row0 = _mm256_loadu_si256(ptr as *const __m256i); - let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); - let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); - let row3 = _mm256_loadu_si256(ptr.add(96) as *const __m256i); - _mm256_deinterleave_rgba_epi8(row0, row1, row2, row3) + unsafe { + let row0 = _mm256_loadu_si256(ptr as *const __m256i); + let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); + let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); + let row3 = _mm256_loadu_si256(ptr.add(96) as *const __m256i); + _mm256_deinterleave_rgba_epi8(row0, row1, row2, row3) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_pack_ps_x4(ptr: *const f32) -> (__m256, __m256, __m256, __m256) { - let row0 = _mm256_loadu_ps(ptr); - let row1 = _mm256_loadu_ps(ptr.add(8)); - let row2 = _mm256_loadu_ps(ptr.add(16)); - let row3 = _mm256_loadu_ps(ptr.add(24)); - (row0, row1, row2, row3) + unsafe { + let row0 = _mm256_loadu_ps(ptr); + let row1 = _mm256_loadu_ps(ptr.add(8)); + let row2 = _mm256_loadu_ps(ptr.add(16)); + let row3 = _mm256_loadu_ps(ptr.add(24)); + (row0, row1, row2, row3) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_pack_ps_x2(ptr: *const f32) -> (__m256, __m256) { - let row0 = _mm256_loadu_ps(ptr); - let row1 = _mm256_loadu_ps(ptr.add(8)); - (row0, row1) + unsafe { + let row0 = _mm256_loadu_ps(ptr); + let row1 = _mm256_loadu_ps(ptr.add(8)); + (row0, row1) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_deinterleave_rgb_ps(ptr: *const f32) -> (__m256, __m256, __m256) { - let row0 = _mm256_loadu_ps(ptr); - let row1 = _mm256_loadu_ps(ptr.add(8)); - let row2 = _mm256_loadu_ps(ptr.add(16)); - _mm256_deinterleave_rgb_ps((row0, row1, row2)) + unsafe { + let row0 = _mm256_loadu_ps(ptr); + let row1 = _mm256_loadu_ps(ptr.add(8)); + let row2 = _mm256_loadu_ps(ptr.add(16)); + _mm256_deinterleave_rgb_ps((row0, row1, row2)) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_deinterleave_rgba_ps( ptr: *const f32, ) -> (__m256, __m256, __m256, __m256) { - let row0 = _mm256_loadu_ps(ptr); - let row1 = _mm256_loadu_ps(ptr.add(8)); - let row2 = _mm256_loadu_ps(ptr.add(16)); - let row3 = _mm256_loadu_ps(ptr.add(24)); - _mm256_deinterleave_rgba_ps((row0, row1, row2, row3)) + unsafe { + let row0 = _mm256_loadu_ps(ptr); + let row1 = _mm256_loadu_ps(ptr.add(8)); + let row2 = _mm256_loadu_ps(ptr.add(16)); + let row3 = _mm256_loadu_ps(ptr.add(24)); + _mm256_deinterleave_rgba_ps((row0, row1, row2, row3)) + } } diff --git a/src/avx/v_store.rs b/src/avx/v_store.rs index 09f244dd..a9e2fdf6 100644 --- a/src/avx/v_store.rs +++ b/src/avx/v_store.rs @@ -37,73 +37,81 @@ use std::arch::x86::*; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm256_store_pack_x4(ptr: *mut u8, v: (__m256i, __m256i, __m256i, __m256i)) { - _mm256_storeu_si256(ptr as *mut __m256i, v.0); - _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); - _mm256_storeu_si256(ptr.add(64) as *mut __m256i, v.2); - _mm256_storeu_si256(ptr.add(96) as *mut __m256i, v.3); +pub(crate) fn _mm256_store_pack_x4(ptr: *mut u8, v: (__m256i, __m256i, __m256i, __m256i)) { + unsafe { + _mm256_storeu_si256(ptr as *mut __m256i, v.0); + _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); + _mm256_storeu_si256(ptr.add(64) as *mut __m256i, v.2); + _mm256_storeu_si256(ptr.add(96) as *mut __m256i, v.3); + } } #[inline(always)] -pub(crate) unsafe fn _mm256_store_pack_x2(ptr: *mut u8, v: (__m256i, __m256i)) { - _mm256_storeu_si256(ptr as *mut __m256i, v.0); - _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); +pub(crate) fn _mm256_store_pack_x2(ptr: *mut u8, v: (__m256i, __m256i)) { + unsafe { + _mm256_storeu_si256(ptr as *mut __m256i, v.0); + _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); + } } #[inline(always)] -pub(crate) unsafe fn _mm256_store_pack_x3(ptr: *mut u8, v: (__m256i, __m256i, __m256i)) { - _mm256_storeu_si256(ptr as *mut __m256i, v.0); - _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); - _mm256_storeu_si256(ptr.add(64) as *mut __m256i, v.2); +pub(crate) fn _mm256_store_pack_x3(ptr: *mut u8, v: (__m256i, __m256i, __m256i)) { + unsafe { + _mm256_storeu_si256(ptr as *mut __m256i, v.0); + _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); + _mm256_storeu_si256(ptr.add(64) as *mut __m256i, v.2); + } } #[inline(always)] -pub(crate) unsafe fn _mm256_store_pack_x3_ps(ptr: *mut f32, v: (__m256, __m256, __m256)) { - _mm256_storeu_ps(ptr, v.0); - _mm256_storeu_ps(ptr.add(8), v.1); - _mm256_storeu_ps(ptr.add(16), v.2); +pub(crate) fn _mm256_store_pack_x3_ps(ptr: *mut f32, v: (__m256, __m256, __m256)) { + unsafe { + _mm256_storeu_ps(ptr, v.0); + _mm256_storeu_ps(ptr.add(8), v.1); + _mm256_storeu_ps(ptr.add(16), v.2); + } } #[inline(always)] -pub(crate) unsafe fn _mm256_store_interleave_rgb(ptr: *mut u8, v: (__m256i, __m256i, __m256i)) { +pub(crate) fn _mm256_store_interleave_rgb(ptr: *mut u8, v: (__m256i, __m256i, __m256i)) { let (v0, v1, v2) = _mm256_interleave_rgb(v.0, v.1, v.2); _mm256_store_pack_x3(ptr, (v0, v1, v2)); } #[inline(always)] -pub(crate) unsafe fn _mm256_store_interleave_rgba( - ptr: *mut u8, - v: (__m256i, __m256i, __m256i, __m256i), -) { +pub(crate) fn _mm256_store_interleave_rgba(ptr: *mut u8, v: (__m256i, __m256i, __m256i, __m256i)) { let (v0, v1, v2, v3) = _mm256_interleave_rgba_epi8((v.0, v.1, v.2, v.3)); _mm256_store_pack_x4(ptr, (v0, v1, v2, v3)); } #[inline(always)] pub(crate) unsafe fn _mm256_store_pack_ps_x4(ptr: *mut f32, v: (__m256, __m256, __m256, __m256)) { - _mm256_storeu_ps(ptr, v.0); - _mm256_storeu_ps(ptr.add(8), v.1); - _mm256_storeu_ps(ptr.add(16), v.2); - _mm256_storeu_ps(ptr.add(24), v.3); + unsafe { + _mm256_storeu_ps(ptr, v.0); + _mm256_storeu_ps(ptr.add(8), v.1); + _mm256_storeu_ps(ptr.add(16), v.2); + _mm256_storeu_ps(ptr.add(24), v.3); + } } #[inline(always)] pub(crate) unsafe fn _mm256_store_pack_ps_x2(ptr: *mut f32, v: (__m256, __m256)) { - _mm256_storeu_ps(ptr, v.0); - _mm256_storeu_ps(ptr.add(8), v.1); + unsafe { + _mm256_storeu_ps(ptr, v.0); + _mm256_storeu_ps(ptr.add(8), v.1); + } } #[inline(always)] -pub(crate) unsafe fn _mm256_store_interleave_rgb_ps(ptr: *mut f32, v: (__m256, __m256, __m256)) { +pub(crate) fn _mm256_store_interleave_rgb_ps(ptr: *mut f32, v: (__m256, __m256, __m256)) { let (v0, v1, v2) = _mm256_interleave_rgb_ps(v); _mm256_store_pack_x3_ps(ptr, (v0, v1, v2)); } #[inline(always)] -pub(crate) unsafe fn _mm256_store_interleave_rgba_ps( - ptr: *mut f32, - v: (__m256, __m256, __m256, __m256), -) { +pub(crate) fn _mm256_store_interleave_rgba_ps(ptr: *mut f32, v: (__m256, __m256, __m256, __m256)) { let (v0, v1, v2, v3) = _mm256_interleave_rgba_ps(v); - _mm256_store_pack_ps_x4(ptr, (v0, v1, v2, v3)); + unsafe { + _mm256_store_pack_ps_x4(ptr, (v0, v1, v2, v3)); + } } diff --git a/src/bilateral/bp8.rs b/src/bilateral/bp8.rs index 8a73bea0..2c8991e9 100644 --- a/src/bilateral/bp8.rs +++ b/src/bilateral/bp8.rs @@ -28,7 +28,7 @@ */ #![allow(clippy::manual_clamp)] -use crate::filter1d::{make_arena, Arena, ArenaPads}; +use crate::filter1d::{Arena, ArenaPads, make_arena}; use crate::{ BlurError, BlurImage, BlurImageMut, EdgeMode2D, FastBlurChannels, Scalar, ThreadingPolicy, }; @@ -341,7 +341,7 @@ pub struct BilateralBlurParams { impl BilateralBlurParams { fn validate(&self) -> Result<(), BlurError> { - if self.kernel_size % 2 == 0 { + if self.kernel_size.is_multiple_of(2) { return Err(BlurError::OddKernel(self.kernel_size)); } if self.spatial_sigma <= 0.0 { diff --git a/src/bilateral/mod.rs b/src/bilateral/mod.rs index a07d8c66..d092b5b3 100644 --- a/src/bilateral/mod.rs +++ b/src/bilateral/mod.rs @@ -30,4 +30,4 @@ mod bp8; #[cfg(all(target_arch = "aarch64", feature = "neon"))] mod neon; -pub use bp8::{bilateral_filter, BilateralBlurParams}; +pub use bp8::{BilateralBlurParams, bilateral_filter}; diff --git a/src/bilateral/neon/bp8.rs b/src/bilateral/neon/bp8.rs index a30d634f..18eeb8e5 100644 --- a/src/bilateral/neon/bp8.rs +++ b/src/bilateral/neon/bp8.rs @@ -27,9 +27,9 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #![allow(clippy::manual_clamp)] +use crate::BilateralBlurParams; use crate::bilateral::bp8::{BilateralStore, BilateralUnit}; use crate::filter1d::Arena; -use crate::BilateralBlurParams; use std::arch::aarch64::*; pub(crate) struct BilateralExecutionUnitNeon<'a, const N: usize> { @@ -40,29 +40,33 @@ pub(crate) struct BilateralExecutionUnitNeon<'a, const N: usize> { } #[inline(always)] -unsafe fn v_lut(v: uint16x8_t, lut: &[f32; 65536]) -> (float32x4_t, float32x4_t) { - let v0 = vld1q_lane_f32::<0>( - lut.get_unchecked(vgetq_lane_u16::<0>(v) as usize), - vdupq_n_f32(0.), - ); - let v1 = vld1q_lane_f32::<1>(lut.get_unchecked(vgetq_lane_u16::<1>(v) as usize), v0); - let v2 = vld1q_lane_f32::<2>(lut.get_unchecked(vgetq_lane_u16::<2>(v) as usize), v1); - let v3 = vld1q_lane_f32::<3>(lut.get_unchecked(vgetq_lane_u16::<3>(v) as usize), v2); - - let v4 = vld1q_lane_f32::<0>( - lut.get_unchecked(vgetq_lane_u16::<4>(v) as usize), - vdupq_n_f32(0.), - ); - let v5 = vld1q_lane_f32::<1>(lut.get_unchecked(vgetq_lane_u16::<5>(v) as usize), v4); - let v6 = vld1q_lane_f32::<2>(lut.get_unchecked(vgetq_lane_u16::<6>(v) as usize), v5); - let v7 = vld1q_lane_f32::<3>(lut.get_unchecked(vgetq_lane_u16::<7>(v) as usize), v6); - (v3, v7) +fn v_lut(v: uint16x8_t, lut: &[f32; 65536]) -> (float32x4_t, float32x4_t) { + unsafe { + let v0 = vld1q_lane_f32::<0>( + lut.get_unchecked(vgetq_lane_u16::<0>(v) as usize), + vdupq_n_f32(0.), + ); + let v1 = vld1q_lane_f32::<1>(lut.get_unchecked(vgetq_lane_u16::<1>(v) as usize), v0); + let v2 = vld1q_lane_f32::<2>(lut.get_unchecked(vgetq_lane_u16::<2>(v) as usize), v1); + let v3 = vld1q_lane_f32::<3>(lut.get_unchecked(vgetq_lane_u16::<3>(v) as usize), v2); + + let v4 = vld1q_lane_f32::<0>( + lut.get_unchecked(vgetq_lane_u16::<4>(v) as usize), + vdupq_n_f32(0.), + ); + let v5 = vld1q_lane_f32::<1>(lut.get_unchecked(vgetq_lane_u16::<5>(v) as usize), v4); + let v6 = vld1q_lane_f32::<2>(lut.get_unchecked(vgetq_lane_u16::<6>(v) as usize), v5); + let v7 = vld1q_lane_f32::<3>(lut.get_unchecked(vgetq_lane_u16::<7>(v) as usize), v6); + (v3, v7) + } } #[inline(always)] -unsafe fn replace_zeros_with_ones(input: float32x4_t) -> float32x4_t { - let is_zero = vceqq_f32(input, vdupq_n_f32(0.0)); - vbslq_f32(is_zero, vdupq_n_f32(1.0), input) +fn replace_zeros_with_ones(input: float32x4_t) -> float32x4_t { + unsafe { + let is_zero = vceqq_f32(input, vdupq_n_f32(0.0)); + vbslq_f32(is_zero, vdupq_n_f32(1.0), input) + } } impl BilateralUnit for BilateralExecutionUnitNeon<'_, N> { diff --git a/src/box_filter/avx/hrgba8.rs b/src/box_filter/avx/hrgba8.rs index 793e5761..3fef5f0e 100644 --- a/src/box_filter/avx/hrgba8.rs +++ b/src/box_filter/avx/hrgba8.rs @@ -53,47 +53,49 @@ pub(crate) unsafe fn load_u8_s32_fast_x2( ptr0: *const u8, ptr1: *const u8, ) -> __m256i { - let sh1 = _mm256_setr_epi8( - 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, 0, -1, -1, -1, 1, -1, -1, -1, - 2, -1, -1, -1, 3, -1, -1, -1, - ); - if CN == 4 { - let v0 = _mm_loadu_si32(ptr0 as *const _); - let v1 = _mm_loadu_si32(ptr1 as *const _); - - _mm256_shuffle_epi8( - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(v0), v1), - sh1, - ) - } else if CN == 3 { - let mut v0 = _mm_loadu_si16(ptr0); - v0 = _mm_insert_epi8::<2>(v0, ptr0.add(2).read_unaligned() as i32); - let mut v1 = _mm_loadu_si16(ptr1); - v1 = _mm_insert_epi8::<2>(v1, ptr1.add(2).read_unaligned() as i32); - _mm256_shuffle_epi8( - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(v0), v1), - sh1, - ) - } else if CN == 2 { - let v0 = _mm_loadu_si16(ptr0); - let v1 = _mm_loadu_si16(ptr1); - _mm256_shuffle_epi8( - _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(v0), v1), - sh1, - ) - } else { - _mm256_shuffle_epi8( - _mm256_inserti128_si256::<1>( - _mm256_castsi128_si256(_mm_cvtsi32_si128(ptr0.read_unaligned() as i32)), - _mm_cvtsi32_si128(ptr1.read_unaligned() as i32), - ), - sh1, - ) + unsafe { + let sh1 = _mm256_setr_epi8( + 0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1, 0, -1, -1, -1, 1, -1, -1, + -1, 2, -1, -1, -1, 3, -1, -1, -1, + ); + if CN == 4 { + let v0 = _mm_loadu_si32(ptr0 as *const _); + let v1 = _mm_loadu_si32(ptr1 as *const _); + + _mm256_shuffle_epi8( + _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(v0), v1), + sh1, + ) + } else if CN == 3 { + let mut v0 = _mm_loadu_si16(ptr0); + v0 = _mm_insert_epi8::<2>(v0, ptr0.add(2).read_unaligned() as i32); + let mut v1 = _mm_loadu_si16(ptr1); + v1 = _mm_insert_epi8::<2>(v1, ptr1.add(2).read_unaligned() as i32); + _mm256_shuffle_epi8( + _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(v0), v1), + sh1, + ) + } else if CN == 2 { + let v0 = _mm_loadu_si16(ptr0); + let v1 = _mm_loadu_si16(ptr1); + _mm256_shuffle_epi8( + _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(v0), v1), + sh1, + ) + } else { + _mm256_shuffle_epi8( + _mm256_inserti128_si256::<1>( + _mm256_castsi128_si256(_mm_cvtsi32_si128(ptr0.read_unaligned() as i32)), + _mm_cvtsi32_si128(ptr1.read_unaligned() as i32), + ), + sh1, + ) + } } } #[target_feature(enable = "avx2")] -unsafe fn box_blur_horizontal_pass_impl( +fn box_blur_horizontal_pass_impl( src: &[u8], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -113,7 +115,7 @@ unsafe fn box_blur_horizontal_pass_impl( let mut yy = start_y; - while yy + 6 < end_y { + while yy + 6 <= end_y { let y = yy; let y_src_shift = y as usize * src_stride as usize; let y_dst_shift = y as usize * dst_stride as usize; diff --git a/src/box_filter/avx/rgba_f32.rs b/src/box_filter/avx/rgba_f32.rs index d71be736..77f7aa74 100644 --- a/src/box_filter/avx/rgba_f32.rs +++ b/src/box_filter/avx/rgba_f32.rs @@ -51,30 +51,32 @@ pub(crate) fn box_blur_horizontal_pass_avx_f32( #[inline(always)] pub(crate) unsafe fn load_f32_x2(ptr0: *const f32, ptr1: *const f32) -> __m256 { - if CN == 4 { - let x0 = _mm_loadu_ps(ptr0); - let x1 = _mm_loadu_ps(ptr1); - _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(x0), x1) - } else if CN == 3 { - let mut j0 = _mm_loadu_si64(ptr0 as *const _); - let mut j1 = _mm_loadu_si64(ptr1 as *const _); - j0 = _mm_insert_epi32::<2>(j0, ptr0.add(2).read_unaligned().to_bits() as i32); - j1 = _mm_insert_epi32::<2>(j1, ptr1.add(2).read_unaligned().to_bits() as i32); - _mm256_insertf128_ps::<1>( - _mm256_castps128_ps256(_mm_castsi128_ps(j0)), - _mm_castsi128_ps(j1), - ) - } else if CN == 2 { - let j0 = _mm_loadu_si64(ptr0 as *const _); - let j1 = _mm_loadu_si64(ptr1 as *const _); - _mm256_insertf128_ps::<1>( - _mm256_castps128_ps256(_mm_castsi128_ps(j0)), - _mm_castsi128_ps(j1), - ) - } else { - let x0 = _mm_load_ss(ptr0 as *const _); - let x1 = _mm_load_ss(ptr1 as *const _); - _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(x0), x1) + unsafe { + if CN == 4 { + let x0 = _mm_loadu_ps(ptr0); + let x1 = _mm_loadu_ps(ptr1); + _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(x0), x1) + } else if CN == 3 { + let mut j0 = _mm_loadu_si64(ptr0 as *const _); + let mut j1 = _mm_loadu_si64(ptr1 as *const _); + j0 = _mm_insert_epi32::<2>(j0, ptr0.add(2).read_unaligned().to_bits() as i32); + j1 = _mm_insert_epi32::<2>(j1, ptr1.add(2).read_unaligned().to_bits() as i32); + _mm256_insertf128_ps::<1>( + _mm256_castps128_ps256(_mm_castsi128_ps(j0)), + _mm_castsi128_ps(j1), + ) + } else if CN == 2 { + let j0 = _mm_loadu_si64(ptr0 as *const _); + let j1 = _mm_loadu_si64(ptr1 as *const _); + _mm256_insertf128_ps::<1>( + _mm256_castps128_ps256(_mm_castsi128_ps(j0)), + _mm_castsi128_ps(j1), + ) + } else { + let x0 = _mm_load_ss(ptr0 as *const _); + let x1 = _mm_load_ss(ptr1 as *const _); + _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(x0), x1) + } } } @@ -83,7 +85,7 @@ struct HorizontalExecutionUnit {} impl HorizontalExecutionUnit { #[target_feature(enable = "avx2")] - unsafe fn pass( + fn pass( &self, src: &[f32], src_stride: u32, @@ -104,7 +106,7 @@ impl HorizontalExecutionUnit { let mut yy = start_y; - while yy + 6 < end_y { + while yy + 6 <= end_y { let y = yy; let y_src_shift = y as usize * src_stride as usize; let y_dst_shift = y as usize * dst_stride as usize; diff --git a/src/box_filter/avx/vrgba8.rs b/src/box_filter/avx/vrgba8.rs index 1b295544..e89a9856 100644 --- a/src/box_filter/avx/vrgba8.rs +++ b/src/box_filter/avx/vrgba8.rs @@ -77,7 +77,7 @@ unsafe fn box_blur_vertical_pass_avx2_def( let v_edge_count = _mm256_set1_epi32(edge_count as i32); let v_weight = _mm256_set1_ps(1f32 / (radius * 2 + 1) as f32); - while cx + 64 < end_x { + while cx + 64 <= end_x { let px = cx as usize; let mut store_0: __m256i; @@ -285,7 +285,7 @@ unsafe fn box_blur_vertical_pass_avx2_def( cx += 64; } - while cx + 32 < end_x { + while cx + 32 <= end_x { let px = cx as usize; let mut store_0: __m256i; @@ -415,7 +415,7 @@ unsafe fn box_blur_vertical_pass_avx2_def( let v_edge_count = _mm_set1_epi32(edge_count as i32); let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); - while cx + 16 < end_x { + while cx + 16 <= end_x { let px = cx as usize; let mut store_0: __m128i; @@ -543,7 +543,7 @@ unsafe fn box_blur_vertical_pass_avx2_def( cx += 16; } - while cx + 8 < end_x { + while cx + 8 <= end_x { let px = cx as usize; let mut store_0: __m128i; diff --git a/src/box_filter/avx/vrgba8_low_radius.rs b/src/box_filter/avx/vrgba8_low_radius.rs index 612bb5b7..9b6eb262 100644 --- a/src/box_filter/avx/vrgba8_low_radius.rs +++ b/src/box_filter/avx/vrgba8_low_radius.rs @@ -29,10 +29,11 @@ use crate::avx::shuffle; use crate::sse::{load_u8_s16_fast, store_u8_s16}; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::arch::x86_64::*; #[target_feature(enable = "avx2")] -pub unsafe fn box_blur_vertical_pass_avx2_def_lr( +pub fn box_blur_vertical_pass_avx2_def_lr( src: &[u8], src_stride: u32, dst: &UnsafeSlice, @@ -46,7 +47,7 @@ pub unsafe fn box_blur_vertical_pass_avx2_def_lr( let unit = ExecutionUnit::default(); unit.execute( src, src_stride, dst, dst_stride, w, height, radius, start_x, end_x, - ); + ) } #[derive(Debug, Default)] @@ -54,7 +55,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "avx2")] - unsafe fn execute( + fn execute( &self, src: &[u8], src_stride: u32, @@ -66,54 +67,38 @@ impl ExecutionUnit { start_x: u32, end_x: u32, ) { - let kernel_size = radius * 2 + 1; - let edge_count = (kernel_size / 2) + 1; - let half_kernel = kernel_size / 2; + unsafe { + let kernel_size = radius * 2 + 1; + let edge_count = (kernel_size / 2) + 1; + let half_kernel = kernel_size / 2; - let mut cx = start_x; + let mut cx = start_x; - let v_edge_count = _mm256_set1_epi8(edge_count as i8); - let v_weight = _mm256_set1_ps(1f32 / (radius * 2 + 1) as f32); + let v_edge_count = _mm256_set1_epi8(edge_count as i8); + let v_weight = _mm256_set1_ps(1f32 / (radius * 2 + 1) as f32); - assert!(end_x >= start_x); + assert!(end_x >= start_x); - let buf_size = end_x as usize - start_x as usize; + let buf_size = end_x as usize - start_x as usize; - let buf_cap = buf_size.div_ceil(16) * 16 + 16; + let buf_cap = buf_size.div_ceil(16) * 16 + 16; - let mut buffer = vec![0u16; buf_cap]; + let mut scratch_buffer = ScratchBuffer::::new(buf_cap); + let buffer = scratch_buffer.as_mut_slice(); - // Configure initial accumulator + // Configure initial accumulator - let mut buf_cx = 0usize; - - while cx + 64 < end_x { - let px = cx as usize; - - let mut store_0: __m256i; - let mut store_1: __m256i; - let mut store_2: __m256i; - let mut store_3: __m256i; - - let s_ptr = src.as_ptr().add(px); - let edge0 = _mm256_loadu_si256(s_ptr as *const _); - let edge1 = _mm256_loadu_si256(s_ptr.add(32) as *const _); - let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); - let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); - - let lo1 = _mm256_unpacklo_epi8(edge1, _mm256_setzero_si256()); - let hi1 = _mm256_unpackhi_epi8(edge1, _mm256_setzero_si256()); - - store_0 = _mm256_maddubs_epi16(lo0, v_edge_count); - store_1 = _mm256_maddubs_epi16(hi0, v_edge_count); + let mut buf_cx = 0usize; - store_2 = _mm256_maddubs_epi16(lo1, v_edge_count); - store_3 = _mm256_maddubs_epi16(hi1, v_edge_count); + while cx + 64 <= end_x { + let px = cx as usize; - for y in 1..=half_kernel as usize { - let y_src_shift = y.min(height as usize - 1) * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + let mut store_0: __m256i; + let mut store_1: __m256i; + let mut store_2: __m256i; + let mut store_3: __m256i; + let s_ptr = src.as_ptr().add(px); let edge0 = _mm256_loadu_si256(s_ptr as *const _); let edge1 = _mm256_loadu_si256(s_ptr.add(32) as *const _); let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); @@ -122,531 +107,571 @@ impl ExecutionUnit { let lo1 = _mm256_unpacklo_epi8(edge1, _mm256_setzero_si256()); let hi1 = _mm256_unpackhi_epi8(edge1, _mm256_setzero_si256()); - store_0 = _mm256_adds_epu16(store_0, lo0); - store_1 = _mm256_adds_epu16(store_1, hi0); + store_0 = _mm256_maddubs_epi16(lo0, v_edge_count); + store_1 = _mm256_maddubs_epi16(hi0, v_edge_count); - store_2 = _mm256_adds_epu16(store_2, lo1); - store_3 = _mm256_adds_epu16(store_3, hi1); - } + store_2 = _mm256_maddubs_epi16(lo1, v_edge_count); + store_3 = _mm256_maddubs_epi16(hi1, v_edge_count); - _mm256_storeu_si256( - buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, - store_0, - ); - _mm256_storeu_si256( - buffer.get_unchecked_mut(buf_cx + 16..).as_mut_ptr() as *mut _, - store_1, - ); - _mm256_storeu_si256( - buffer.get_unchecked_mut(buf_cx + 32..).as_mut_ptr() as *mut _, - store_2, - ); - _mm256_storeu_si256( - buffer.get_unchecked_mut(buf_cx + 48..).as_mut_ptr() as *mut _, - store_3, - ); - - buf_cx += 64; - cx += 64; - } + for y in 1..=half_kernel as usize { + let y_src_shift = y.min(height as usize - 1) * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); + + let edge0 = _mm256_loadu_si256(s_ptr as *const _); + let edge1 = _mm256_loadu_si256(s_ptr.add(32) as *const _); + let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); + let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); + + let lo1 = _mm256_unpacklo_epi8(edge1, _mm256_setzero_si256()); + let hi1 = _mm256_unpackhi_epi8(edge1, _mm256_setzero_si256()); + + store_0 = _mm256_adds_epu16(store_0, lo0); + store_1 = _mm256_adds_epu16(store_1, hi0); - while cx + 32 < end_x { - let px = cx as usize; + store_2 = _mm256_adds_epu16(store_2, lo1); + store_3 = _mm256_adds_epu16(store_3, hi1); + } - let mut store_0: __m256i; - let mut store_1: __m256i; + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, + store_0, + ); + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx + 16..).as_mut_ptr() as *mut _, + store_1, + ); + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx + 32..).as_mut_ptr() as *mut _, + store_2, + ); + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx + 48..).as_mut_ptr() as *mut _, + store_3, + ); - let s_ptr = src.as_ptr().add(px); - let edge0 = _mm256_loadu_si256(s_ptr as *const _); - let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); - let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); + buf_cx += 64; + cx += 64; + } - store_0 = _mm256_maddubs_epi16(lo0, v_edge_count); - store_1 = _mm256_maddubs_epi16(hi0, v_edge_count); + while cx + 32 <= end_x { + let px = cx as usize; - for y in 1..=half_kernel as usize { - let y_src_shift = y.min(height as usize - 1) * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + let mut store_0: __m256i; + let mut store_1: __m256i; + let s_ptr = src.as_ptr().add(px); let edge0 = _mm256_loadu_si256(s_ptr as *const _); let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); - store_0 = _mm256_adds_epu16(store_0, lo0); - store_1 = _mm256_adds_epu16(store_1, hi0); - } - - _mm256_storeu_si256( - buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, - store_0, - ); - _mm256_storeu_si256( - buffer.get_unchecked_mut(buf_cx + 16..).as_mut_ptr() as *mut _, - store_1, - ); - - buf_cx += 32; - cx += 32; - } + store_0 = _mm256_maddubs_epi16(lo0, v_edge_count); + store_1 = _mm256_maddubs_epi16(hi0, v_edge_count); - while cx + 16 < end_x { - let px = cx as usize; + for y in 1..=half_kernel as usize { + let y_src_shift = y.min(height as usize - 1) * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - let mut store_0: __m256i; + let edge0 = _mm256_loadu_si256(s_ptr as *const _); + let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); + let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); - let s_ptr = src.as_ptr().add(px); - let edge0 = _mm256_castsi128_si256(_mm_loadu_si128(s_ptr as *const _)); + store_0 = _mm256_adds_epu16(store_0, lo0); + store_1 = _mm256_adds_epu16(store_1, hi0); + } - const MASK: i32 = shuffle(3, 1, 2, 0); + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, + store_0, + ); + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx + 16..).as_mut_ptr() as *mut _, + store_1, + ); - let lo0 = _mm256_unpacklo_epi8( - _mm256_permute4x64_epi64::(edge0), - _mm256_setzero_si256(), - ); + buf_cx += 32; + cx += 32; + } - store_0 = _mm256_maddubs_epi16(lo0, v_edge_count); + while cx + 16 <= end_x { + let px = cx as usize; - for y in 1..=half_kernel as usize { - let y_src_shift = y.min(height as usize - 1) * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + let mut store_0: __m256i; + let s_ptr = src.as_ptr().add(px); let edge0 = _mm256_castsi128_si256(_mm_loadu_si128(s_ptr as *const _)); + + const MASK: i32 = shuffle(3, 1, 2, 0); + let lo0 = _mm256_unpacklo_epi8( _mm256_permute4x64_epi64::(edge0), _mm256_setzero_si256(), ); - store_0 = _mm256_adds_epu16(store_0, lo0); - } - - _mm256_storeu_si256( - buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, - store_0, - ); + store_0 = _mm256_maddubs_epi16(lo0, v_edge_count); - buf_cx += 16; - cx += 16; - } + for y in 1..=half_kernel as usize { + let y_src_shift = y.min(height as usize - 1) * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - let v_edge_count = _mm_set1_epi8(edge_count as i8); + let edge0 = _mm256_castsi128_si256(_mm_loadu_si128(s_ptr as *const _)); + let lo0 = _mm256_unpacklo_epi8( + _mm256_permute4x64_epi64::(edge0), + _mm256_setzero_si256(), + ); - while cx + 8 < end_x { - let px = cx as usize; + store_0 = _mm256_adds_epu16(store_0, lo0); + } - let s_ptr = src.as_ptr().add(px); - let edge0 = _mm_loadu_si64(s_ptr as *const _); + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, + store_0, + ); - let lo0 = _mm_unpacklo_epi8(edge0, _mm_setzero_si128()); + buf_cx += 16; + cx += 16; + } - let mut store_0 = _mm_maddubs_epi16(lo0, v_edge_count); + let v_edge_count = _mm_set1_epi8(edge_count as i8); - for y in 1..=half_kernel as usize { - let y_src_shift = y.min(height as usize - 1) * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + while cx + 8 <= end_x { + let px = cx as usize; + let s_ptr = src.as_ptr().add(px); let edge0 = _mm_loadu_si64(s_ptr as *const _); + let lo0 = _mm_unpacklo_epi8(edge0, _mm_setzero_si128()); - store_0 = _mm_adds_epu16(store_0, lo0); - } + let mut store_0 = _mm_maddubs_epi16(lo0, v_edge_count); - _mm_storeu_si128( - buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, - store_0, - ); + for y in 1..=half_kernel as usize { + let y_src_shift = y.min(height as usize - 1) * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - buf_cx += 8; - cx += 8; - } + let edge0 = _mm_loadu_si64(s_ptr as *const _); + let lo0 = _mm_unpacklo_epi8(edge0, _mm_setzero_si128()); - while cx < end_x { - let px = cx as usize; - - let s_ptr = src.get_unchecked(px); - let edge0 = _mm_setr_epi16(*s_ptr as i16, 0, 0, 0, 0, 0, 0, 0); + store_0 = _mm_adds_epu16(store_0, lo0); + } - let lo0 = _mm_unpacklo_epi8(edge0, _mm_setzero_si128()); + _mm_storeu_si128( + buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, + store_0, + ); - let mut store_0 = _mm_maddubs_epi16(lo0, v_edge_count); + buf_cx += 8; + cx += 8; + } - for y in 1..=half_kernel as usize { - let y_src_shift = y.min(height as usize - 1) * src_stride as usize; - let s_ptr = src.get_unchecked(y_src_shift + px); + while cx < end_x { + let px = cx as usize; + let s_ptr = src.get_unchecked(px); let edge0 = _mm_setr_epi16(*s_ptr as i16, 0, 0, 0, 0, 0, 0, 0); + let lo0 = _mm_unpacklo_epi8(edge0, _mm_setzero_si128()); - store_0 = _mm_adds_epu16(store_0, lo0); + let mut store_0 = _mm_maddubs_epi16(lo0, v_edge_count); + + for y in 1..=half_kernel as usize { + let y_src_shift = y.min(height as usize - 1) * src_stride as usize; + let s_ptr = src.get_unchecked(y_src_shift + px); + + let edge0 = _mm_setr_epi16(*s_ptr as i16, 0, 0, 0, 0, 0, 0, 0); + let lo0 = _mm_unpacklo_epi8(edge0, _mm_setzero_si128()); + + store_0 = _mm_adds_epu16(store_0, lo0); + } + + _mm_storeu_si16( + buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, + store_0, + ); + + buf_cx += 1; + cx += 1; } - _mm_storeu_si16( - buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, - store_0, - ); + for y in 0..height as usize { + let mut buf_cx = 0usize; + let mut cx = start_x as usize; - buf_cx += 1; - cx += 1; - } + // preload edge pixels + let next = + (y + half_kernel as usize + 1).min(height as usize - 1) * src_stride as usize; + let previous = + (y as i64 - half_kernel as i64).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y; - for y in 0..height as usize { - let mut buf_cx = 0usize; - let mut cx = start_x as usize; - - // preload edge pixels - let next = - (y + half_kernel as usize + 1).min(height as usize - 1) * src_stride as usize; - let previous = (y as i64 - half_kernel as i64).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y; - - while cx + 64 < end_x as usize { - let px = cx; - - let mut store_0 = - _mm256_loadu_si256(buffer.get_unchecked(buf_cx..).as_ptr() as *const _); - let mut store_1 = - _mm256_loadu_si256(buffer.get_unchecked(buf_cx + 16..).as_ptr() as *const _); - let mut store_2 = - _mm256_loadu_si256(buffer.get_unchecked(buf_cx + 32..).as_ptr() as *const _); - let mut store_3 = - _mm256_loadu_si256(buffer.get_unchecked(buf_cx + 48..).as_ptr() as *const _); - - { + while cx + 64 <= end_x as usize { let px = cx; - let jw0 = _mm256_unpacklo_epi16(store_0, _mm256_setzero_si256()); - let jw1 = _mm256_unpackhi_epi16(store_0, _mm256_setzero_si256()); - let jw2 = _mm256_unpacklo_epi16(store_1, _mm256_setzero_si256()); - let jw3 = _mm256_unpackhi_epi16(store_1, _mm256_setzero_si256()); - let jw4 = _mm256_unpacklo_epi16(store_2, _mm256_setzero_si256()); - let jw5 = _mm256_unpackhi_epi16(store_2, _mm256_setzero_si256()); - let jw6 = _mm256_unpacklo_epi16(store_3, _mm256_setzero_si256()); - let jw7 = _mm256_unpackhi_epi16(store_3, _mm256_setzero_si256()); - - let store0_ps = _mm256_cvtepi32_ps(jw0); - let store1_ps = _mm256_cvtepi32_ps(jw1); - let store2_ps = _mm256_cvtepi32_ps(jw2); - let store3_ps = _mm256_cvtepi32_ps(jw3); - let store4_ps = _mm256_cvtepi32_ps(jw4); - let store5_ps = _mm256_cvtepi32_ps(jw5); - let store6_ps = _mm256_cvtepi32_ps(jw6); - let store7_ps = _mm256_cvtepi32_ps(jw7); - - let r0 = _mm256_mul_ps(store0_ps, v_weight); - let r1 = _mm256_mul_ps(store1_ps, v_weight); - let r2 = _mm256_mul_ps(store2_ps, v_weight); - let r3 = _mm256_mul_ps(store3_ps, v_weight); - let r4 = _mm256_mul_ps(store4_ps, v_weight); - let r5 = _mm256_mul_ps(store5_ps, v_weight); - let r6 = _mm256_mul_ps(store6_ps, v_weight); - let r7 = _mm256_mul_ps(store7_ps, v_weight); - - let scale_store_0 = _mm256_cvtps_epi32(r0); - let scale_store_1 = _mm256_cvtps_epi32(r1); - let scale_store_2 = _mm256_cvtps_epi32(r2); - let scale_store_3 = _mm256_cvtps_epi32(r3); - let scale_store_4 = _mm256_cvtps_epi32(r4); - let scale_store_5 = _mm256_cvtps_epi32(r5); - let scale_store_6 = _mm256_cvtps_epi32(r6); - let scale_store_7 = _mm256_cvtps_epi32(r7); - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - - let set0 = _mm256_packus_epi32(scale_store_0, scale_store_1); - let set1 = _mm256_packus_epi32(scale_store_2, scale_store_3); - let set2 = _mm256_packus_epi32(scale_store_4, scale_store_5); - let set3 = _mm256_packus_epi32(scale_store_6, scale_store_7); - - let full_set0 = _mm256_packus_epi16(set0, set1); - let full_set1 = _mm256_packus_epi16(set2, set3); - _mm256_storeu_si256(ptr as *mut _, full_set0); - _mm256_storeu_si256(ptr.add(32) as *mut _, full_set1); + let mut store_0 = + _mm256_loadu_si256(buffer.get_unchecked(buf_cx..).as_ptr() as *const _); + let mut store_1 = _mm256_loadu_si256( + buffer.get_unchecked(buf_cx + 16..).as_ptr() as *const _, + ); + let mut store_2 = _mm256_loadu_si256( + buffer.get_unchecked(buf_cx + 32..).as_ptr() as *const _, + ); + let mut store_3 = _mm256_loadu_si256( + buffer.get_unchecked(buf_cx + 48..).as_ptr() as *const _, + ); + + { + let px = cx; + + let jw0 = _mm256_unpacklo_epi16(store_0, _mm256_setzero_si256()); + let jw1 = _mm256_unpackhi_epi16(store_0, _mm256_setzero_si256()); + let jw2 = _mm256_unpacklo_epi16(store_1, _mm256_setzero_si256()); + let jw3 = _mm256_unpackhi_epi16(store_1, _mm256_setzero_si256()); + let jw4 = _mm256_unpacklo_epi16(store_2, _mm256_setzero_si256()); + let jw5 = _mm256_unpackhi_epi16(store_2, _mm256_setzero_si256()); + let jw6 = _mm256_unpacklo_epi16(store_3, _mm256_setzero_si256()); + let jw7 = _mm256_unpackhi_epi16(store_3, _mm256_setzero_si256()); + + let store0_ps = _mm256_cvtepi32_ps(jw0); + let store1_ps = _mm256_cvtepi32_ps(jw1); + let store2_ps = _mm256_cvtepi32_ps(jw2); + let store3_ps = _mm256_cvtepi32_ps(jw3); + let store4_ps = _mm256_cvtepi32_ps(jw4); + let store5_ps = _mm256_cvtepi32_ps(jw5); + let store6_ps = _mm256_cvtepi32_ps(jw6); + let store7_ps = _mm256_cvtepi32_ps(jw7); + + let r0 = _mm256_mul_ps(store0_ps, v_weight); + let r1 = _mm256_mul_ps(store1_ps, v_weight); + let r2 = _mm256_mul_ps(store2_ps, v_weight); + let r3 = _mm256_mul_ps(store3_ps, v_weight); + let r4 = _mm256_mul_ps(store4_ps, v_weight); + let r5 = _mm256_mul_ps(store5_ps, v_weight); + let r6 = _mm256_mul_ps(store6_ps, v_weight); + let r7 = _mm256_mul_ps(store7_ps, v_weight); + + let scale_store_0 = _mm256_cvtps_epi32(r0); + let scale_store_1 = _mm256_cvtps_epi32(r1); + let scale_store_2 = _mm256_cvtps_epi32(r2); + let scale_store_3 = _mm256_cvtps_epi32(r3); + let scale_store_4 = _mm256_cvtps_epi32(r4); + let scale_store_5 = _mm256_cvtps_epi32(r5); + let scale_store_6 = _mm256_cvtps_epi32(r6); + let scale_store_7 = _mm256_cvtps_epi32(r7); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + + let set0 = _mm256_packus_epi32(scale_store_0, scale_store_1); + let set1 = _mm256_packus_epi32(scale_store_2, scale_store_3); + let set2 = _mm256_packus_epi32(scale_store_4, scale_store_5); + let set3 = _mm256_packus_epi32(scale_store_6, scale_store_7); + + let full_set0 = _mm256_packus_epi16(set0, set1); + let full_set1 = _mm256_packus_epi16(set2, set3); + _mm256_storeu_si256(ptr as *mut _, full_set0); + _mm256_storeu_si256(ptr.add(32) as *mut _, full_set1); + } + + { + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge0 = _mm256_loadu_si256(s_ptr as *const _); + let edge1 = _mm256_loadu_si256(s_ptr.add(32) as *const _); + let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); + let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); + + let lo1 = _mm256_unpacklo_epi8(edge1, _mm256_setzero_si256()); + let hi1 = _mm256_unpackhi_epi8(edge1, _mm256_setzero_si256()); + + store_0 = _mm256_subs_epu16(store_0, lo0); + store_1 = _mm256_subs_epu16(store_1, hi0); + + store_2 = _mm256_subs_epu16(store_2, lo1); + store_3 = _mm256_subs_epu16(store_3, hi1); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge0 = _mm256_loadu_si256(s_ptr as *const _); + let edge1 = _mm256_loadu_si256(s_ptr.add(32) as *const _); + let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); + let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); + + let lo1 = _mm256_unpacklo_epi8(edge1, _mm256_setzero_si256()); + let hi1 = _mm256_unpackhi_epi8(edge1, _mm256_setzero_si256()); + + store_0 = _mm256_adds_epu16(store_0, lo0); + store_1 = _mm256_adds_epu16(store_1, hi0); + + store_2 = _mm256_adds_epu16(store_2, lo1); + store_3 = _mm256_adds_epu16(store_3, hi1); + } + + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, + store_0, + ); + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx + 16..).as_mut_ptr() as *mut _, + store_1, + ); + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx + 32..).as_mut_ptr() as *mut _, + store_2, + ); + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx + 48..).as_mut_ptr() as *mut _, + store_3, + ); + } + + cx += 64; + buf_cx += 64; } - unsafe { + while cx + 32 <= end_x as usize { + let px = cx; + + let mut store_0 = + _mm256_loadu_si256(buffer.get_unchecked(buf_cx..).as_ptr() as *const _); + let mut store_1 = _mm256_loadu_si256( + buffer.get_unchecked(buf_cx + 16..).as_ptr() as *const _, + ); + + { + let px = cx; + + let jw0 = _mm256_unpacklo_epi16(store_0, _mm256_setzero_si256()); + let jw1 = _mm256_unpackhi_epi16(store_0, _mm256_setzero_si256()); + let jw2 = _mm256_unpacklo_epi16(store_1, _mm256_setzero_si256()); + let jw3 = _mm256_unpackhi_epi16(store_1, _mm256_setzero_si256()); + + let store0_ps = _mm256_cvtepi32_ps(jw0); + let store1_ps = _mm256_cvtepi32_ps(jw1); + let store2_ps = _mm256_cvtepi32_ps(jw2); + let store3_ps = _mm256_cvtepi32_ps(jw3); + + let r0 = _mm256_mul_ps(store0_ps, v_weight); + let r1 = _mm256_mul_ps(store1_ps, v_weight); + let r2 = _mm256_mul_ps(store2_ps, v_weight); + let r3 = _mm256_mul_ps(store3_ps, v_weight); + + let scale_store_0 = _mm256_cvtps_epi32(r0); + let scale_store_1 = _mm256_cvtps_epi32(r1); + let scale_store_2 = _mm256_cvtps_epi32(r2); + let scale_store_3 = _mm256_cvtps_epi32(r3); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + + let set0 = _mm256_packus_epi32(scale_store_0, scale_store_1); + let set1 = _mm256_packus_epi32(scale_store_2, scale_store_3); + + let full_set0 = _mm256_packus_epi16(set0, set1); + _mm256_storeu_si256(ptr as *mut _, full_set0); + } + // subtract previous { let s_ptr = src.as_ptr().add(previous + px); let edge0 = _mm256_loadu_si256(s_ptr as *const _); - let edge1 = _mm256_loadu_si256(s_ptr.add(32) as *const _); let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); - let lo1 = _mm256_unpacklo_epi8(edge1, _mm256_setzero_si256()); - let hi1 = _mm256_unpackhi_epi8(edge1, _mm256_setzero_si256()); - store_0 = _mm256_subs_epu16(store_0, lo0); store_1 = _mm256_subs_epu16(store_1, hi0); - - store_2 = _mm256_subs_epu16(store_2, lo1); - store_3 = _mm256_subs_epu16(store_3, hi1); } // add next { let s_ptr = src.as_ptr().add(next + px); let edge0 = _mm256_loadu_si256(s_ptr as *const _); - let edge1 = _mm256_loadu_si256(s_ptr.add(32) as *const _); let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); - let lo1 = _mm256_unpacklo_epi8(edge1, _mm256_setzero_si256()); - let hi1 = _mm256_unpackhi_epi8(edge1, _mm256_setzero_si256()); - store_0 = _mm256_adds_epu16(store_0, lo0); store_1 = _mm256_adds_epu16(store_1, hi0); - - store_2 = _mm256_adds_epu16(store_2, lo1); - store_3 = _mm256_adds_epu16(store_3, hi1); } - _mm256_storeu_si256(buffer.get_unchecked(buf_cx..).as_ptr() as *mut _, store_0); _mm256_storeu_si256( - buffer.get_unchecked(buf_cx + 16..).as_ptr() as *mut _, - store_1, + buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, + store_0, ); _mm256_storeu_si256( - buffer.get_unchecked(buf_cx + 32..).as_ptr() as *mut _, - store_2, - ); - _mm256_storeu_si256( - buffer.get_unchecked(buf_cx + 48..).as_ptr() as *mut _, - store_3, + buffer.get_unchecked_mut(buf_cx + 16..).as_mut_ptr() as *mut _, + store_1, ); - } - - cx += 64; - buf_cx += 64; - } - - while cx + 32 < end_x as usize { - let px = cx; - - let mut store_0 = - _mm256_loadu_si256(buffer.get_unchecked(buf_cx..).as_ptr() as *const _); - let mut store_1 = - _mm256_loadu_si256(buffer.get_unchecked(buf_cx + 16..).as_ptr() as *const _); - - { - let px = cx; - - let jw0 = _mm256_unpacklo_epi16(store_0, _mm256_setzero_si256()); - let jw1 = _mm256_unpackhi_epi16(store_0, _mm256_setzero_si256()); - let jw2 = _mm256_unpacklo_epi16(store_1, _mm256_setzero_si256()); - let jw3 = _mm256_unpackhi_epi16(store_1, _mm256_setzero_si256()); - - let store0_ps = _mm256_cvtepi32_ps(jw0); - let store1_ps = _mm256_cvtepi32_ps(jw1); - let store2_ps = _mm256_cvtepi32_ps(jw2); - let store3_ps = _mm256_cvtepi32_ps(jw3); - - let r0 = _mm256_mul_ps(store0_ps, v_weight); - let r1 = _mm256_mul_ps(store1_ps, v_weight); - let r2 = _mm256_mul_ps(store2_ps, v_weight); - let r3 = _mm256_mul_ps(store3_ps, v_weight); - - let scale_store_0 = _mm256_cvtps_epi32(r0); - let scale_store_1 = _mm256_cvtps_epi32(r1); - let scale_store_2 = _mm256_cvtps_epi32(r2); - let scale_store_3 = _mm256_cvtps_epi32(r3); - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - - let set0 = _mm256_packus_epi32(scale_store_0, scale_store_1); - let set1 = _mm256_packus_epi32(scale_store_2, scale_store_3); - - let full_set0 = _mm256_packus_epi16(set0, set1); - _mm256_storeu_si256(ptr as *mut _, full_set0); - } - - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge0 = _mm256_loadu_si256(s_ptr as *const _); - let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); - let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); - store_0 = _mm256_subs_epu16(store_0, lo0); - store_1 = _mm256_subs_epu16(store_1, hi0); + cx += 32; + buf_cx += 32; } - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge0 = _mm256_loadu_si256(s_ptr as *const _); - let lo0 = _mm256_unpacklo_epi8(edge0, _mm256_setzero_si256()); - let hi0 = _mm256_unpackhi_epi8(edge0, _mm256_setzero_si256()); - - store_0 = _mm256_adds_epu16(store_0, lo0); - store_1 = _mm256_adds_epu16(store_1, hi0); - } - - _mm256_storeu_si256(buffer.get_unchecked(buf_cx..).as_ptr() as *mut _, store_0); - _mm256_storeu_si256( - buffer.get_unchecked(buf_cx + 16..).as_ptr() as *mut _, - store_1, - ); + while cx + 16 <= end_x as usize { + let px = cx; - cx += 32; - buf_cx += 32; - } + let mut store_0 = + _mm256_loadu_si256(buffer.get_unchecked(buf_cx..).as_ptr() as *const _); - while cx + 16 < end_x as usize { - let px = cx; + const MASK: i32 = shuffle(3, 1, 2, 0); - let mut store_0 = - _mm256_loadu_si256(buffer.get_unchecked(buf_cx..).as_ptr() as *const _); + { + let px = cx; - const MASK: i32 = shuffle(3, 1, 2, 0); + let jw0 = _mm256_unpacklo_epi16(store_0, _mm256_setzero_si256()); + let jw1 = _mm256_unpackhi_epi16(store_0, _mm256_setzero_si256()); - { - let px = cx; + let store0_ps = _mm256_cvtepi32_ps(jw0); + let store1_ps = _mm256_cvtepi32_ps(jw1); - let jw0 = _mm256_unpacklo_epi16(store_0, _mm256_setzero_si256()); - let jw1 = _mm256_unpackhi_epi16(store_0, _mm256_setzero_si256()); + let r0 = _mm256_mul_ps(store0_ps, v_weight); + let r1 = _mm256_mul_ps(store1_ps, v_weight); - let store0_ps = _mm256_cvtepi32_ps(jw0); - let store1_ps = _mm256_cvtepi32_ps(jw1); + let scale_store_0 = _mm256_cvtps_epi32(r0); + let scale_store_1 = _mm256_cvtps_epi32(r1); - let r0 = _mm256_mul_ps(store0_ps, v_weight); - let r1 = _mm256_mul_ps(store1_ps, v_weight); + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let scale_store_0 = _mm256_cvtps_epi32(r0); - let scale_store_1 = _mm256_cvtps_epi32(r1); + let set0 = _mm256_packus_epi32(scale_store_0, scale_store_1); - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let full_set0 = _mm256_packus_epi16(set0, set0); + _mm_storeu_si128( + ptr as *mut _, + _mm256_castsi256_si128(_mm256_permute4x64_epi64::(full_set0)), + ); + } - let set0 = _mm256_packus_epi32(scale_store_0, scale_store_1); + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge0 = _mm256_castsi128_si256(_mm_loadu_si128(s_ptr as *const _)); + let lo0 = _mm256_unpacklo_epi8( + _mm256_permute4x64_epi64::(edge0), + _mm256_setzero_si256(), + ); - let full_set0 = _mm256_packus_epi16(set0, set0); - _mm_storeu_si128( - ptr as *mut _, - _mm256_castsi256_si128(_mm256_permute4x64_epi64::(full_set0)), - ); - } + store_0 = _mm256_subs_epu16(store_0, lo0); + } - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge0 = _mm256_castsi128_si256(_mm_loadu_si128(s_ptr as *const _)); - let lo0 = _mm256_unpacklo_epi8( - _mm256_permute4x64_epi64::(edge0), - _mm256_setzero_si256(), - ); + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge0 = _mm256_castsi128_si256(_mm_loadu_si128(s_ptr as *const _)); + let lo0 = _mm256_unpacklo_epi8( + _mm256_permute4x64_epi64::(edge0), + _mm256_setzero_si256(), + ); - store_0 = _mm256_subs_epu16(store_0, lo0); - } + store_0 = _mm256_adds_epu16(store_0, lo0); + } - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge0 = _mm256_castsi128_si256(_mm_loadu_si128(s_ptr as *const _)); - let lo0 = _mm256_unpacklo_epi8( - _mm256_permute4x64_epi64::(edge0), - _mm256_setzero_si256(), + _mm256_storeu_si256( + buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, + store_0, ); - store_0 = _mm256_adds_epu16(store_0, lo0); + cx += 16; + buf_cx += 16; } - _mm256_storeu_si256(buffer.get_unchecked(buf_cx..).as_ptr() as *mut _, store_0); + let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); - cx += 16; - buf_cx += 16; - } + while cx + 8 <= end_x as usize { + let px = cx; - let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); + let mut store_0 = + _mm_loadu_si128(buffer.get_unchecked(buf_cx..).as_ptr() as *const _); - while cx + 8 < end_x as usize { - let px = cx; + { + let px = cx; - let mut store_0 = - _mm_loadu_si128(buffer.get_unchecked(buf_cx..).as_ptr() as *const _); + let jw = store_0; - { - let px = cx; + let store_0 = _mm_unpacklo_epi16(jw, _mm_setzero_si128()); + let store_1 = _mm_unpackhi_epi16(jw, _mm_setzero_si128()); - let jw = store_0; + let store0_ps = _mm_cvtepi32_ps(store_0); + let store1_ps = _mm_cvtepi32_ps(store_1); - let store_0 = _mm_unpacklo_epi16(jw, _mm_setzero_si128()); - let store_1 = _mm_unpackhi_epi16(jw, _mm_setzero_si128()); + let r0 = _mm_mul_ps(store0_ps, v_weight); + let r1 = _mm_mul_ps(store1_ps, v_weight); - let store0_ps = _mm_cvtepi32_ps(store_0); - let store1_ps = _mm_cvtepi32_ps(store_1); + let scale_store_0 = _mm_cvtps_epi32(r0); + let scale_store_1 = _mm_cvtps_epi32(r1); - let r0 = _mm_mul_ps(store0_ps, v_weight); - let r1 = _mm_mul_ps(store1_ps, v_weight); + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let scale_store_0 = _mm_cvtps_epi32(r0); - let scale_store_1 = _mm_cvtps_epi32(r1); + let set0 = _mm_packus_epi32(scale_store_0, scale_store_1); - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let full_set = _mm_packus_epi16(set0, _mm_setzero_si128()); + _mm_storeu_si64(ptr as *mut _, full_set); + } - let set0 = _mm_packus_epi32(scale_store_0, scale_store_1); + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = _mm_loadu_si64(s_ptr as *const _); + let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); - let full_set = _mm_packus_epi16(set0, _mm_setzero_si128()); - _mm_storeu_si64(ptr as *mut _, full_set); - } + store_0 = _mm_subs_epu16(store_0, lo0); + } - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = _mm_loadu_si64(s_ptr as *const _); - let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = _mm_loadu_si64(s_ptr as *const _); + let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); - store_0 = _mm_subs_epu16(store_0, lo0); - } + store_0 = _mm_adds_epu16(store_0, lo0); + } - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = _mm_loadu_si64(s_ptr as *const _); - let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); + _mm_storeu_si128( + buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, + store_0, + ); - store_0 = _mm_adds_epu16(store_0, lo0); + cx += 8; + buf_cx += 8; } - _mm_storeu_si128(buffer.get_unchecked(buf_cx..).as_ptr() as *mut _, store_0); + const TAIL_CN: usize = 1; - cx += 8; - buf_cx += 8; - } + for x in cx..end_x as usize { + let px = x * TAIL_CN; - const TAIL_CN: usize = 1; + let mut store = + _mm_loadu_si16(buffer.get_unchecked(buf_cx..).as_ptr() as *const _); - for x in cx..end_x as usize { - let px = x * TAIL_CN; + { + let px = x; + let store = _mm_unpacklo_epi16(store, _mm_setzero_si128()); + let r0 = _mm_mul_ps(_mm_cvtepi32_ps(store), v_weight); + let ptr = unsafe_dst.slice.as_ptr().add(y_dst_shift + px) as *mut u8; + store_u8_s16::(ptr, _mm_cvtps_epi32(r0)); + } - let mut store = _mm_loadu_si16(buffer.get_unchecked(buf_cx..).as_ptr() as *const _); + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge_colors = load_u8_s16_fast::(s_ptr); + store = _mm_subs_epu16(store, edge_colors); + } - { - let px = x; - let store = _mm_unpacklo_epi16(store, _mm_setzero_si128()); - let r0 = _mm_mul_ps(_mm_cvtepi32_ps(store), v_weight); - let ptr = unsafe_dst.slice.as_ptr().add(y_dst_shift + px) as *mut u8; - store_u8_s16::(ptr, _mm_cvtps_epi32(r0)); - } + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge_colors = load_u8_s16_fast::(s_ptr); + store = _mm_adds_epu16(store, edge_colors); + } - // subtract previous - unsafe { - let s_ptr = src.as_ptr().add(previous + px); - let edge_colors = load_u8_s16_fast::(s_ptr); - store = _mm_subs_epu16(store, edge_colors); - } + _mm_storeu_si16( + buffer.get_unchecked_mut(buf_cx..).as_mut_ptr() as *mut _, + store, + ); - // add next - unsafe { - let s_ptr = src.as_ptr().add(next + px); - let edge_colors = load_u8_s16_fast::(s_ptr); - store = _mm_adds_epu16(store, edge_colors); + buf_cx += 1; } - - _mm_storeu_si16(buffer.get_unchecked(buf_cx..).as_ptr() as *mut _, store); - - buf_cx += 1; } } } diff --git a/src/box_filter/avx/vsum8.rs b/src/box_filter/avx/vsum8.rs index 2c8f1658..d3713e1c 100644 --- a/src/box_filter/avx/vsum8.rs +++ b/src/box_filter/avx/vsum8.rs @@ -43,7 +43,7 @@ pub(crate) fn avx_ring_vertical_row_summ( } #[target_feature(enable = "avx2")] -unsafe fn avx_ring_vertical_row_summ_impl( +fn avx_ring_vertical_row_summ_impl( src: &[&[u8]; 2], dst: &mut [u8], working_row: &mut [u32], @@ -155,8 +155,6 @@ unsafe fn avx_ring_vertical_row_summ_impl( let working_rem = working_row.chunks_exact_mut(32).into_remainder(); let dst_rem = dst.chunks_exact_mut(32).into_remainder(); - let chunks16 = previous_row.chunks_exact(16).len(); - for (((src_next, src_previous), buffer), dst) in next_rem .chunks_exact(16) .zip(previous_rem.chunks_exact(16)) @@ -224,12 +222,16 @@ unsafe fn avx_ring_vertical_row_summ_impl( } } + let next_rem = next_rem.chunks_exact(16).remainder(); + let previous_rem = previous_row.chunks_exact(16).remainder(); + let working_rem = working_row.chunks_exact_mut(16).into_remainder(); + let dst_rem = dst.chunks_exact_mut(16).into_remainder(); + for (((src_next, src_previous), buffer), dst) in next_rem .iter() .zip(previous_rem.iter()) .zip(working_rem.iter_mut()) .zip(dst_rem.iter_mut()) - .skip(chunks16 * 16) { let mut weight0 = *buffer; diff --git a/src/box_filter/box_blur.rs b/src/box_filter/box_blur.rs index 1b9cf711..ea6d4fc8 100644 --- a/src/box_filter/box_blur.rs +++ b/src/box_filter/box_blur.rs @@ -29,13 +29,13 @@ use crate::channels_configuration::FastBlurChannels; use crate::primitives::PrimitiveCast; use crate::to_storage::ToStorage; use crate::unsafe_slice::UnsafeSlice; -use crate::util::check_slice_size; +use crate::util::{ScratchBuffer, check_slice_size}; use crate::{BlurError, BlurImage, BlurImageMut, ThreadingPolicy}; #[cfg(feature = "nightly_f16")] use core::f16; use novtb::{ParallelZonedIterator, TbSliceMut}; -use num_traits::cast::FromPrimitive; use num_traits::AsPrimitive; +use num_traits::cast::FromPrimitive; use std::fmt::Debug; /// Both kernels are expected to be odd. @@ -92,10 +92,10 @@ impl BoxBlurParameters { } fn validate(&self) -> Result<(), BlurError> { - if self.x_axis_kernel % 2 == 0 { + if self.x_axis_kernel.is_multiple_of(2) { return Err(BlurError::OddKernel(self.x_axis_kernel as usize)); } - if self.y_axis_kernel % 2 == 0 { + if self.y_axis_kernel.is_multiple_of(2) { return Err(BlurError::OddKernel(self.y_axis_kernel as usize)); } Ok(()) @@ -251,8 +251,8 @@ trait BoxBlurHorizontalPass { impl BoxBlurHorizontalPass for f32 { #[allow(clippy::type_complexity)] - fn get_horizontal_pass( - ) -> fn(&[f32], u32, &UnsafeSlice, u32, u32, u32, u32, u32) { + fn get_horizontal_pass() + -> fn(&[f32], u32, &UnsafeSlice, u32, u32, u32, u32, u32) { #[cfg(all(target_arch = "aarch64", feature = "neon"))] { use crate::box_filter::neon::box_blur_horizontal_pass_neon_rgba_f32; @@ -282,16 +282,16 @@ impl BoxBlurHorizontalPass for f32 { #[cfg(feature = "nightly_f16")] impl BoxBlurHorizontalPass for f16 { #[allow(clippy::type_complexity)] - fn get_horizontal_pass( - ) -> fn(&[f16], u32, &UnsafeSlice, u32, u32, u32, u32, u32) { + fn get_horizontal_pass() + -> fn(&[f16], u32, &UnsafeSlice, u32, u32, u32, u32, u32) { box_blur_horizontal_pass_impl:: } } impl BoxBlurHorizontalPass for u16 { #[allow(clippy::type_complexity)] - fn get_horizontal_pass( - ) -> fn(&[u16], u32, &UnsafeSlice, u32, u32, u32, u32, u32) { + fn get_horizontal_pass() + -> fn(&[u16], u32, &UnsafeSlice, u32, u32, u32, u32, u32) { #[cfg(all(target_arch = "aarch64", feature = "neon"))] { use crate::box_filter::neon::box_blur_horizontal_pass_neon_rgba16; @@ -313,8 +313,8 @@ impl BoxBlurHorizontalPass for u16 { impl BoxBlurHorizontalPass for u8 { #[allow(clippy::type_complexity)] - fn get_horizontal_pass( - ) -> fn(&[u8], u32, &UnsafeSlice, u32, u32, u32, u32, u32) { + fn get_horizontal_pass() + -> fn(&[u8], u32, &UnsafeSlice, u32, u32, u32, u32, u32) { let mut _dispatcher_horizontal: fn( src: &[u8], src_stride: u32, @@ -438,7 +438,8 @@ fn box_blur_vertical_pass_impl( let buf_size = end_x - start_x; let buf_cap = buf_size as usize; - let mut buffer = vec![J::default(); buf_cap]; + let mut scratch_buffer = ScratchBuffer::::new(buf_cap); + let buffer = scratch_buffer.as_mut_slice(); let src_lane = &src[start_x as usize..end_x as usize]; @@ -790,8 +791,8 @@ impl RingBufferHandler for f16 { trait VRowSum { #[allow(clippy::type_complexity)] - fn ring_vertical_row_summ( - ) -> fn(src: &[&[T]; 2], dst: &mut [T], working_row: &mut [J], radius: u32); + fn ring_vertical_row_summ() + -> fn(src: &[&[T]; 2], dst: &mut [T], working_row: &mut [J], radius: u32); } impl VRowSum for () { @@ -963,7 +964,8 @@ where .for_each_enumerated(&pool, |cy, dst_rows| { let source_y = cy * tile_size; - let mut working_row = vec![J::default(); working_stride]; + let mut working_row_scratch = ScratchBuffer::::new(working_stride); + let working_row = working_row_scratch.as_mut_slice(); let ring_size = y_kernel_size + 1; let mut buffer = vec![T::default(); working_stride * ring_size]; @@ -1070,7 +1072,7 @@ where ring_vsum( &capture, &mut dst0[..width as usize * CN], - &mut working_row, + working_row, y_radius, ); } @@ -1080,7 +1082,8 @@ where } }); } else { - let mut working_row = vec![J::default(); working_stride]; + let mut working_row_scratch = ScratchBuffer::::new(working_stride); + let working_row = working_row_scratch.as_mut_slice(); let ring_size = y_kernel_size + 1; let mut buffer = vec![T::default(); working_stride * ring_size]; @@ -1157,7 +1160,7 @@ where ring_vsum( &capture, &mut dst0[..width as usize * CN], - &mut working_row, + working_row, y_radius, ); } @@ -1409,7 +1412,7 @@ fn create_box_gauss(sigma: f32, n: usize) -> Vec { let w_ideal = (12.0 * sigma * sigma / n_float).sqrt() + 1.0; let mut wl: u32 = w_ideal.floor() as u32; - if wl % 2 == 0 { + if wl.is_multiple_of(2) { wl -= 1; }; @@ -1428,13 +1431,13 @@ fn create_box_gauss(sigma: f32, n: usize) -> Vec { for i in 0..n { if i < m { let mut new_val = wl / 2; - if new_val % 2 == 0 { + if new_val.is_multiple_of(2) { new_val += 1; } sizes.push(new_val); } else { let mut new_val = wu / 2; - if new_val % 2 == 0 { + if new_val.is_multiple_of(2) { new_val += 1; } sizes.push(new_val); diff --git a/src/box_filter/neon/mod.rs b/src/box_filter/neon/mod.rs index 658b202a..8768ae7c 100644 --- a/src/box_filter/neon/mod.rs +++ b/src/box_filter/neon/mod.rs @@ -35,15 +35,15 @@ mod vsum16; mod vsum8; mod vsum_f32; -pub(crate) use rgba16::{box_blur_horizontal_pass_neon_rgba16, box_blur_vertical_pass_neon_rgba16}; -pub(crate) use rgba8::{box_blur_horizontal_pass_neon, box_blur_vertical_pass_neon}; -#[cfg(feature = "rdm")] -pub(crate) use rgba8_q0_31::{box_blur_horizontal_pass_neon_rdm, box_blur_vertical_pass_neon_rdm}; pub(crate) use rgba_f32::{ box_blur_horizontal_pass_neon_rgba_f32, box_blur_vertical_pass_neon_rgba_f32, }; -pub(crate) use vsum16::neon_ring_vertical_row_summ16; +pub(crate) use rgba8::{box_blur_horizontal_pass_neon, box_blur_vertical_pass_neon}; +#[cfg(feature = "rdm")] +pub(crate) use rgba8_q0_31::{box_blur_horizontal_pass_neon_rdm, box_blur_vertical_pass_neon_rdm}; +pub(crate) use rgba16::{box_blur_horizontal_pass_neon_rgba16, box_blur_vertical_pass_neon_rgba16}; +pub(crate) use vsum_f32::neon_ring_vertical_row_summ_f32; pub(crate) use vsum8::neon_ring_vertical_row_summ; #[cfg(feature = "rdm")] pub(crate) use vsum8::neon_ring_vertical_row_summ_rdm; -pub(crate) use vsum_f32::neon_ring_vertical_row_summ_f32; +pub(crate) use vsum16::neon_ring_vertical_row_summ16; diff --git a/src/box_filter/neon/rgba16.rs b/src/box_filter/neon/rgba16.rs index f1b86825..7d766cbe 100644 --- a/src/box_filter/neon/rgba16.rs +++ b/src/box_filter/neon/rgba16.rs @@ -29,50 +29,57 @@ use std::arch::aarch64::*; use crate::neon::{load_u16, store_u16_u32_x4, store_u16x4, vmulq_u32_f32}; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; #[inline(always)] -unsafe fn mul_set_v4( +fn mul_set_v4( s1: uint32x4_t, s2: uint32x4_t, s3: uint32x4_t, s4: uint32x4_t, w: float32x4_t, ) -> (uint32x4_t, uint32x4_t, uint32x4_t, uint32x4_t) { - let cv1 = vcvtq_f32_u32(s1); - let cv2 = vcvtq_f32_u32(s2); - let cv3 = vcvtq_f32_u32(s3); - let cv4 = vcvtq_f32_u32(s4); - - let m1 = vmulq_f32(cv1, w); - let m2 = vmulq_f32(cv2, w); - let m3 = vmulq_f32(cv3, w); - let m4 = vmulq_f32(cv4, w); - - let cv1 = vcvtaq_u32_f32(m1); - let cv2 = vcvtaq_u32_f32(m2); - let cv3 = vcvtaq_u32_f32(m3); - let cv4 = vcvtaq_u32_f32(m4); - (cv1, cv2, cv3, cv4) + unsafe { + let cv1 = vcvtq_f32_u32(s1); + let cv2 = vcvtq_f32_u32(s2); + let cv3 = vcvtq_f32_u32(s3); + let cv4 = vcvtq_f32_u32(s4); + + let m1 = vmulq_f32(cv1, w); + let m2 = vmulq_f32(cv2, w); + let m3 = vmulq_f32(cv3, w); + let m4 = vmulq_f32(cv4, w); + + let cv1 = vcvtaq_u32_f32(m1); + let cv2 = vcvtaq_u32_f32(m2); + let cv3 = vcvtaq_u32_f32(m3); + let cv4 = vcvtaq_u32_f32(m4); + (cv1, cv2, cv3, cv4) + } } #[inline(always)] -unsafe fn mul_set_v2(s1: uint32x4_t, s2: uint32x4_t, w: float32x4_t) -> (uint32x4_t, uint32x4_t) { - let cv1 = vcvtq_f32_u32(s1); - let cv2 = vcvtq_f32_u32(s2); +fn mul_set_v2(s1: uint32x4_t, s2: uint32x4_t, w: float32x4_t) -> (uint32x4_t, uint32x4_t) { + unsafe { + let cv1 = vcvtq_f32_u32(s1); + let cv2 = vcvtq_f32_u32(s2); - let m1 = vmulq_f32(cv1, w); - let m2 = vmulq_f32(cv2, w); + let m1 = vmulq_f32(cv1, w); + let m2 = vmulq_f32(cv2, w); - let cv1 = vcvtaq_u32_f32(m1); - let cv2 = vcvtaq_u32_f32(m2); - (cv1, cv2) + let cv1 = vcvtaq_u32_f32(m1); + let cv2 = vcvtaq_u32_f32(m2); + (cv1, cv2) + } } #[inline(always)] -unsafe fn mul_set(s1: uint32x4_t, w: float32x4_t) -> uint32x4_t { - let cv1 = vcvtq_f32_u32(s1); - let m1 = vmulq_f32(cv1, w); - vcvtaq_u32_f32(m1) +fn mul_set(s1: uint32x4_t, w: float32x4_t) -> uint32x4_t { + unsafe { + let cv1 = vcvtq_f32_u32(s1); + let m1 = vmulq_f32(cv1, w); + vcvtaq_u32_f32(m1) + } } pub(crate) fn box_blur_horizontal_pass_neon_rgba16( @@ -110,7 +117,7 @@ fn box_blur_horizontal_pass_neon_impl16( let mut yy = start_y; - while yy + 4 < end_y { + while yy + 4 <= end_y { let y = yy; let y_src_shift = y as usize * src_stride as usize; let y_dst_shift = y as usize * dst_stride as usize; @@ -268,7 +275,7 @@ fn box_blur_horizontal_pass_neon_impl16( for x in 0..width { let px = x as usize * CN; - let scale_store = unsafe { vmulq_u32_f32(store, v_weight) }; + let scale_store = vmulq_u32_f32(store, v_weight); let bytes_offset = y_dst_shift + px; unsafe { @@ -317,7 +324,8 @@ pub(crate) fn box_blur_vertical_pass_neon_rgba16( } } -unsafe fn box_blur_vertical_pass_neon_any_rgba16( +#[target_feature(enable = "neon")] +fn box_blur_vertical_pass_neon_any_rgba16( src: &[u16], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -328,199 +336,52 @@ unsafe fn box_blur_vertical_pass_neon_any_rgba16( start_x: u32, end_x: u32, ) { - let kernel_size = radius * 2 + 1; - let edge_count = (kernel_size / 2) + 1; - let v_edge_count = vdupq_n_u16(edge_count as u16); - - let half_kernel = kernel_size / 2; - - let start_x = start_x as usize; - let end_x = end_x as usize; + unsafe { + let kernel_size = radius * 2 + 1; + let edge_count = (kernel_size / 2) + 1; + let v_edge_count = vdupq_n_u16(edge_count as u16); - let v_weight = vdupq_n_f32(1f32 / (radius * 2 + 1) as f32); + let half_kernel = kernel_size / 2; - assert!(end_x >= start_x); + let start_x = start_x as usize; + let end_x = end_x as usize; - let buf_size = end_x - start_x; + let v_weight = vdupq_n_f32(1f32 / (radius * 2 + 1) as f32); - let buf_cap = buf_size.div_ceil(4) * 4 + 4; + assert!(end_x >= start_x); - let mut buffer = vec![0u32; buf_cap]; + let buf_size = end_x - start_x; - let mut cx = start_x; - let mut buf_cx = 0usize; + let buf_cap = buf_size.div_ceil(4) * 4 + 4; - while cx + 16 < end_x { - let px = cx; + let mut scratch_buffer = ScratchBuffer::::new(buf_cap); + let buffer = scratch_buffer.as_mut_slice(); - let mut store_0: uint32x4_t; - let mut store_1: uint32x4_t; - let mut store_2: uint32x4_t; - let mut store_3: uint32x4_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_u16(s_ptr); - let edge1 = vld1q_u16(s_ptr.add(8)); - - store_0 = vmull_u16(vget_low_u16(edge0), vget_low_u16(v_edge_count)); - store_1 = vmull_high_u16(edge0, v_edge_count); + let mut cx = start_x; + let mut buf_cx = 0usize; - store_2 = vmull_u16(vget_low_u16(edge1), vget_low_u16(v_edge_count)); - store_3 = vmull_high_u16(edge1, v_edge_count); + while cx + 16 <= end_x { + let px = cx; - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + let mut store_0: uint32x4_t; + let mut store_1: uint32x4_t; + let mut store_2: uint32x4_t; + let mut store_3: uint32x4_t; + let s_ptr = src.as_ptr().add(px); let edge0 = vld1q_u16(s_ptr); let edge1 = vld1q_u16(s_ptr.add(8)); - store_0 = vaddw_u16(store_0, vget_low_u16(edge0)); - store_1 = vaddw_high_u16(store_1, edge0); - store_2 = vaddw_u16(store_2, vget_low_u16(edge1)); - store_3 = vaddw_high_u16(store_3, edge1); - } - - vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - vst1q_u32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); - vst1q_u32(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_2); - vst1q_u32( - buffer.get_unchecked_mut(buf_cx + 12..).as_mut_ptr(), - store_3, - ); - - cx += 16; - buf_cx += 16; - } - - while cx + 8 < end_x { - let px = cx; - - let mut store_0: uint32x4_t; - let mut store_1: uint32x4_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_u16(s_ptr); - - store_0 = vmull_u16(vget_low_u16(edge0), vget_low_u16(v_edge_count)); - store_1 = vmull_high_u16(edge0, v_edge_count); - - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - - let edge0 = vld1q_u16(s_ptr); - - store_0 = vaddw_u16(store_0, vget_low_u16(edge0)); - store_1 = vaddw_high_u16(store_1, edge0); - } + store_0 = vmull_u16(vget_low_u16(edge0), vget_low_u16(v_edge_count)); + store_1 = vmull_high_u16(edge0, v_edge_count); - vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - vst1q_u32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); + store_2 = vmull_u16(vget_low_u16(edge1), vget_low_u16(v_edge_count)); + store_3 = vmull_high_u16(edge1, v_edge_count); - cx += 8; - buf_cx += 8; - } - - while cx + 4 < end_x { - let px = cx; - - let mut store_0: uint32x4_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1_u16(s_ptr); - - store_0 = vmull_u16(edge0, vget_low_u16(v_edge_count)); - - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - - let edge0 = vld1_u16(s_ptr); - - store_0 = vaddw_u16(store_0, edge0); - } - - vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - - cx += 4; - buf_cx += 4; - } - - while cx < end_x { - let px = cx; - - let mut store_0: uint32x4_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1_lane_u16::<0>(s_ptr, vdup_n_u16(0)); - - store_0 = vmull_u16(edge0, vget_low_u16(v_edge_count)); - - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - - let edge0 = vld1_lane_u16::<0>(s_ptr, vdup_n_u16(0)); - store_0 = vaddw_u16(store_0, edge0); - } - - vst1q_lane_u32::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - - cx += 1; - buf_cx += 1; - } - - for y in 0..height { - let mut buf_cx = 0usize; - let mut cx = start_x; - - // preload edge pixels - let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; - let previous = (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y as usize; - - while cx + 16 < end_x { - let px = cx; - - let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); - let mut store_1 = vld1q_u32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); - let mut store_2 = vld1q_u32(buffer.get_unchecked(buf_cx + 8..).as_ptr()); - let mut store_3 = vld1q_u32(buffer.get_unchecked(buf_cx + 12..).as_ptr()); - - { - let px = cx; - let (scale_store_0, scale_store_1, scale_store_2, scale_store_3) = - mul_set_v4(store_0, store_1, store_2, store_3, v_weight); - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - let px_16_hi0 = vqmovn_u32(scale_store_1); - let px_16_lo1 = vqmovn_u32(scale_store_2); - let px_16_hi2 = vqmovn_u32(scale_store_3); - let px0 = vcombine_u16(px_16_lo0, px_16_hi0); - let px1 = vcombine_u16(px_16_lo1, px_16_hi2); - - vst1q_u16(ptr, px0); - vst1q_u16(ptr.add(8), px1); - } - - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge0 = vld1q_u16(s_ptr); - let edge1 = vld1q_u16(s_ptr.add(8)); - - store_0 = vsubw_u16(store_0, vget_low_u16(edge0)); - store_1 = vsubw_high_u16(store_1, edge0); - store_2 = vsubw_u16(store_2, vget_low_u16(edge1)); - store_3 = vsubw_high_u16(store_3, edge1); - } + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); let edge0 = vld1q_u16(s_ptr); let edge1 = vld1q_u16(s_ptr.add(8)); @@ -542,40 +403,26 @@ unsafe fn box_blur_vertical_pass_neon_any_rgba16( buf_cx += 16; } - while cx + 8 < end_x { + while cx + 8 <= end_x { let px = cx; - let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); - let mut store_1 = vld1q_u32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); + let mut store_0: uint32x4_t; + let mut store_1: uint32x4_t; - { - let px = cx; - let (scale_store_0, scale_store_1) = mul_set_v2(store_0, store_1, v_weight); - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - let px_16_hi0 = vqmovn_u32(scale_store_1); - let px0 = vcombine_u16(px_16_lo0, px_16_hi0); - vst1q_u16(ptr, px0); - } + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1q_u16(s_ptr); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = vld1q_u16(s_ptr); + store_0 = vmull_u16(vget_low_u16(edge0), vget_low_u16(v_edge_count)); + store_1 = vmull_high_u16(edge0, v_edge_count); - store_0 = vsubw_u16(store_0, vget_low_u16(edge)); - store_1 = vsubw_high_u16(store_1, edge); - } + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = vld1q_u16(s_ptr); + let edge0 = vld1q_u16(s_ptr); - store_0 = vaddw_u16(store_0, vget_low_u16(edge)); - store_1 = vaddw_high_u16(store_1, edge); + store_0 = vaddw_u16(store_0, vget_low_u16(edge0)); + store_1 = vaddw_high_u16(store_1, edge0); } vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); @@ -585,36 +432,23 @@ unsafe fn box_blur_vertical_pass_neon_any_rgba16( buf_cx += 8; } - while cx + 4 < end_x { + while cx + 4 <= end_x { let px = cx; - let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_0: uint32x4_t; - { - let px = cx; + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1_u16(s_ptr); - let scale_store_0 = mul_set(store_0, v_weight); + store_0 = vmull_u16(edge0, vget_low_u16(v_edge_count)); - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - vst1_u16(ptr, px_16_lo0); - } - - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = vld1_u16(s_ptr); - - store_0 = vsubw_u16(store_0, edge); - } + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = vld1_u16(s_ptr); + let edge0 = vld1_u16(s_ptr); - store_0 = vaddw_u16(store_0, edge); + store_0 = vaddw_u16(store_0, edge0); } vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); @@ -626,31 +460,19 @@ unsafe fn box_blur_vertical_pass_neon_any_rgba16( while cx < end_x { let px = cx; - let mut store_0 = - vld1q_lane_u32::<0>(buffer.get_unchecked(buf_cx..).as_ptr(), vdupq_n_u32(0)); + let mut store_0: uint32x4_t; - { - let px = cx; - let scale_store_0 = mul_set(store_0, v_weight); - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - vst1_lane_u16::<0>(ptr, px_16_lo0); - } + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1_lane_u16::<0>(s_ptr, vdup_n_u16(0)); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = vld1_lane_u16::<0>(s_ptr, vdup_n_u16(0)); + store_0 = vmull_u16(edge0, vget_low_u16(v_edge_count)); - store_0 = vsubw_u16(store_0, edge); - } + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = vld1_lane_u16::<0>(s_ptr, vdup_n_u16(0)); - store_0 = vaddw_u16(store_0, edge); + let edge0 = vld1_lane_u16::<0>(s_ptr, vdup_n_u16(0)); + store_0 = vaddw_u16(store_0, edge0); } vst1q_lane_u32::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); @@ -658,5 +480,195 @@ unsafe fn box_blur_vertical_pass_neon_any_rgba16( cx += 1; buf_cx += 1; } + + for y in 0..height { + let mut buf_cx = 0usize; + let mut cx = start_x; + + // preload edge pixels + let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; + let previous = + (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y as usize; + + while cx + 16 <= end_x { + let px = cx; + + let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_1 = vld1q_u32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); + let mut store_2 = vld1q_u32(buffer.get_unchecked(buf_cx + 8..).as_ptr()); + let mut store_3 = vld1q_u32(buffer.get_unchecked(buf_cx + 12..).as_ptr()); + + { + let px = cx; + let (scale_store_0, scale_store_1, scale_store_2, scale_store_3) = + mul_set_v4(store_0, store_1, store_2, store_3, v_weight); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + let px_16_hi0 = vqmovn_u32(scale_store_1); + let px_16_lo1 = vqmovn_u32(scale_store_2); + let px_16_hi2 = vqmovn_u32(scale_store_3); + let px0 = vcombine_u16(px_16_lo0, px_16_hi0); + let px1 = vcombine_u16(px_16_lo1, px_16_hi2); + + vst1q_u16(ptr, px0); + vst1q_u16(ptr.add(8), px1); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge0 = vld1q_u16(s_ptr); + let edge1 = vld1q_u16(s_ptr.add(8)); + + store_0 = vsubw_u16(store_0, vget_low_u16(edge0)); + store_1 = vsubw_high_u16(store_1, edge0); + store_2 = vsubw_u16(store_2, vget_low_u16(edge1)); + store_3 = vsubw_high_u16(store_3, edge1); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge0 = vld1q_u16(s_ptr); + let edge1 = vld1q_u16(s_ptr.add(8)); + + store_0 = vaddw_u16(store_0, vget_low_u16(edge0)); + store_1 = vaddw_high_u16(store_1, edge0); + store_2 = vaddw_u16(store_2, vget_low_u16(edge1)); + store_3 = vaddw_high_u16(store_3, edge1); + } + + vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + vst1q_u32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); + vst1q_u32(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_2); + vst1q_u32( + buffer.get_unchecked_mut(buf_cx + 12..).as_mut_ptr(), + store_3, + ); + + cx += 16; + buf_cx += 16; + } + + while cx + 8 <= end_x { + let px = cx; + + let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_1 = vld1q_u32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); + + { + let px = cx; + let (scale_store_0, scale_store_1) = mul_set_v2(store_0, store_1, v_weight); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + let px_16_hi0 = vqmovn_u32(scale_store_1); + let px0 = vcombine_u16(px_16_lo0, px_16_hi0); + vst1q_u16(ptr, px0); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = vld1q_u16(s_ptr); + + store_0 = vsubw_u16(store_0, vget_low_u16(edge)); + store_1 = vsubw_high_u16(store_1, edge); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = vld1q_u16(s_ptr); + + store_0 = vaddw_u16(store_0, vget_low_u16(edge)); + store_1 = vaddw_high_u16(store_1, edge); + } + + vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + vst1q_u32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); + + cx += 8; + buf_cx += 8; + } + + while cx + 4 <= end_x { + let px = cx; + + let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); + + { + let px = cx; + + let scale_store_0 = mul_set(store_0, v_weight); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + vst1_u16(ptr, px_16_lo0); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = vld1_u16(s_ptr); + + store_0 = vsubw_u16(store_0, edge); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = vld1_u16(s_ptr); + + store_0 = vaddw_u16(store_0, edge); + } + + vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + + cx += 4; + buf_cx += 4; + } + + while cx < end_x { + let px = cx; + + let mut store_0 = + vld1q_lane_u32::<0>(buffer.get_unchecked(buf_cx..).as_ptr(), vdupq_n_u32(0)); + + { + let px = cx; + let scale_store_0 = mul_set(store_0, v_weight); + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + vst1_lane_u16::<0>(ptr, px_16_lo0); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = vld1_lane_u16::<0>(s_ptr, vdup_n_u16(0)); + + store_0 = vsubw_u16(store_0, edge); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = vld1_lane_u16::<0>(s_ptr, vdup_n_u16(0)); + store_0 = vaddw_u16(store_0, edge); + } + + vst1q_lane_u32::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + + cx += 1; + buf_cx += 1; + } + } } } diff --git a/src/box_filter/neon/rgba8.rs b/src/box_filter/neon/rgba8.rs index ffdd275c..39cd758e 100644 --- a/src/box_filter/neon/rgba8.rs +++ b/src/box_filter/neon/rgba8.rs @@ -31,80 +31,89 @@ use crate::neon::{ load_u8, load_u8_u16, store_u8_s32_x4, store_u8_u32, vmulq_u16_low_f32, vmulq_u32_f32, }; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; #[inline(always)] -unsafe fn mul_set_v4( +fn mul_set_v4( s1: uint32x4_t, s2: uint32x4_t, s3: uint32x4_t, s4: uint32x4_t, w: float32x4_t, ) -> (uint32x4_t, uint32x4_t, uint32x4_t, uint32x4_t) { - let cv1 = vcvtq_f32_u32(s1); - let cv2 = vcvtq_f32_u32(s2); - let cv3 = vcvtq_f32_u32(s3); - let cv4 = vcvtq_f32_u32(s4); - - let m1 = vmulq_f32(cv1, w); - let m2 = vmulq_f32(cv2, w); - let m3 = vmulq_f32(cv3, w); - let m4 = vmulq_f32(cv4, w); - - let cv1 = vcvtaq_u32_f32(m1); - let cv2 = vcvtaq_u32_f32(m2); - let cv3 = vcvtaq_u32_f32(m3); - let cv4 = vcvtaq_u32_f32(m4); - (cv1, cv2, cv3, cv4) + unsafe { + let cv1 = vcvtq_f32_u32(s1); + let cv2 = vcvtq_f32_u32(s2); + let cv3 = vcvtq_f32_u32(s3); + let cv4 = vcvtq_f32_u32(s4); + + let m1 = vmulq_f32(cv1, w); + let m2 = vmulq_f32(cv2, w); + let m3 = vmulq_f32(cv3, w); + let m4 = vmulq_f32(cv4, w); + + let cv1 = vcvtaq_u32_f32(m1); + let cv2 = vcvtaq_u32_f32(m2); + let cv3 = vcvtaq_u32_f32(m3); + let cv4 = vcvtaq_u32_f32(m4); + (cv1, cv2, cv3, cv4) + } } #[inline(always)] -unsafe fn mul_set_v2(s1: uint32x4_t, s2: uint32x4_t, w: float32x4_t) -> (uint32x4_t, uint32x4_t) { - let cv1 = vcvtq_f32_u32(s1); - let cv2 = vcvtq_f32_u32(s2); +fn mul_set_v2(s1: uint32x4_t, s2: uint32x4_t, w: float32x4_t) -> (uint32x4_t, uint32x4_t) { + unsafe { + let cv1 = vcvtq_f32_u32(s1); + let cv2 = vcvtq_f32_u32(s2); - let m1 = vmulq_f32(cv1, w); - let m2 = vmulq_f32(cv2, w); + let m1 = vmulq_f32(cv1, w); + let m2 = vmulq_f32(cv2, w); - let cv1 = vcvtaq_u32_f32(m1); - let cv2 = vcvtaq_u32_f32(m2); - (cv1, cv2) + let cv1 = vcvtaq_u32_f32(m1); + let cv2 = vcvtaq_u32_f32(m2); + (cv1, cv2) + } } #[inline(always)] -unsafe fn mul_set(s1: uint32x4_t, w: float32x4_t) -> uint32x4_t { - let cv1 = vcvtq_f32_u32(s1); - let m1 = vmulq_f32(cv1, w); - vcvtaq_u32_f32(m1) +fn mul_set(s1: uint32x4_t, w: float32x4_t) -> uint32x4_t { + unsafe { + let cv1 = vcvtq_f32_u32(s1); + let m1 = vmulq_f32(cv1, w); + vcvtaq_u32_f32(m1) + } } #[inline(always)] -unsafe fn mul_set_low_u16_v4( +fn mul_set_low_u16_v4( s1: uint16x8_t, s2: uint16x8_t, s3: uint16x8_t, s4: uint16x8_t, w: float32x4_t, ) -> (uint32x4_t, uint32x4_t, uint32x4_t, uint32x4_t) { - let s1 = vmovl_u16(vget_low_u16(s1)); - let s2 = vmovl_u16(vget_low_u16(s2)); - let s3 = vmovl_u16(vget_low_u16(s3)); - let s4 = vmovl_u16(vget_low_u16(s4)); - - let cv1 = vcvtq_f32_u32(s1); - let cv2 = vcvtq_f32_u32(s2); - let cv3 = vcvtq_f32_u32(s3); - let cv4 = vcvtq_f32_u32(s4); - - let m1 = vmulq_f32(cv1, w); - let m2 = vmulq_f32(cv2, w); - let m3 = vmulq_f32(cv3, w); - let m4 = vmulq_f32(cv4, w); - - let cv1 = vcvtaq_u32_f32(m1); - let cv2 = vcvtaq_u32_f32(m2); - let cv3 = vcvtaq_u32_f32(m3); - let cv4 = vcvtaq_u32_f32(m4); - (cv1, cv2, cv3, cv4) + unsafe { + let s1 = vmovl_u16(vget_low_u16(s1)); + let s2 = vmovl_u16(vget_low_u16(s2)); + let s3 = vmovl_u16(vget_low_u16(s3)); + let s4 = vmovl_u16(vget_low_u16(s4)); + + let cv1 = vcvtq_f32_u32(s1); + let cv2 = vcvtq_f32_u32(s2); + let cv3 = vcvtq_f32_u32(s3); + let cv4 = vcvtq_f32_u32(s4); + + let m1 = vmulq_f32(cv1, w); + let m2 = vmulq_f32(cv2, w); + let m3 = vmulq_f32(cv3, w); + let m4 = vmulq_f32(cv4, w); + + let cv1 = vcvtaq_u32_f32(m1); + let cv2 = vcvtaq_u32_f32(m2); + let cv3 = vcvtaq_u32_f32(m3); + let cv4 = vcvtaq_u32_f32(m4); + (cv1, cv2, cv3, cv4) + } } pub(crate) fn box_blur_horizontal_pass_neon( @@ -117,17 +126,20 @@ pub(crate) fn box_blur_horizontal_pass_neon( start_y: u32, end_y: u32, ) { - if radius < 240 { - box_blur_horizontal_pass_neon_impl_low_rad::( - src, src_stride, dst, dst_stride, width, radius, start_y, end_y, - ); - } else { - box_blur_horizontal_pass_neon_impl::( - src, src_stride, dst, dst_stride, width, radius, start_y, end_y, - ); + unsafe { + if radius < 240 { + box_blur_horizontal_pass_neon_impl_low_rad::( + src, src_stride, dst, dst_stride, width, radius, start_y, end_y, + ); + } else { + box_blur_horizontal_pass_neon_impl::( + src, src_stride, dst, dst_stride, width, radius, start_y, end_y, + ); + } } } +#[target_feature(enable = "neon")] fn box_blur_horizontal_pass_neon_impl_low_rad( src: &[u8], src_stride: u32, @@ -140,9 +152,9 @@ fn box_blur_horizontal_pass_neon_impl_low_rad( ) { let kernel_size = radius * 2 + 1; let edge_count = (kernel_size / 2) + 1; - let v_edge_count = unsafe { vdup_n_u8(edge_count as u8) }; + let v_edge_count = vdup_n_u8(edge_count as u8); - let v_weight = unsafe { vdupq_n_f32(1f32 / (radius * 2 + 1) as f32) }; + let v_weight = vdupq_n_f32(1f32 / (radius * 2 + 1) as f32); let half_kernel = kernel_size / 2; @@ -377,7 +389,7 @@ fn box_blur_horizontal_pass_neon_impl_low_rad( for x in 0..width { let px = x as usize * CN; - let scale_store = unsafe { vmulq_u16_low_f32(store, v_weight) }; + let scale_store = vmulq_u16_low_f32(store, v_weight); let bytes_offset = y_dst_shift + px; @@ -409,6 +421,7 @@ fn box_blur_horizontal_pass_neon_impl_low_rad( } } +#[target_feature(enable = "neon")] fn box_blur_horizontal_pass_neon_impl( src: &[u8], src_stride: u32, @@ -421,9 +434,9 @@ fn box_blur_horizontal_pass_neon_impl( ) { let kernel_size = radius * 2 + 1; let edge_count = (kernel_size / 2) + 1; - let v_edge_count = unsafe { vdupq_n_u16(edge_count as u16) }; + let v_edge_count = vdupq_n_u16(edge_count as u16); - let v_weight = unsafe { vdupq_n_f32(1f32 / (radius * 2 + 1) as f32) }; + let v_weight = vdupq_n_f32(1f32 / (radius * 2 + 1) as f32); let half_kernel = kernel_size / 2; @@ -592,7 +605,7 @@ fn box_blur_horizontal_pass_neon_impl( for x in 0..width { let px = x as usize * CN; - let scale_store = unsafe { vmulq_u32_f32(store, v_weight) }; + let scale_store = vmulq_u32_f32(store, v_weight); let bytes_offset = y_dst_shift + px; unsafe { @@ -647,7 +660,8 @@ pub(crate) fn box_blur_vertical_pass_neon( } } -unsafe fn box_blur_vertical_pass_neon_low_rad( +#[target_feature(enable = "neon")] +fn box_blur_vertical_pass_neon_low_rad( src: &[u8], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -658,235 +672,60 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( start_x: u32, end_x: u32, ) { - let kernel_size = radius * 2 + 1; - let edge_count = (kernel_size / 2) + 1; - let v_edge_count = vdupq_n_u8(edge_count as u8); - - let half_kernel = kernel_size / 2; - - let start_x = start_x as usize; - let end_x = end_x as usize; - - let v_weight = vdupq_n_f32(1f32 / (radius * 2 + 1) as f32); - - assert!(end_x >= start_x); - - let buf_size = end_x - start_x; - - let buf_cap = buf_size.div_ceil(8) * 8 + 8; - - let mut buffer = vec![0u16; buf_cap]; - - let mut cx = start_x; - - // Configure initial accumulator - - let mut buf_cx = 0usize; - - while cx + 32 < end_x { - let px = cx; - - let mut store_0: uint16x8_t; - let mut store_1: uint16x8_t; - let mut store_2: uint16x8_t; - let mut store_3: uint16x8_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_u8(s_ptr); - let edge1 = vld1q_u8(s_ptr.add(16)); - - store_0 = vmull_u8(vget_low_u8(edge0), vget_low_u8(v_edge_count)); - store_1 = vmull_high_u8(edge0, v_edge_count); - - store_2 = vmull_u8(vget_low_u8(edge1), vget_low_u8(v_edge_count)); - store_3 = vmull_high_u8(edge1, v_edge_count); - - for y in 1..=half_kernel as usize { - let y_src_shift = y.min(height as usize - 1) * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - - let edge0 = vld1q_u8(s_ptr); - let edge1 = vld1q_u8(s_ptr.add(16)); - - store_0 = vaddw_u8(store_0, vget_low_u8(edge0)); - store_1 = vaddw_high_u8(store_1, edge0); - store_2 = vaddw_u8(store_2, vget_low_u8(edge1)); - store_3 = vaddw_high_u8(store_3, edge1); - } - - vst1q_u16(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - vst1q_u16(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_1); - vst1q_u16( - buffer.get_unchecked_mut(buf_cx + 16..).as_mut_ptr(), - store_2, - ); - vst1q_u16( - buffer.get_unchecked_mut(buf_cx + 24..).as_mut_ptr(), - store_3, - ); - - cx += 32; - buf_cx += 32; - } - - while cx + 16 < end_x { - let px = cx; - - let mut store_0: uint16x8_t; - let mut store_1: uint16x8_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_u8(s_ptr); - - store_0 = vmull_u8(vget_low_u8(edge0), vget_low_u8(v_edge_count)); - store_1 = vmull_high_u8(edge0, v_edge_count); - - for y in 1..=half_kernel as usize { - let y_src_shift = y.min(height as usize - 1) * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - - let edge0 = vld1q_u8(s_ptr); - - store_0 = vaddw_u8(store_0, vget_low_u8(edge0)); - store_1 = vaddw_high_u8(store_1, edge0); - } - - vst1q_u16(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - vst1q_u16(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_1); - - cx += 16; - buf_cx += 16; - } - - while cx + 8 < end_x { - let px = cx; - - let mut store_0: uint16x8_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1_u8(s_ptr); - - store_0 = vmull_u8(edge0, vget_low_u8(v_edge_count)); - - for y in 1..=half_kernel as usize { - let y_src_shift = y.min(height as usize - 1) * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - - let edge0 = vld1_u8(s_ptr); - - store_0 = vaddw_u8(store_0, edge0); - } - - vst1q_u16(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - - cx += 8; - buf_cx += 8; - } - - while cx < end_x { - let px = cx; + unsafe { + let kernel_size = radius * 2 + 1; + let edge_count = (kernel_size / 2) + 1; + let v_edge_count = vdupq_n_u8(edge_count as u8); - let mut store_0: uint16x8_t; + let half_kernel = kernel_size / 2; - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_lane_u8::<0>(s_ptr, vdupq_n_u8(0)); + let start_x = start_x as usize; + let end_x = end_x as usize; - store_0 = vmull_u8(vget_low_u8(edge0), vget_low_u8(v_edge_count)); + let v_weight = vdupq_n_f32(1f32 / (radius * 2 + 1) as f32); - for y in 1..=half_kernel as usize { - let y_src_shift = y.min(height as usize - 1) * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + assert!(end_x >= start_x); - let edge0 = vld1q_lane_u8::<0>(s_ptr, vdupq_n_u8(0)); + let buf_size = end_x - start_x; - store_0 = vaddw_u8(store_0, vget_low_u8(edge0)); - } + let buf_cap = buf_size.div_ceil(8) * 8 + 8; - vst1q_lane_u16::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + let mut scratch_buffer = ScratchBuffer::::new(buf_cap); + let buffer = scratch_buffer.as_mut_slice(); - cx += 1; - buf_cx += 1; - } - - for y in 0..height as usize { let mut cx = start_x; + // Configure initial accumulator + let mut buf_cx = 0usize; - while cx + 32 < end_x { + while cx + 32 <= end_x { let px = cx; - let mut store_0 = vld1q_u16(buffer.get_unchecked(buf_cx..).as_ptr()); - let mut store_1 = vld1q_u16(buffer.get_unchecked(buf_cx + 8..).as_ptr()); - let mut store_2 = vld1q_u16(buffer.get_unchecked(buf_cx + 16..).as_ptr()); - let mut store_3 = vld1q_u16(buffer.get_unchecked(buf_cx + 24..).as_ptr()); + let mut store_0: uint16x8_t; + let mut store_1: uint16x8_t; + let mut store_2: uint16x8_t; + let mut store_3: uint16x8_t; - let next = - (y + half_kernel as usize + 1).min(height as usize - 1) * src_stride as usize; - let previous = - (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y; - - { - let px = cx; - - let lw0 = vmovl_u16(vget_low_u16(store_0)); - let lw1 = vmovl_high_u16(store_0); - let lw2 = vmovl_u16(vget_low_u16(store_1)); - let lw3 = vmovl_high_u16(store_1); - - let lw4 = vmovl_u16(vget_low_u16(store_2)); - let lw5 = vmovl_high_u16(store_2); - let lw6 = vmovl_u16(vget_low_u16(store_3)); - let lw7 = vmovl_high_u16(store_3); - - let (scale_store_0, scale_store_1, scale_store_2, scale_store_3) = - mul_set_v4(lw0, lw1, lw2, lw3, v_weight); - - let (scale_store_4, scale_store_5, scale_store_6, scale_store_7) = - mul_set_v4(lw4, lw5, lw6, lw7, v_weight); - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - let px_16_hi0 = vqmovn_u32(scale_store_1); - let px_16_lo1 = vqmovn_u32(scale_store_2); - let px_16_hi2 = vqmovn_u32(scale_store_3); - let px_16_lo3 = vqmovn_u32(scale_store_4); - let px_16_hi4 = vqmovn_u32(scale_store_5); - let px_16_lo5 = vqmovn_u32(scale_store_6); - let px_16_hi6 = vqmovn_u32(scale_store_7); - - let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); - let px1 = vqmovn_u16(vcombine_u16(px_16_lo1, px_16_hi2)); - let px2 = vqmovn_u16(vcombine_u16(px_16_lo3, px_16_hi4)); - let px3 = vqmovn_u16(vcombine_u16(px_16_lo5, px_16_hi6)); - vst1q_u8(ptr, vcombine_u8(px0, px1)); - vst1q_u8(ptr.add(16), vcombine_u8(px2, px3)); - } + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1q_u8(s_ptr); + let edge1 = vld1q_u8(s_ptr.add(16)); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge0 = vld1q_u8(s_ptr); - let edge1 = vld1q_u8(s_ptr.add(16)); + store_0 = vmull_u8(vget_low_u8(edge0), vget_low_u8(v_edge_count)); + store_1 = vmull_high_u8(edge0, v_edge_count); - store_0 = vsubw_u8(store_0, vget_low_u8(edge0)); - store_1 = vsubw_high_u8(store_1, edge0); + store_2 = vmull_u8(vget_low_u8(edge1), vget_low_u8(v_edge_count)); + store_3 = vmull_high_u8(edge1, v_edge_count); - store_2 = vsubw_u8(store_2, vget_low_u8(edge1)); - store_3 = vsubw_high_u8(store_3, edge1); - } + for y in 1..=half_kernel as usize { + let y_src_shift = y.min(height as usize - 1) * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); let edge0 = vld1q_u8(s_ptr); let edge1 = vld1q_u8(s_ptr.add(16)); store_0 = vaddw_u8(store_0, vget_low_u8(edge0)); store_1 = vaddw_high_u8(store_1, edge0); - store_2 = vaddw_u8(store_2, vget_low_u8(edge1)); store_3 = vaddw_high_u8(store_3, edge1); } @@ -906,57 +745,26 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( buf_cx += 32; } - while cx + 16 < end_x { + while cx + 16 <= end_x { let px = cx; - let mut store_0 = vld1q_u16(buffer.get_unchecked(buf_cx..).as_ptr()); - let mut store_1 = vld1q_u16(buffer.get_unchecked(buf_cx + 8..).as_ptr()); - - let next = - (y + half_kernel as usize + 1).min(height as usize - 1) * src_stride as usize; - let previous = - (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y; - - { - let px = cx; - - let lw0 = vmovl_u16(vget_low_u16(store_0)); - let lw1 = vmovl_high_u16(store_0); - let lw2 = vmovl_u16(vget_low_u16(store_1)); - let lw3 = vmovl_high_u16(store_1); - - let (scale_store_0, scale_store_1, scale_store_2, scale_store_3) = - mul_set_v4(lw0, lw1, lw2, lw3, v_weight); + let mut store_0: uint16x8_t; + let mut store_1: uint16x8_t; - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - let px_16_hi0 = vqmovn_u32(scale_store_1); - let px_16_lo1 = vqmovn_u32(scale_store_2); - let px_16_hi2 = vqmovn_u32(scale_store_3); + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1q_u8(s_ptr); - let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); - let px1 = vqmovn_u16(vcombine_u16(px_16_lo1, px_16_hi2)); - vst1q_u8(ptr, vcombine_u8(px0, px1)); - } + store_0 = vmull_u8(vget_low_u8(edge0), vget_low_u8(v_edge_count)); + store_1 = vmull_high_u8(edge0, v_edge_count); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = vld1q_u8(s_ptr); - - store_0 = vsubw_u8(store_0, vget_low_u8(edge)); - store_1 = vsubw_high_u8(store_1, edge); - } + for y in 1..=half_kernel as usize { + let y_src_shift = y.min(height as usize - 1) * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = vld1q_u8(s_ptr); + let edge0 = vld1q_u8(s_ptr); - store_0 = vaddw_u8(store_0, vget_low_u8(edge)); - store_1 = vaddw_high_u8(store_1, edge); + store_0 = vaddw_u8(store_0, vget_low_u8(edge0)); + store_1 = vaddw_high_u8(store_1, edge0); } vst1q_u16(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); @@ -966,49 +774,23 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( buf_cx += 16; } - while cx + 8 < end_x { + while cx + 8 <= end_x { let px = cx; - let mut store_0 = vld1q_u16(buffer.get_unchecked(buf_cx..).as_ptr()); - - // preload edge pixels - let next = - (y + half_kernel as usize + 1).min(height as usize - 1) * src_stride as usize; - let previous = - (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y; - - { - let px = cx; + let mut store_0: uint16x8_t; - let lw0 = vmovl_u16(vget_low_u16(store_0)); - let lw1 = vmovl_high_u16(store_0); - - let scale_store_0 = vmulq_u32_f32(lw0, v_weight); - let scale_store_1 = vmulq_u32_f32(lw1, v_weight); - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - let px_16_hi0 = vqmovn_u32(scale_store_1); - let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); - vst1_u8(ptr, px0); - } + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1_u8(s_ptr); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = vld1_u8(s_ptr); + store_0 = vmull_u8(edge0, vget_low_u8(v_edge_count)); - store_0 = vsubw_u8(store_0, edge); - } + for y in 1..=half_kernel as usize { + let y_src_shift = y.min(height as usize - 1) * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = vld1_u8(s_ptr); + let edge0 = vld1_u8(s_ptr); - store_0 = vaddw_u8(store_0, edge); + store_0 = vaddw_u8(store_0, edge0); } vst1q_u16(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); @@ -1017,396 +799,368 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( buf_cx += 8; } - const TAIL_CN: usize = 1; + while cx < end_x { + let px = cx; - for (x, buf_cx) in (cx..end_x).zip(buf_cx..buf_cap) { - let px = x; + let mut store_0: uint16x8_t; - let mut store = - vld1q_lane_u16::<0>(buffer.get_unchecked(buf_cx..).as_ptr(), vdupq_n_u16(0)); + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1q_lane_u8::<0>(s_ptr, vdupq_n_u8(0)); - // preload edge pixels - let next = - (y + half_kernel as usize + 1).min(height as usize - 1) * src_stride as usize; - let previous = (y as i64 - half_kernel as i64).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y; + store_0 = vmull_u8(vget_low_u8(edge0), vget_low_u8(v_edge_count)); - { - let px = x; - - let scale_store = vmulq_u16_low_f32(store, v_weight); + for y in 1..=half_kernel as usize { + let y_src_shift = y.min(height as usize - 1) * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - let bytes_offset = y_dst_shift + px; - let dst_ptr = unsafe_dst.slice.as_ptr().add(bytes_offset) as *mut u8; - store_u8_u32::(dst_ptr, scale_store); - } + let edge0 = vld1q_lane_u8::<0>(s_ptr, vdupq_n_u8(0)); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge_colors = load_u8::(s_ptr); - store = vsubw_u8(store, edge_colors); + store_0 = vaddw_u8(store_0, vget_low_u8(edge0)); } - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge_colors = load_u8::(s_ptr); - store = vaddw_u8(store, edge_colors); - } + vst1q_lane_u16::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - vst1q_lane_u16::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store); + cx += 1; + buf_cx += 1; } - } -} -unsafe fn box_blur_vertical_pass_neon_any( - src: &[u8], - src_stride: u32, - unsafe_dst: &UnsafeSlice, - dst_stride: u32, - _: u32, - height: u32, - radius: u32, - start_x: u32, - end_x: u32, -) { - let kernel_size = radius * 2 + 1; - let edge_count = (kernel_size / 2) + 1; - let v_edge_count = vdupq_n_u16(edge_count as u16); - - let half_kernel = kernel_size / 2; - - let start_x = start_x as usize; - let end_x = end_x as usize; + for y in 0..height as usize { + let mut cx = start_x; - let v_weight = vdupq_n_f32(1f32 / (radius * 2 + 1) as f32); + let mut buf_cx = 0usize; - assert!(end_x >= start_x); + while cx + 32 <= end_x { + let px = cx; - let buf_size = end_x - start_x; + let mut store_0 = vld1q_u16(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_1 = vld1q_u16(buffer.get_unchecked(buf_cx + 8..).as_ptr()); + let mut store_2 = vld1q_u16(buffer.get_unchecked(buf_cx + 16..).as_ptr()); + let mut store_3 = vld1q_u16(buffer.get_unchecked(buf_cx + 24..).as_ptr()); + + let next = + (y + half_kernel as usize + 1).min(height as usize - 1) * src_stride as usize; + let previous = + (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y; + + { + let px = cx; + + let lw0 = vmovl_u16(vget_low_u16(store_0)); + let lw1 = vmovl_high_u16(store_0); + let lw2 = vmovl_u16(vget_low_u16(store_1)); + let lw3 = vmovl_high_u16(store_1); + + let lw4 = vmovl_u16(vget_low_u16(store_2)); + let lw5 = vmovl_high_u16(store_2); + let lw6 = vmovl_u16(vget_low_u16(store_3)); + let lw7 = vmovl_high_u16(store_3); + + let (scale_store_0, scale_store_1, scale_store_2, scale_store_3) = + mul_set_v4(lw0, lw1, lw2, lw3, v_weight); + + let (scale_store_4, scale_store_5, scale_store_6, scale_store_7) = + mul_set_v4(lw4, lw5, lw6, lw7, v_weight); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + let px_16_hi0 = vqmovn_u32(scale_store_1); + let px_16_lo1 = vqmovn_u32(scale_store_2); + let px_16_hi2 = vqmovn_u32(scale_store_3); + let px_16_lo3 = vqmovn_u32(scale_store_4); + let px_16_hi4 = vqmovn_u32(scale_store_5); + let px_16_lo5 = vqmovn_u32(scale_store_6); + let px_16_hi6 = vqmovn_u32(scale_store_7); + + let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); + let px1 = vqmovn_u16(vcombine_u16(px_16_lo1, px_16_hi2)); + let px2 = vqmovn_u16(vcombine_u16(px_16_lo3, px_16_hi4)); + let px3 = vqmovn_u16(vcombine_u16(px_16_lo5, px_16_hi6)); + vst1q_u8(ptr, vcombine_u8(px0, px1)); + vst1q_u8(ptr.add(16), vcombine_u8(px2, px3)); + } - let buf_cap = buf_size.div_ceil(4) * 4 + 4; + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge0 = vld1q_u8(s_ptr); + let edge1 = vld1q_u8(s_ptr.add(16)); - let mut buffer = vec![0u32; buf_cap]; + store_0 = vsubw_u8(store_0, vget_low_u8(edge0)); + store_1 = vsubw_high_u8(store_1, edge0); - let mut cx = start_x; - let mut buf_cx = 0usize; + store_2 = vsubw_u8(store_2, vget_low_u8(edge1)); + store_3 = vsubw_high_u8(store_3, edge1); + } - while cx + 32 < end_x { - let px = cx; + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge0 = vld1q_u8(s_ptr); + let edge1 = vld1q_u8(s_ptr.add(16)); - let mut store_0: uint32x4_t; - let mut store_1: uint32x4_t; - let mut store_2: uint32x4_t; - let mut store_3: uint32x4_t; - let mut store_4: uint32x4_t; - let mut store_5: uint32x4_t; - let mut store_6: uint32x4_t; - let mut store_7: uint32x4_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_u8(s_ptr); - let edge1 = vld1q_u8(s_ptr.add(16)); + store_0 = vaddw_u8(store_0, vget_low_u8(edge0)); + store_1 = vaddw_high_u8(store_1, edge0); - let lo0 = vmovl_u8(vget_low_u8(edge0)); - let hi0 = vmovl_high_u8(edge0); - let lo1 = vmovl_u8(vget_low_u8(edge1)); - let hi1 = vmovl_high_u8(edge1); + store_2 = vaddw_u8(store_2, vget_low_u8(edge1)); + store_3 = vaddw_high_u8(store_3, edge1); + } - let i16_l0 = vget_low_u16(lo0); - let i16_h0 = lo0; - let i16_l1 = vget_low_u16(hi0); - let i16_h1 = hi0; + vst1q_u16(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + vst1q_u16(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_1); + vst1q_u16( + buffer.get_unchecked_mut(buf_cx + 16..).as_mut_ptr(), + store_2, + ); + vst1q_u16( + buffer.get_unchecked_mut(buf_cx + 24..).as_mut_ptr(), + store_3, + ); - let i16_l01 = vget_low_u16(lo1); - let i16_h01 = lo1; - let i16_l11 = vget_low_u16(hi1); - let i16_h11 = hi1; + cx += 32; + buf_cx += 32; + } - store_0 = vmull_u16(i16_l0, vget_low_u16(v_edge_count)); - store_1 = vmull_high_u16(i16_h0, v_edge_count); + while cx + 16 <= end_x { + let px = cx; - store_2 = vmull_u16(i16_l1, vget_low_u16(v_edge_count)); - store_3 = vmull_high_u16(i16_h1, v_edge_count); + let mut store_0 = vld1q_u16(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_1 = vld1q_u16(buffer.get_unchecked(buf_cx + 8..).as_ptr()); - store_4 = vmull_u16(i16_l01, vget_low_u16(v_edge_count)); - store_5 = vmull_high_u16(i16_h01, v_edge_count); + let next = + (y + half_kernel as usize + 1).min(height as usize - 1) * src_stride as usize; + let previous = + (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y; - store_6 = vmull_u16(i16_l11, vget_low_u16(v_edge_count)); - store_7 = vmull_high_u16(i16_h11, v_edge_count); + { + let px = cx; - for y in 1..=half_kernel.min(height) { - let y_src_shift = y as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + let lw0 = vmovl_u16(vget_low_u16(store_0)); + let lw1 = vmovl_high_u16(store_0); + let lw2 = vmovl_u16(vget_low_u16(store_1)); + let lw3 = vmovl_high_u16(store_1); - let edge0 = vld1q_u8(s_ptr); - let edge1 = vld1q_u8(s_ptr.add(16)); + let (scale_store_0, scale_store_1, scale_store_2, scale_store_3) = + mul_set_v4(lw0, lw1, lw2, lw3, v_weight); - let lo0 = vmovl_u8(vget_low_u8(edge0)); - let hi0 = vmovl_high_u8(edge0); - let lo1 = vmovl_u8(vget_low_u8(edge1)); - let hi1 = vmovl_high_u8(edge1); + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + let px_16_hi0 = vqmovn_u32(scale_store_1); + let px_16_lo1 = vqmovn_u32(scale_store_2); + let px_16_hi2 = vqmovn_u32(scale_store_3); - let i16_l0 = vget_low_u16(lo0); - let i16_h0 = lo0; - let i16_l1 = vget_low_u16(hi0); - let i16_h1 = hi0; + let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); + let px1 = vqmovn_u16(vcombine_u16(px_16_lo1, px_16_hi2)); + vst1q_u8(ptr, vcombine_u8(px0, px1)); + } - let i16_l01 = vget_low_u16(lo1); - let i16_h01 = lo1; - let i16_l11 = vget_low_u16(hi1); - let i16_h11 = hi1; + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = vld1q_u8(s_ptr); - store_0 = vaddw_u16(store_0, i16_l0); - store_1 = vaddw_high_u16(store_1, i16_h0); - store_2 = vaddw_u16(store_2, i16_l1); - store_3 = vaddw_high_u16(store_3, i16_h1); + store_0 = vsubw_u8(store_0, vget_low_u8(edge)); + store_1 = vsubw_high_u8(store_1, edge); + } - store_4 = vaddw_u16(store_4, i16_l01); - store_5 = vaddw_high_u16(store_5, i16_h01); - store_6 = vaddw_u16(store_6, i16_l11); - store_7 = vaddw_high_u16(store_7, i16_h11); - } + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = vld1q_u8(s_ptr); - vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - vst1q_u32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); - vst1q_u32(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_2); - vst1q_u32( - buffer.get_unchecked_mut(buf_cx + 12..).as_mut_ptr(), - store_3, - ); - vst1q_u32( - buffer.get_unchecked_mut(buf_cx + 16..).as_mut_ptr(), - store_4, - ); - vst1q_u32( - buffer.get_unchecked_mut(buf_cx + 20..).as_mut_ptr(), - store_5, - ); - vst1q_u32( - buffer.get_unchecked_mut(buf_cx + 24..).as_mut_ptr(), - store_6, - ); - vst1q_u32( - buffer.get_unchecked_mut(buf_cx + 28..).as_mut_ptr(), - store_7, - ); - - cx += 32; - buf_cx += 32; - } + store_0 = vaddw_u8(store_0, vget_low_u8(edge)); + store_1 = vaddw_high_u8(store_1, edge); + } - while cx + 16 < end_x { - let px = cx; + vst1q_u16(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + vst1q_u16(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_1); - let mut store_0: uint32x4_t; - let mut store_1: uint32x4_t; - let mut store_2: uint32x4_t; - let mut store_3: uint32x4_t; + cx += 16; + buf_cx += 16; + } - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_u8(s_ptr); + while cx + 8 <= end_x { + let px = cx; - let lo0 = vmovl_u8(vget_low_u8(edge0)); - let hi0 = vmovl_high_u8(edge0); + let mut store_0 = vld1q_u16(buffer.get_unchecked(buf_cx..).as_ptr()); - let i16_l0 = vget_low_u16(lo0); - let i16_h0 = lo0; - let i16_l1 = vget_low_u16(hi0); - let i16_h1 = hi0; + // preload edge pixels + let next = + (y + half_kernel as usize + 1).min(height as usize - 1) * src_stride as usize; + let previous = + (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y; - store_0 = vmull_u16(i16_l0, vget_low_u16(v_edge_count)); - store_1 = vmull_high_u16(i16_h0, v_edge_count); + { + let px = cx; - store_2 = vmull_u16(i16_l1, vget_low_u16(v_edge_count)); - store_3 = vmull_high_u16(i16_h1, v_edge_count); + let lw0 = vmovl_u16(vget_low_u16(store_0)); + let lw1 = vmovl_high_u16(store_0); - for y in 1..=half_kernel.min(height) { - let y_src_shift = y as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + let scale_store_0 = vmulq_u32_f32(lw0, v_weight); + let scale_store_1 = vmulq_u32_f32(lw1, v_weight); - let edge0 = vld1q_u8(s_ptr); + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + let px_16_hi0 = vqmovn_u32(scale_store_1); + let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); + vst1_u8(ptr, px0); + } - let lo0 = vmovl_u8(vget_low_u8(edge0)); - let hi0 = vmovl_high_u8(edge0); + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = vld1_u8(s_ptr); - let i16_l0 = vget_low_u16(lo0); - let i16_h0 = lo0; - let i16_l1 = vget_low_u16(hi0); - let i16_h1 = hi0; + store_0 = vsubw_u8(store_0, edge); + } - store_0 = vaddw_u16(store_0, i16_l0); - store_1 = vaddw_high_u16(store_1, i16_h0); - store_2 = vaddw_u16(store_2, i16_l1); - store_3 = vaddw_high_u16(store_3, i16_h1); - } + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = vld1_u8(s_ptr); - vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - vst1q_u32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); - vst1q_u32(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_2); - vst1q_u32( - buffer.get_unchecked_mut(buf_cx + 12..).as_mut_ptr(), - store_3, - ); + store_0 = vaddw_u8(store_0, edge); + } - cx += 16; - buf_cx += 16; - } + vst1q_u16(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - while cx + 8 < end_x { - let px = cx; + cx += 8; + buf_cx += 8; + } - let mut store_0: uint32x4_t; - let mut store_1: uint32x4_t; + const TAIL_CN: usize = 1; - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1_u8(s_ptr); + for (x, buf_cx) in (cx..end_x).zip(buf_cx..buf_cap) { + let px = x; - let lo0 = vmovl_u8(edge0); + let mut store = + vld1q_lane_u16::<0>(buffer.get_unchecked(buf_cx..).as_ptr(), vdupq_n_u16(0)); - let i16_l0 = vget_low_u16(lo0); - let i16_h0 = lo0; + // preload edge pixels + let next = + (y + half_kernel as usize + 1).min(height as usize - 1) * src_stride as usize; + let previous = + (y as i64 - half_kernel as i64).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y; - store_0 = vmull_u16(i16_l0, vget_low_u16(v_edge_count)); - store_1 = vmull_high_u16(i16_h0, v_edge_count); + { + let px = x; - for y in 1..=half_kernel.min(height) { - let y_src_shift = y as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + let scale_store = vmulq_u16_low_f32(store, v_weight); - let edge0 = vld1_u8(s_ptr); + let bytes_offset = y_dst_shift + px; + let dst_ptr = unsafe_dst.slice.as_ptr().add(bytes_offset) as *mut u8; + store_u8_u32::(dst_ptr, scale_store); + } - let lo0 = vmovl_u8(edge0); + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge_colors = load_u8::(s_ptr); + store = vsubw_u8(store, edge_colors); + } - let i16_l0 = vget_low_u16(lo0); - let i16_h0 = lo0; + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge_colors = load_u8::(s_ptr); + store = vaddw_u8(store, edge_colors); + } - store_0 = vaddw_u16(store_0, i16_l0); - store_1 = vaddw_high_u16(store_1, i16_h0); + vst1q_lane_u16::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store); + } } - - vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - vst1q_u32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); - - cx += 8; - buf_cx += 8; } +} - while cx < end_x { - let px = cx; - - let mut store_0: uint32x4_t; +#[target_feature(enable = "neon")] +fn box_blur_vertical_pass_neon_any( + src: &[u8], + src_stride: u32, + unsafe_dst: &UnsafeSlice, + dst_stride: u32, + _: u32, + height: u32, + radius: u32, + start_x: u32, + end_x: u32, +) { + let kernel_size = radius * 2 + 1; + let edge_count = (kernel_size / 2) + 1; + let v_edge_count = vdupq_n_u16(edge_count as u16); - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1_lane_u8::<0>(s_ptr, vdup_n_u8(0)); + let half_kernel = kernel_size / 2; - let lo0 = vmovl_u8(edge0); + let start_x = start_x as usize; + let end_x = end_x as usize; - let i16_l0 = vget_low_u16(lo0); + unsafe { + let v_weight = vdupq_n_f32(1f32 / (radius * 2 + 1) as f32); - store_0 = vmull_u16(i16_l0, vget_low_u16(v_edge_count)); + assert!(end_x >= start_x); - for y in 1..=half_kernel.min(height) { - let y_src_shift = y as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + let buf_size = end_x - start_x; - let edge0 = vld1_lane_u8::<0>(s_ptr, vdup_n_u8(0)); - let lo0 = vmovl_u8(edge0); - let i16_l0 = vget_low_u16(lo0); - store_0 = vaddw_u16(store_0, i16_l0); - } + let buf_cap = buf_size.div_ceil(4) * 4 + 4; - vst1q_lane_u32::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + let mut scratch_buffer = ScratchBuffer::::new(buf_cap); + let buffer = scratch_buffer.as_mut_slice(); - cx += 1; - buf_cx += 1; - } - - for y in 0..height { - let mut buf_cx = 0usize; let mut cx = start_x; + let mut buf_cx = 0usize; - // preload edge pixels - let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; - let previous = (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y as usize; - - while cx + 32 < end_x { + while cx + 32 <= end_x { let px = cx; - let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); - let mut store_1 = vld1q_u32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); - let mut store_2 = vld1q_u32(buffer.get_unchecked(buf_cx + 8..).as_ptr()); - let mut store_3 = vld1q_u32(buffer.get_unchecked(buf_cx + 12..).as_ptr()); - let mut store_4 = vld1q_u32(buffer.get_unchecked(buf_cx + 16..).as_ptr()); - let mut store_5 = vld1q_u32(buffer.get_unchecked(buf_cx + 20..).as_ptr()); - let mut store_6 = vld1q_u32(buffer.get_unchecked(buf_cx + 24..).as_ptr()); - let mut store_7 = vld1q_u32(buffer.get_unchecked(buf_cx + 28..).as_ptr()); + let mut store_0: uint32x4_t; + let mut store_1: uint32x4_t; + let mut store_2: uint32x4_t; + let mut store_3: uint32x4_t; + let mut store_4: uint32x4_t; + let mut store_5: uint32x4_t; + let mut store_6: uint32x4_t; + let mut store_7: uint32x4_t; - { - let px = cx; + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1q_u8(s_ptr); + let edge1 = vld1q_u8(s_ptr.add(16)); - let (scale_store_0, scale_store_1, scale_store_2, scale_store_3) = - mul_set_v4(store_0, store_1, store_2, store_3, v_weight); - - let (scale_store_4, scale_store_5, scale_store_6, scale_store_7) = - mul_set_v4(store_4, store_5, store_6, store_7, v_weight); - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - let px_16_hi0 = vqmovn_u32(scale_store_1); - let px_16_lo1 = vqmovn_u32(scale_store_2); - let px_16_hi2 = vqmovn_u32(scale_store_3); - let px_16_lo3 = vqmovn_u32(scale_store_4); - let px_16_hi4 = vqmovn_u32(scale_store_5); - let px_16_lo5 = vqmovn_u32(scale_store_6); - let px_16_hi6 = vqmovn_u32(scale_store_7); - let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); - let px1 = vqmovn_u16(vcombine_u16(px_16_lo1, px_16_hi2)); - let px2 = vqmovn_u16(vcombine_u16(px_16_lo3, px_16_hi4)); - let px3 = vqmovn_u16(vcombine_u16(px_16_lo5, px_16_hi6)); - - vst1q_u8(ptr, vcombine_u8(px0, px1)); - vst1q_u8(ptr.add(16), vcombine_u8(px2, px3)); - } + let lo0 = vmovl_u8(vget_low_u8(edge0)); + let hi0 = vmovl_high_u8(edge0); + let lo1 = vmovl_u8(vget_low_u8(edge1)); + let hi1 = vmovl_high_u8(edge1); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge0 = vld1q_u8(s_ptr); - let edge1 = vld1q_u8(s_ptr.add(16)); + let i16_l0 = vget_low_u16(lo0); + let i16_h0 = lo0; + let i16_l1 = vget_low_u16(hi0); + let i16_h1 = hi0; - let lo0 = vmovl_u8(vget_low_u8(edge0)); - let hi0 = vmovl_high_u8(edge0); - let lo1 = vmovl_u8(vget_low_u8(edge1)); - let hi1 = vmovl_high_u8(edge1); + let i16_l01 = vget_low_u16(lo1); + let i16_h01 = lo1; + let i16_l11 = vget_low_u16(hi1); + let i16_h11 = hi1; - let i16_l0 = vget_low_u16(lo0); - let i16_h0 = lo0; - let i16_l1 = vget_low_u16(hi0); - let i16_h1 = hi0; + store_0 = vmull_u16(i16_l0, vget_low_u16(v_edge_count)); + store_1 = vmull_high_u16(i16_h0, v_edge_count); - let i16_l01 = vget_low_u16(lo1); - let i16_h01 = lo1; - let i16_l11 = vget_low_u16(hi1); - let i16_h11 = hi1; + store_2 = vmull_u16(i16_l1, vget_low_u16(v_edge_count)); + store_3 = vmull_high_u16(i16_h1, v_edge_count); - store_0 = vsubw_u16(store_0, i16_l0); - store_1 = vsubw_high_u16(store_1, i16_h0); - store_2 = vsubw_u16(store_2, i16_l1); - store_3 = vsubw_high_u16(store_3, i16_h1); + store_4 = vmull_u16(i16_l01, vget_low_u16(v_edge_count)); + store_5 = vmull_high_u16(i16_h01, v_edge_count); - store_4 = vsubw_u16(store_4, i16_l01); - store_5 = vsubw_high_u16(store_5, i16_h01); - store_6 = vsubw_u16(store_6, i16_l11); - store_7 = vsubw_high_u16(store_7, i16_h11); - } + store_6 = vmull_u16(i16_l11, vget_low_u16(v_edge_count)); + store_7 = vmull_high_u16(i16_h11, v_edge_count); + + for y in 1..=half_kernel.min(height) { + let y_src_shift = y as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); let edge0 = vld1q_u8(s_ptr); let edge1 = vld1q_u8(s_ptr.add(16)); @@ -1464,55 +1218,39 @@ unsafe fn box_blur_vertical_pass_neon_any( buf_cx += 32; } - while cx + 16 < end_x { + while cx + 16 <= end_x { let px = cx; - let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); - let mut store_1 = vld1q_u32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); - let mut store_2 = vld1q_u32(buffer.get_unchecked(buf_cx + 8..).as_ptr()); - let mut store_3 = vld1q_u32(buffer.get_unchecked(buf_cx + 12..).as_ptr()); + let mut store_0: uint32x4_t; + let mut store_1: uint32x4_t; + let mut store_2: uint32x4_t; + let mut store_3: uint32x4_t; - { - let px = cx; + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1q_u8(s_ptr); - let (scale_store_0, scale_store_1, scale_store_2, scale_store_3) = - mul_set_v4(store_0, store_1, store_2, store_3, v_weight); - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - let px_16_hi0 = vqmovn_u32(scale_store_1); - let px_16_lo1 = vqmovn_u32(scale_store_2); - let px_16_hi2 = vqmovn_u32(scale_store_3); - let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); - let px1 = vqmovn_u16(vcombine_u16(px_16_lo1, px_16_hi2)); - vst1q_u8(ptr, vcombine_u8(px0, px1)); - } + let lo0 = vmovl_u8(vget_low_u8(edge0)); + let hi0 = vmovl_high_u8(edge0); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = vld1q_u8(s_ptr); - let lo0 = vmovl_u8(vget_low_u8(edge)); - let hi0 = vmovl_high_u8(edge); + let i16_l0 = vget_low_u16(lo0); + let i16_h0 = lo0; + let i16_l1 = vget_low_u16(hi0); + let i16_h1 = hi0; - let i16_l0 = vget_low_u16(lo0); - let i16_h0 = lo0; - let i16_l1 = vget_low_u16(hi0); - let i16_h1 = hi0; + store_0 = vmull_u16(i16_l0, vget_low_u16(v_edge_count)); + store_1 = vmull_high_u16(i16_h0, v_edge_count); - store_0 = vsubw_u16(store_0, i16_l0); - store_1 = vsubw_high_u16(store_1, i16_h0); - store_2 = vsubw_u16(store_2, i16_l1); - store_3 = vsubw_high_u16(store_3, i16_h1); - } + store_2 = vmull_u16(i16_l1, vget_low_u16(v_edge_count)); + store_3 = vmull_high_u16(i16_h1, v_edge_count); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = vld1q_u8(s_ptr); - let lo0 = vmovl_u8(vget_low_u8(edge)); - let hi0 = vmovl_high_u8(edge); + for y in 1..=half_kernel.min(height) { + let y_src_shift = y as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); + + let edge0 = vld1q_u8(s_ptr); + + let lo0 = vmovl_u8(vget_low_u8(edge0)); + let hi0 = vmovl_high_u8(edge0); let i16_l0 = vget_low_u16(lo0); let i16_h0 = lo0; @@ -1537,43 +1275,30 @@ unsafe fn box_blur_vertical_pass_neon_any( buf_cx += 16; } - while cx + 8 < end_x { + while cx + 8 <= end_x { let px = cx; - let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); - let mut store_1 = vld1q_u32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); + let mut store_0: uint32x4_t; + let mut store_1: uint32x4_t; - { - let px = cx; + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1_u8(s_ptr); - let (scale_store_0, scale_store_1) = mul_set_v2(store_0, store_1, v_weight); + let lo0 = vmovl_u8(edge0); - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - let px_16_hi0 = vqmovn_u32(scale_store_1); - let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); - vst1_u8(ptr, px0); - } + let i16_l0 = vget_low_u16(lo0); + let i16_h0 = lo0; - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = vld1_u8(s_ptr); - let lo0 = vmovl_u8(edge); + store_0 = vmull_u16(i16_l0, vget_low_u16(v_edge_count)); + store_1 = vmull_high_u16(i16_h0, v_edge_count); - let i16_l0 = vget_low_u16(lo0); - let i16_h0 = lo0; + for y in 1..=half_kernel.min(height) { + let y_src_shift = y as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - store_0 = vsubw_u16(store_0, i16_l0); - store_1 = vsubw_high_u16(store_1, i16_h0); - } + let edge0 = vld1_u8(s_ptr); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = vld1_u8(s_ptr); - let lo0 = vmovl_u8(edge); + let lo0 = vmovl_u8(edge0); let i16_l0 = vget_low_u16(lo0); let i16_h0 = lo0; @@ -1592,47 +1317,345 @@ unsafe fn box_blur_vertical_pass_neon_any( while cx < end_x { let px = cx; - let mut store_0 = - vld1q_lane_u32::<0>(buffer.get_unchecked(buf_cx..).as_ptr(), vdupq_n_u32(0)); + let mut store_0: uint32x4_t; + + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1_lane_u8::<0>(s_ptr, vdup_n_u8(0)); + + let lo0 = vmovl_u8(edge0); + + let i16_l0 = vget_low_u16(lo0); + + store_0 = vmull_u16(i16_l0, vget_low_u16(v_edge_count)); + + for y in 1..=half_kernel.min(height) { + let y_src_shift = y as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); + + let edge0 = vld1_lane_u8::<0>(s_ptr, vdup_n_u8(0)); + let lo0 = vmovl_u8(edge0); + let i16_l0 = vget_low_u16(lo0); + store_0 = vaddw_u16(store_0, i16_l0); + } + + vst1q_lane_u32::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + + cx += 1; + buf_cx += 1; + } + + for y in 0..height { + let mut buf_cx = 0usize; + let mut cx = start_x; + + // preload edge pixels + let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; + let previous = + (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y as usize; - { + while cx + 32 <= end_x { let px = cx; - let scale_store_0 = mul_set(store_0, v_weight); + let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_1 = vld1q_u32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); + let mut store_2 = vld1q_u32(buffer.get_unchecked(buf_cx + 8..).as_ptr()); + let mut store_3 = vld1q_u32(buffer.get_unchecked(buf_cx + 12..).as_ptr()); + let mut store_4 = vld1q_u32(buffer.get_unchecked(buf_cx + 16..).as_ptr()); + let mut store_5 = vld1q_u32(buffer.get_unchecked(buf_cx + 20..).as_ptr()); + let mut store_6 = vld1q_u32(buffer.get_unchecked(buf_cx + 24..).as_ptr()); + let mut store_7 = vld1q_u32(buffer.get_unchecked(buf_cx + 28..).as_ptr()); + + { + let px = cx; + + let (scale_store_0, scale_store_1, scale_store_2, scale_store_3) = + mul_set_v4(store_0, store_1, store_2, store_3, v_weight); + + let (scale_store_4, scale_store_5, scale_store_6, scale_store_7) = + mul_set_v4(store_4, store_5, store_6, store_7, v_weight); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + let px_16_hi0 = vqmovn_u32(scale_store_1); + let px_16_lo1 = vqmovn_u32(scale_store_2); + let px_16_hi2 = vqmovn_u32(scale_store_3); + let px_16_lo3 = vqmovn_u32(scale_store_4); + let px_16_hi4 = vqmovn_u32(scale_store_5); + let px_16_lo5 = vqmovn_u32(scale_store_6); + let px_16_hi6 = vqmovn_u32(scale_store_7); + let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); + let px1 = vqmovn_u16(vcombine_u16(px_16_lo1, px_16_hi2)); + let px2 = vqmovn_u16(vcombine_u16(px_16_lo3, px_16_hi4)); + let px3 = vqmovn_u16(vcombine_u16(px_16_lo5, px_16_hi6)); + + vst1q_u8(ptr, vcombine_u8(px0, px1)); + vst1q_u8(ptr.add(16), vcombine_u8(px2, px3)); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge0 = vld1q_u8(s_ptr); + let edge1 = vld1q_u8(s_ptr.add(16)); + + let lo0 = vmovl_u8(vget_low_u8(edge0)); + let hi0 = vmovl_high_u8(edge0); + let lo1 = vmovl_u8(vget_low_u8(edge1)); + let hi1 = vmovl_high_u8(edge1); + + let i16_l0 = vget_low_u16(lo0); + let i16_h0 = lo0; + let i16_l1 = vget_low_u16(hi0); + let i16_h1 = hi0; + + let i16_l01 = vget_low_u16(lo1); + let i16_h01 = lo1; + let i16_l11 = vget_low_u16(hi1); + let i16_h11 = hi1; + + store_0 = vsubw_u16(store_0, i16_l0); + store_1 = vsubw_high_u16(store_1, i16_h0); + store_2 = vsubw_u16(store_2, i16_l1); + store_3 = vsubw_high_u16(store_3, i16_h1); + + store_4 = vsubw_u16(store_4, i16_l01); + store_5 = vsubw_high_u16(store_5, i16_h01); + store_6 = vsubw_u16(store_6, i16_l11); + store_7 = vsubw_high_u16(store_7, i16_h11); + } - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vqmovn_u32(scale_store_0); - let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, vdup_n_u16(0))); - vst1_lane_u8::<0>(ptr, px0); + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge0 = vld1q_u8(s_ptr); + let edge1 = vld1q_u8(s_ptr.add(16)); + + let lo0 = vmovl_u8(vget_low_u8(edge0)); + let hi0 = vmovl_high_u8(edge0); + let lo1 = vmovl_u8(vget_low_u8(edge1)); + let hi1 = vmovl_high_u8(edge1); + + let i16_l0 = vget_low_u16(lo0); + let i16_h0 = lo0; + let i16_l1 = vget_low_u16(hi0); + let i16_h1 = hi0; + + let i16_l01 = vget_low_u16(lo1); + let i16_h01 = lo1; + let i16_l11 = vget_low_u16(hi1); + let i16_h11 = hi1; + + store_0 = vaddw_u16(store_0, i16_l0); + store_1 = vaddw_high_u16(store_1, i16_h0); + store_2 = vaddw_u16(store_2, i16_l1); + store_3 = vaddw_high_u16(store_3, i16_h1); + + store_4 = vaddw_u16(store_4, i16_l01); + store_5 = vaddw_high_u16(store_5, i16_h01); + store_6 = vaddw_u16(store_6, i16_l11); + store_7 = vaddw_high_u16(store_7, i16_h11); + } + + vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + vst1q_u32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); + vst1q_u32(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_2); + vst1q_u32( + buffer.get_unchecked_mut(buf_cx + 12..).as_mut_ptr(), + store_3, + ); + vst1q_u32( + buffer.get_unchecked_mut(buf_cx + 16..).as_mut_ptr(), + store_4, + ); + vst1q_u32( + buffer.get_unchecked_mut(buf_cx + 20..).as_mut_ptr(), + store_5, + ); + vst1q_u32( + buffer.get_unchecked_mut(buf_cx + 24..).as_mut_ptr(), + store_6, + ); + vst1q_u32( + buffer.get_unchecked_mut(buf_cx + 28..).as_mut_ptr(), + store_7, + ); + + cx += 32; + buf_cx += 32; } - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = vld1_lane_u8::<0>(s_ptr, vdup_n_u8(0)); - let lo0 = vmovl_u8(edge); + while cx + 16 <= end_x { + let px = cx; - let i16_l0 = vget_low_u16(lo0); + let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_1 = vld1q_u32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); + let mut store_2 = vld1q_u32(buffer.get_unchecked(buf_cx + 8..).as_ptr()); + let mut store_3 = vld1q_u32(buffer.get_unchecked(buf_cx + 12..).as_ptr()); + + { + let px = cx; + + let (scale_store_0, scale_store_1, scale_store_2, scale_store_3) = + mul_set_v4(store_0, store_1, store_2, store_3, v_weight); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + let px_16_hi0 = vqmovn_u32(scale_store_1); + let px_16_lo1 = vqmovn_u32(scale_store_2); + let px_16_hi2 = vqmovn_u32(scale_store_3); + let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); + let px1 = vqmovn_u16(vcombine_u16(px_16_lo1, px_16_hi2)); + vst1q_u8(ptr, vcombine_u8(px0, px1)); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = vld1q_u8(s_ptr); + let lo0 = vmovl_u8(vget_low_u8(edge)); + let hi0 = vmovl_high_u8(edge); + + let i16_l0 = vget_low_u16(lo0); + let i16_h0 = lo0; + let i16_l1 = vget_low_u16(hi0); + let i16_h1 = hi0; + + store_0 = vsubw_u16(store_0, i16_l0); + store_1 = vsubw_high_u16(store_1, i16_h0); + store_2 = vsubw_u16(store_2, i16_l1); + store_3 = vsubw_high_u16(store_3, i16_h1); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = vld1q_u8(s_ptr); + let lo0 = vmovl_u8(vget_low_u8(edge)); + let hi0 = vmovl_high_u8(edge); + + let i16_l0 = vget_low_u16(lo0); + let i16_h0 = lo0; + let i16_l1 = vget_low_u16(hi0); + let i16_h1 = hi0; + + store_0 = vaddw_u16(store_0, i16_l0); + store_1 = vaddw_high_u16(store_1, i16_h0); + store_2 = vaddw_u16(store_2, i16_l1); + store_3 = vaddw_high_u16(store_3, i16_h1); + } - store_0 = vsubw_u16(store_0, i16_l0); + vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + vst1q_u32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); + vst1q_u32(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_2); + vst1q_u32( + buffer.get_unchecked_mut(buf_cx + 12..).as_mut_ptr(), + store_3, + ); + + cx += 16; + buf_cx += 16; } - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = vld1_lane_u8::<0>(s_ptr, vdup_n_u8(0)); - let lo0 = vmovl_u8(edge); + while cx + 8 <= end_x { + let px = cx; - let i16_l0 = vget_low_u16(lo0); + let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_1 = vld1q_u32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); - store_0 = vaddw_u16(store_0, i16_l0); + { + let px = cx; + + let (scale_store_0, scale_store_1) = mul_set_v2(store_0, store_1, v_weight); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + let px_16_hi0 = vqmovn_u32(scale_store_1); + let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, px_16_hi0)); + vst1_u8(ptr, px0); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = vld1_u8(s_ptr); + let lo0 = vmovl_u8(edge); + + let i16_l0 = vget_low_u16(lo0); + let i16_h0 = lo0; + + store_0 = vsubw_u16(store_0, i16_l0); + store_1 = vsubw_high_u16(store_1, i16_h0); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = vld1_u8(s_ptr); + let lo0 = vmovl_u8(edge); + + let i16_l0 = vget_low_u16(lo0); + let i16_h0 = lo0; + + store_0 = vaddw_u16(store_0, i16_l0); + store_1 = vaddw_high_u16(store_1, i16_h0); + } + + vst1q_u32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + vst1q_u32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); + + cx += 8; + buf_cx += 8; } - vst1q_lane_u32::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + while cx < end_x { + let px = cx; - cx += 1; - buf_cx += 1; + let mut store_0 = + vld1q_lane_u32::<0>(buffer.get_unchecked(buf_cx..).as_ptr(), vdupq_n_u32(0)); + + { + let px = cx; + + let scale_store_0 = mul_set(store_0, v_weight); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vqmovn_u32(scale_store_0); + let px0 = vqmovn_u16(vcombine_u16(px_16_lo0, vdup_n_u16(0))); + vst1_lane_u8::<0>(ptr, px0); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = vld1_lane_u8::<0>(s_ptr, vdup_n_u8(0)); + let lo0 = vmovl_u8(edge); + + let i16_l0 = vget_low_u16(lo0); + + store_0 = vsubw_u16(store_0, i16_l0); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = vld1_lane_u8::<0>(s_ptr, vdup_n_u8(0)); + let lo0 = vmovl_u8(edge); + + let i16_l0 = vget_low_u16(lo0); + + store_0 = vaddw_u16(store_0, i16_l0); + } + + vst1q_lane_u32::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + + cx += 1; + buf_cx += 1; + } } } } diff --git a/src/box_filter/neon/rgba8_q0_31.rs b/src/box_filter/neon/rgba8_q0_31.rs index c0be1215..8e342a1e 100644 --- a/src/box_filter/neon/rgba8_q0_31.rs +++ b/src/box_filter/neon/rgba8_q0_31.rs @@ -29,55 +29,62 @@ use std::arch::aarch64::*; use crate::neon::{load_u8, load_u8_u16, store_u8_s32, vmulq_u16_low_s32}; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; #[inline(always)] -unsafe fn mul_set_v4( +fn mul_set_v4( s1: uint32x4_t, s2: uint32x4_t, s3: uint32x4_t, s4: uint32x4_t, w: int32x4_t, ) -> (int32x4_t, int32x4_t, int32x4_t, int32x4_t) { - let m1 = vqrdmulhq_s32(vreinterpretq_s32_u32(s1), w); - let m2 = vqrdmulhq_s32(vreinterpretq_s32_u32(s2), w); - let m3 = vqrdmulhq_s32(vreinterpretq_s32_u32(s3), w); - let m4 = vqrdmulhq_s32(vreinterpretq_s32_u32(s4), w); + unsafe { + let m1 = vqrdmulhq_s32(vreinterpretq_s32_u32(s1), w); + let m2 = vqrdmulhq_s32(vreinterpretq_s32_u32(s2), w); + let m3 = vqrdmulhq_s32(vreinterpretq_s32_u32(s3), w); + let m4 = vqrdmulhq_s32(vreinterpretq_s32_u32(s4), w); - (m1, m2, m3, m4) + (m1, m2, m3, m4) + } } #[inline(always)] -unsafe fn mul_set_v2(s1: uint32x4_t, s2: uint32x4_t, w: int32x4_t) -> (int32x4_t, int32x4_t) { - let m1 = vqrdmulhq_s32(vreinterpretq_s32_u32(s1), w); - let m2 = vqrdmulhq_s32(vreinterpretq_s32_u32(s2), w); +fn mul_set_v2(s1: uint32x4_t, s2: uint32x4_t, w: int32x4_t) -> (int32x4_t, int32x4_t) { + unsafe { + let m1 = vqrdmulhq_s32(vreinterpretq_s32_u32(s1), w); + let m2 = vqrdmulhq_s32(vreinterpretq_s32_u32(s2), w); - (m1, m2) + (m1, m2) + } } #[inline(always)] -unsafe fn mul_set(s1: uint32x4_t, w: int32x4_t) -> int32x4_t { - vqrdmulhq_s32(vreinterpretq_s32_u32(s1), w) +fn mul_set(s1: uint32x4_t, w: int32x4_t) -> int32x4_t { + unsafe { vqrdmulhq_s32(vreinterpretq_s32_u32(s1), w) } } #[inline(always)] -unsafe fn mul_set_low_u16_v4( +fn mul_set_low_u16_v4( s1: uint16x8_t, s2: uint16x8_t, s3: uint16x8_t, s4: uint16x8_t, w: int32x4_t, ) -> (int32x4_t, int32x4_t, int32x4_t, int32x4_t) { - let s1 = vmovl_u16(vget_low_u16(s1)); - let s2 = vmovl_u16(vget_low_u16(s2)); - let s3 = vmovl_u16(vget_low_u16(s3)); - let s4 = vmovl_u16(vget_low_u16(s4)); + unsafe { + let s1 = vmovl_u16(vget_low_u16(s1)); + let s2 = vmovl_u16(vget_low_u16(s2)); + let s3 = vmovl_u16(vget_low_u16(s3)); + let s4 = vmovl_u16(vget_low_u16(s4)); - let m1 = vqrdmulhq_s32(vreinterpretq_s32_u32(s1), w); - let m2 = vqrdmulhq_s32(vreinterpretq_s32_u32(s2), w); - let m3 = vqrdmulhq_s32(vreinterpretq_s32_u32(s3), w); - let m4 = vqrdmulhq_s32(vreinterpretq_s32_u32(s4), w); + let m1 = vqrdmulhq_s32(vreinterpretq_s32_u32(s1), w); + let m2 = vqrdmulhq_s32(vreinterpretq_s32_u32(s2), w); + let m3 = vqrdmulhq_s32(vreinterpretq_s32_u32(s3), w); + let m4 = vqrdmulhq_s32(vreinterpretq_s32_u32(s4), w); - (m1, m2, m3, m4) + (m1, m2, m3, m4) + } } pub(crate) fn box_blur_horizontal_pass_neon_rdm( @@ -104,7 +111,7 @@ pub(crate) fn box_blur_horizontal_pass_neon_rdm( } #[target_feature(enable = "rdm")] -unsafe fn box_blur_horizontal_pass_neon_impl_low_rad( +fn box_blur_horizontal_pass_neon_impl_low_rad( src: &[u8], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -127,7 +134,7 @@ unsafe fn box_blur_horizontal_pass_neon_impl_low_rad( let mut yy = start_y; - while yy + 4 < end_y { + while yy + 4 <= end_y { let y = yy; let y_src_shift = y as usize * src_stride as usize; let y_dst_shift = y as usize * dst_stride as usize; @@ -388,7 +395,7 @@ unsafe fn box_blur_horizontal_pass_neon_impl_low_rad( } #[target_feature(enable = "rdm")] -unsafe fn box_blur_horizontal_pass_neon_impl( +fn box_blur_horizontal_pass_neon_impl( src: &[u8], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -411,7 +418,7 @@ unsafe fn box_blur_horizontal_pass_neon_impl( let mut yy = start_y; - while yy + 4 < end_y { + while yy + 4 <= end_y { let y = yy; let y_src_shift = y as usize * src_stride as usize; let y_dst_shift = y as usize * dst_stride as usize; @@ -656,7 +663,7 @@ pub(crate) fn box_blur_vertical_pass_neon_rdm( } #[target_feature(enable = "rdm")] -unsafe fn box_blur_vertical_pass_neon_low_rad( +fn box_blur_vertical_pass_neon_low_rad( src: &[u8], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -687,7 +694,8 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( let buf_cap = buf_size.div_ceil(8) * 8 + 8; - let mut buffer = vec![0u16; buf_cap]; + let mut scratch_buffer = ScratchBuffer::::new(buf_cap); + let buffer = scratch_buffer.as_mut_slice(); let mut cx = start_x; @@ -695,7 +703,7 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( let mut buf_cx = 0usize; - while cx + 32 < end_x { + while cx + 32 <= end_x { let px = cx; let mut store_0: uint16x8_t; @@ -741,7 +749,7 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( buf_cx += 32; } - while cx + 16 < end_x { + while cx + 16 <= end_x { let px = cx; let mut store_0: uint16x8_t; @@ -770,7 +778,7 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( buf_cx += 16; } - while cx + 8 < end_x { + while cx + 8 <= end_x { let px = cx; let mut store_0: uint16x8_t; @@ -919,7 +927,7 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( buf_cx += 32; } - while cx + 16 < end_x { + while cx + 16 <= end_x { let px = cx; let mut store_0 = vld1q_u16(buffer.get_unchecked(buf_cx..).as_ptr()); @@ -980,7 +988,7 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( buf_cx += 16; } - while cx + 8 < end_x { + while cx + 8 <= end_x { let px = cx; let mut store_0 = vld1q_u16(buffer.get_unchecked(buf_cx..).as_ptr()); @@ -1077,7 +1085,7 @@ unsafe fn box_blur_vertical_pass_neon_low_rad( } #[target_feature(enable = "rdm")] -unsafe fn box_blur_vertical_pass_neon_any( +fn box_blur_vertical_pass_neon_any( src: &[u8], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -1108,12 +1116,13 @@ unsafe fn box_blur_vertical_pass_neon_any( let buf_cap = buf_size.div_ceil(4) * 4 + 4; - let mut buffer = vec![0u32; buf_cap]; + let mut scratch_buffer = ScratchBuffer::::new(buf_cap); + let buffer = scratch_buffer.as_mut_slice(); let mut cx = start_x; let mut buf_cx = 0usize; - while cx + 32 < end_x { + while cx + 32 <= end_x { let px = cx; let mut store_0: uint32x4_t; @@ -1217,7 +1226,7 @@ unsafe fn box_blur_vertical_pass_neon_any( buf_cx += 32; } - while cx + 16 < end_x { + while cx + 16 <= end_x { let px = cx; let mut store_0: uint32x4_t; @@ -1274,7 +1283,7 @@ unsafe fn box_blur_vertical_pass_neon_any( buf_cx += 16; } - while cx + 8 < end_x { + while cx + 8 <= end_x { let px = cx; let mut store_0: uint32x4_t; @@ -1353,7 +1362,7 @@ unsafe fn box_blur_vertical_pass_neon_any( (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; let y_dst_shift = dst_stride as usize * y as usize; - while cx + 32 < end_x { + while cx + 32 <= end_x { let px = cx; let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); @@ -1485,7 +1494,7 @@ unsafe fn box_blur_vertical_pass_neon_any( buf_cx += 32; } - while cx + 16 < end_x { + while cx + 16 <= end_x { let px = cx; let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); @@ -1558,7 +1567,7 @@ unsafe fn box_blur_vertical_pass_neon_any( buf_cx += 16; } - while cx + 8 < end_x { + while cx + 8 <= end_x { let px = cx; let mut store_0 = vld1q_u32(buffer.get_unchecked(buf_cx..).as_ptr()); diff --git a/src/box_filter/neon/rgba_f32.rs b/src/box_filter/neon/rgba_f32.rs index b2e0b8b4..c415ba91 100644 --- a/src/box_filter/neon/rgba_f32.rs +++ b/src/box_filter/neon/rgba_f32.rs @@ -29,6 +29,7 @@ use std::arch::aarch64::*; use crate::neon::{load_f32_fast, store_f32}; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; pub(crate) fn box_blur_horizontal_pass_neon_rgba_f32( src: &[f32], @@ -65,7 +66,7 @@ fn box_blur_horizontal_pass_neon_impl_f32( let mut yy = start_y; - while yy + 4 < end_y { + while yy + 4 <= end_y { let y = yy; let y_src_shift = y as usize * src_stride as usize; let y_dst_shift = y as usize * dst_stride as usize; @@ -270,7 +271,8 @@ pub(crate) fn box_blur_vertical_pass_neon_rgba_f32( } } -unsafe fn box_blur_vertical_pass_neon_any_rgba_f32( +#[target_feature(enable = "neon")] +fn box_blur_vertical_pass_neon_any_rgba_f32( src: &[f32], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -281,206 +283,53 @@ unsafe fn box_blur_vertical_pass_neon_any_rgba_f32( start_x: u32, end_x: u32, ) { - let kernel_size = radius * 2 + 1; - let edge_count = (kernel_size / 2) + 1; - let v_edge_count = vdupq_n_f32(edge_count as f32); - - let half_kernel = kernel_size / 2; - - let start_x = start_x as usize; - let end_x = end_x as usize; - - let v_weight = vdupq_n_f32(1f32 / (radius * 2 + 1) as f32); - - assert!(end_x >= start_x); + unsafe { + let kernel_size = radius * 2 + 1; + let edge_count = (kernel_size / 2) + 1; + let v_edge_count = vdupq_n_f32(edge_count as f32); - let buf_size = end_x - start_x; + let half_kernel = kernel_size / 2; - let buf_cap = buf_size.div_ceil(4) * 4 + 4; + let start_x = start_x as usize; + let end_x = end_x as usize; - let mut buffer = vec![0f32; buf_cap]; + let v_weight = vdupq_n_f32(1f32 / (radius * 2 + 1) as f32); - let mut cx = start_x; - let mut buf_cx = 0usize; + assert!(end_x >= start_x); - while cx + 16 < end_x { - let px = cx; + let buf_size = end_x - start_x; - let mut store_0: float32x4_t; - let mut store_1: float32x4_t; - let mut store_2: float32x4_t; - let mut store_3: float32x4_t; + let buf_cap = buf_size.div_ceil(4) * 4 + 4; - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_f32(s_ptr); - let edge1 = vld1q_f32(s_ptr.add(4)); - let edge2 = vld1q_f32(s_ptr.add(8)); - let edge3 = vld1q_f32(s_ptr.add(12)); + let mut scratch_buffer = ScratchBuffer::::new(buf_cap); + let buffer = scratch_buffer.as_mut_slice(); - store_0 = vmulq_f32(edge0, v_edge_count); - store_1 = vmulq_f32(edge1, v_edge_count); + let mut cx = start_x; + let mut buf_cx = 0usize; - store_2 = vmulq_f32(edge2, v_edge_count); - store_3 = vmulq_f32(edge3, v_edge_count); + while cx + 16 <= end_x { + let px = cx; - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + let mut store_0: float32x4_t; + let mut store_1: float32x4_t; + let mut store_2: float32x4_t; + let mut store_3: float32x4_t; + let s_ptr = src.as_ptr().add(px); let edge0 = vld1q_f32(s_ptr); let edge1 = vld1q_f32(s_ptr.add(4)); let edge2 = vld1q_f32(s_ptr.add(8)); let edge3 = vld1q_f32(s_ptr.add(12)); - store_0 = vaddq_f32(store_0, edge0); - store_1 = vaddq_f32(store_1, edge1); - store_2 = vaddq_f32(store_2, edge2); - store_3 = vaddq_f32(store_3, edge3); - } - - vst1q_f32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - vst1q_f32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); - vst1q_f32(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_2); - vst1q_f32( - buffer.get_unchecked_mut(buf_cx + 12..).as_mut_ptr(), - store_3, - ); - - cx += 16; - buf_cx += 16; - } - - while cx + 8 < end_x { - let px = cx; - - let mut store_0: float32x4_t; - let mut store_1: float32x4_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_f32(s_ptr); - let edge1 = vld1q_f32(s_ptr.add(4)); - - store_0 = vmulq_f32(edge0, v_edge_count); - store_1 = vmulq_f32(edge1, v_edge_count); - - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - - let edge0 = vld1q_f32(s_ptr); - let edge1 = vld1q_f32(s_ptr.add(4)); - - store_0 = vaddq_f32(store_0, edge0); - store_1 = vaddq_f32(store_1, edge1); - } - - vst1q_f32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - vst1q_f32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); - - cx += 8; - buf_cx += 8; - } - - while cx + 4 < end_x { - let px = cx; - - let mut store_0: float32x4_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_f32(s_ptr); + store_0 = vmulq_f32(edge0, v_edge_count); + store_1 = vmulq_f32(edge1, v_edge_count); - store_0 = vmulq_f32(edge0, v_edge_count); - - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - - let edge0 = vld1q_f32(s_ptr); - - store_0 = vaddq_f32(store_0, edge0); - } - - vst1q_f32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - - cx += 4; - buf_cx += 4; - } - - while cx < end_x { - let px = cx; - - let mut store_0: float32x4_t; - - let s_ptr = src.as_ptr().add(px); - let edge0 = vld1q_lane_f32::<0>(s_ptr, vdupq_n_f32(0.)); - - store_0 = vmulq_f32(edge0, v_edge_count); - - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - - let edge0 = vld1q_lane_f32::<0>(s_ptr, vdupq_n_f32(0.)); - store_0 = vaddq_f32(store_0, edge0); - } - - vst1q_lane_f32::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); - - cx += 1; - buf_cx += 1; - } - - for y in 0..height { - let mut buf_cx = 0usize; - let mut cx = start_x; - - // preload edge pixels - let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; - let previous = (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y as usize; - - while cx + 16 < end_x { - let px = cx; - - let mut store_0 = vld1q_f32(buffer.get_unchecked(buf_cx..).as_ptr()); - let mut store_1 = vld1q_f32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); - let mut store_2 = vld1q_f32(buffer.get_unchecked(buf_cx + 8..).as_ptr()); - let mut store_3 = vld1q_f32(buffer.get_unchecked(buf_cx + 12..).as_ptr()); - - { - let px = cx; - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vmulq_f32(store_0, v_weight); - let px_16_hi0 = vmulq_f32(store_1, v_weight); - let px_16_lo1 = vmulq_f32(store_2, v_weight); - let px_16_hi2 = vmulq_f32(store_3, v_weight); - - vst1q_f32(ptr, px_16_lo0); - vst1q_f32(ptr.add(4), px_16_hi0); - vst1q_f32(ptr.add(8), px_16_lo1); - vst1q_f32(ptr.add(12), px_16_hi2); - } - - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - - let edge0 = vld1q_f32(s_ptr); - let edge1 = vld1q_f32(s_ptr.add(4)); - let edge2 = vld1q_f32(s_ptr.add(8)); - let edge3 = vld1q_f32(s_ptr.add(12)); + store_2 = vmulq_f32(edge2, v_edge_count); + store_3 = vmulq_f32(edge3, v_edge_count); - store_0 = vsubq_f32(store_0, edge0); - store_1 = vsubq_f32(store_1, edge1); - store_2 = vsubq_f32(store_2, edge2); - store_3 = vsubq_f32(store_3, edge3); - } - - // add next - { - let s_ptr = src.as_ptr().add(next + px); + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); let edge0 = vld1q_f32(s_ptr); let edge1 = vld1q_f32(s_ptr.add(4)); @@ -505,38 +354,23 @@ unsafe fn box_blur_vertical_pass_neon_any_rgba_f32( buf_cx += 16; } - while cx + 8 < end_x { + while cx + 8 <= end_x { let px = cx; - let mut store_0 = vld1q_f32(buffer.get_unchecked(buf_cx..).as_ptr()); - let mut store_1 = vld1q_f32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); - - { - let px = cx; - - let px_16_lo0 = vmulq_f32(store_0, v_weight); - let px_16_hi0 = vmulq_f32(store_1, v_weight); - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let mut store_0: float32x4_t; + let mut store_1: float32x4_t; - vst1q_f32(ptr, px_16_lo0); - vst1q_f32(ptr.add(4), px_16_hi0); - } + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1q_f32(s_ptr); + let edge1 = vld1q_f32(s_ptr.add(4)); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge0 = vld1q_f32(s_ptr); - let edge1 = vld1q_f32(s_ptr.add(4)); + store_0 = vmulq_f32(edge0, v_edge_count); + store_1 = vmulq_f32(edge1, v_edge_count); - store_0 = vsubq_f32(store_0, edge0); - store_1 = vsubq_f32(store_1, edge1); - } + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); let edge0 = vld1q_f32(s_ptr); let edge1 = vld1q_f32(s_ptr.add(4)); @@ -551,33 +385,23 @@ unsafe fn box_blur_vertical_pass_neon_any_rgba_f32( buf_cx += 8; } - while cx + 4 < end_x { + while cx + 4 <= end_x { let px = cx; - let mut store_0 = vld1q_f32(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_0: float32x4_t; - { - let px = cx; - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vmulq_f32(store_0, v_weight); - vst1q_f32(ptr, px_16_lo0); - } + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1q_f32(s_ptr); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = vld1q_f32(s_ptr); + store_0 = vmulq_f32(edge0, v_edge_count); - store_0 = vsubq_f32(store_0, edge); - } + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = vld1q_f32(s_ptr); + let edge0 = vld1q_f32(s_ptr); - store_0 = vaddq_f32(store_0, edge); + store_0 = vaddq_f32(store_0, edge0); } vst1q_f32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); @@ -589,30 +413,19 @@ unsafe fn box_blur_vertical_pass_neon_any_rgba_f32( while cx < end_x { let px = cx; - let mut store_0 = - vld1q_lane_f32::<0>(buffer.get_unchecked(buf_cx..).as_ptr(), vdupq_n_f32(0.)); + let mut store_0: float32x4_t; - { - let px = cx; - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - let px_16_lo0 = vmulq_f32(store_0, v_weight); - vst1q_lane_f32::<0>(ptr, px_16_lo0); - } + let s_ptr = src.as_ptr().add(px); + let edge0 = vld1q_lane_f32::<0>(s_ptr, vdupq_n_f32(0.)); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = vld1q_lane_f32::<0>(s_ptr, vdupq_n_f32(0.)); + store_0 = vmulq_f32(edge0, v_edge_count); - store_0 = vsubq_f32(store_0, edge); - } + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = vld1q_lane_f32::<0>(s_ptr, vdupq_n_f32(0.)); - store_0 = vaddq_f32(store_0, edge); + let edge0 = vld1q_lane_f32::<0>(s_ptr, vdupq_n_f32(0.)); + store_0 = vaddq_f32(store_0, edge0); } vst1q_lane_f32::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); @@ -620,5 +433,198 @@ unsafe fn box_blur_vertical_pass_neon_any_rgba_f32( cx += 1; buf_cx += 1; } + + for y in 0..height { + let mut buf_cx = 0usize; + let mut cx = start_x; + + // preload edge pixels + let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; + let previous = + (y as isize - half_kernel as isize).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y as usize; + + while cx + 16 < end_x { + let px = cx; + + let mut store_0 = vld1q_f32(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_1 = vld1q_f32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); + let mut store_2 = vld1q_f32(buffer.get_unchecked(buf_cx + 8..).as_ptr()); + let mut store_3 = vld1q_f32(buffer.get_unchecked(buf_cx + 12..).as_ptr()); + + { + let px = cx; + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vmulq_f32(store_0, v_weight); + let px_16_hi0 = vmulq_f32(store_1, v_weight); + let px_16_lo1 = vmulq_f32(store_2, v_weight); + let px_16_hi2 = vmulq_f32(store_3, v_weight); + + vst1q_f32(ptr, px_16_lo0); + vst1q_f32(ptr.add(4), px_16_hi0); + vst1q_f32(ptr.add(8), px_16_lo1); + vst1q_f32(ptr.add(12), px_16_hi2); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + + let edge0 = vld1q_f32(s_ptr); + let edge1 = vld1q_f32(s_ptr.add(4)); + let edge2 = vld1q_f32(s_ptr.add(8)); + let edge3 = vld1q_f32(s_ptr.add(12)); + + store_0 = vsubq_f32(store_0, edge0); + store_1 = vsubq_f32(store_1, edge1); + store_2 = vsubq_f32(store_2, edge2); + store_3 = vsubq_f32(store_3, edge3); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + + let edge0 = vld1q_f32(s_ptr); + let edge1 = vld1q_f32(s_ptr.add(4)); + let edge2 = vld1q_f32(s_ptr.add(8)); + let edge3 = vld1q_f32(s_ptr.add(12)); + + store_0 = vaddq_f32(store_0, edge0); + store_1 = vaddq_f32(store_1, edge1); + store_2 = vaddq_f32(store_2, edge2); + store_3 = vaddq_f32(store_3, edge3); + } + + vst1q_f32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + vst1q_f32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); + vst1q_f32(buffer.get_unchecked_mut(buf_cx + 8..).as_mut_ptr(), store_2); + vst1q_f32( + buffer.get_unchecked_mut(buf_cx + 12..).as_mut_ptr(), + store_3, + ); + + cx += 16; + buf_cx += 16; + } + + while cx + 8 < end_x { + let px = cx; + + let mut store_0 = vld1q_f32(buffer.get_unchecked(buf_cx..).as_ptr()); + let mut store_1 = vld1q_f32(buffer.get_unchecked(buf_cx + 4..).as_ptr()); + + { + let px = cx; + + let px_16_lo0 = vmulq_f32(store_0, v_weight); + let px_16_hi0 = vmulq_f32(store_1, v_weight); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + + vst1q_f32(ptr, px_16_lo0); + vst1q_f32(ptr.add(4), px_16_hi0); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge0 = vld1q_f32(s_ptr); + let edge1 = vld1q_f32(s_ptr.add(4)); + + store_0 = vsubq_f32(store_0, edge0); + store_1 = vsubq_f32(store_1, edge1); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge0 = vld1q_f32(s_ptr); + let edge1 = vld1q_f32(s_ptr.add(4)); + + store_0 = vaddq_f32(store_0, edge0); + store_1 = vaddq_f32(store_1, edge1); + } + + vst1q_f32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + vst1q_f32(buffer.get_unchecked_mut(buf_cx + 4..).as_mut_ptr(), store_1); + + cx += 8; + buf_cx += 8; + } + + while cx + 4 < end_x { + let px = cx; + + let mut store_0 = vld1q_f32(buffer.get_unchecked(buf_cx..).as_ptr()); + + { + let px = cx; + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vmulq_f32(store_0, v_weight); + vst1q_f32(ptr, px_16_lo0); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = vld1q_f32(s_ptr); + + store_0 = vsubq_f32(store_0, edge); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = vld1q_f32(s_ptr); + + store_0 = vaddq_f32(store_0, edge); + } + + vst1q_f32(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + + cx += 4; + buf_cx += 4; + } + + while cx < end_x { + let px = cx; + + let mut store_0 = + vld1q_lane_f32::<0>(buffer.get_unchecked(buf_cx..).as_ptr(), vdupq_n_f32(0.)); + + { + let px = cx; + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px_16_lo0 = vmulq_f32(store_0, v_weight); + vst1q_lane_f32::<0>(ptr, px_16_lo0); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = vld1q_lane_f32::<0>(s_ptr, vdupq_n_f32(0.)); + + store_0 = vsubq_f32(store_0, edge); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = vld1q_lane_f32::<0>(s_ptr, vdupq_n_f32(0.)); + store_0 = vaddq_f32(store_0, edge); + } + + vst1q_lane_f32::<0>(buffer.get_unchecked_mut(buf_cx..).as_mut_ptr(), store_0); + + cx += 1; + buf_cx += 1; + } + } } } diff --git a/src/box_filter/neon/vsum8.rs b/src/box_filter/neon/vsum8.rs index ed8573ad..da1ec8ed 100644 --- a/src/box_filter/neon/vsum8.rs +++ b/src/box_filter/neon/vsum8.rs @@ -139,8 +139,6 @@ pub(crate) fn neon_ring_vertical_row_summ_rdm_impl( let weight = (((1i64 << 31) - 1) as f64 / (radius as f64 * 2. + 1.)) as i32; let v_weight = unsafe { vdupq_n_s32(weight) }; - let chunks = previous_row.chunks_exact(16).len(); - for (((src_next, src_previous), buffer), dst) in next_row .chunks_exact(16) .zip(previous_row.chunks_exact(16)) @@ -197,12 +195,16 @@ pub(crate) fn neon_ring_vertical_row_summ_rdm_impl( } } - for (((src_next, src_previous), buffer), dst) in next_row + let next_rem = next_row.chunks_exact(16).remainder(); + let previous_rem = previous_row.chunks_exact(16).remainder(); + let working_rem = working_row.chunks_exact_mut(16).into_remainder(); + let dst_rem = dst.chunks_exact_mut(16).into_remainder(); + + for (((src_next, src_previous), buffer), dst) in next_rem .iter() - .zip(previous_row.iter()) - .zip(working_row.iter_mut()) - .zip(dst.iter_mut()) - .skip(chunks * 16) + .zip(previous_rem.iter()) + .zip(working_rem.iter_mut()) + .zip(dst_rem.iter_mut()) { let mut weight0 = *buffer; diff --git a/src/box_filter/sse/mod.rs b/src/box_filter/sse/mod.rs index 13329d8d..c0833ec5 100644 --- a/src/box_filter/sse/mod.rs +++ b/src/box_filter/sse/mod.rs @@ -33,9 +33,9 @@ mod vsum16; mod vsum8; mod vsum_f32; -pub(crate) use rgba16::{box_blur_horizontal_pass_sse16, box_blur_vertical_pass_sse16}; -pub(crate) use rgba8::{box_blur_horizontal_pass_sse, box_blur_vertical_pass_sse}; pub(crate) use rgba_f32::box_blur_horizontal_pass_sse_f32; -pub(crate) use vsum16::sse_ring_vertical_row_summ16; -pub(crate) use vsum8::sse_ring_vertical_row_summ; +pub(crate) use rgba8::{box_blur_horizontal_pass_sse, box_blur_vertical_pass_sse}; +pub(crate) use rgba16::{box_blur_horizontal_pass_sse16, box_blur_vertical_pass_sse16}; pub(crate) use vsum_f32::sse_ring_vertical_row_summ_f32; +pub(crate) use vsum8::sse_ring_vertical_row_summ; +pub(crate) use vsum16::sse_ring_vertical_row_summ16; diff --git a/src/box_filter/sse/rgba16.rs b/src/box_filter/sse/rgba16.rs index e2e82b12..16f22363 100644 --- a/src/box_filter/sse/rgba16.rs +++ b/src/box_filter/sse/rgba16.rs @@ -58,7 +58,7 @@ struct HorizontalExecutionUnit {} impl HorizontalExecutionUnit { #[target_feature(enable = "sse4.1")] - unsafe fn pass( + fn pass( &self, src: &[u16], src_stride: u32, @@ -69,211 +69,213 @@ impl HorizontalExecutionUnit { start_y: u32, end_y: u32, ) { - let kernel_size = radius * 2 + 1; - let edge_count = (kernel_size / 2) + 1; - let v_edge_count = _mm_set1_epi32(edge_count as i32); - - let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); - - let half_kernel = kernel_size / 2; + unsafe { + let kernel_size = radius * 2 + 1; + let edge_count = (kernel_size / 2) + 1; + let v_edge_count = _mm_set1_epi32(edge_count as i32); - let mut yy = start_y; + let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); - while yy + 4 < end_y { - let y = yy; - let y_src_shift = y as usize * src_stride as usize; - let y_dst_shift = y as usize * dst_stride as usize; + let half_kernel = kernel_size / 2; - let mut store_0: __m128i; - let mut store_1: __m128i; - let mut store_2: __m128i; - let mut store_3: __m128i; + let mut yy = start_y; - unsafe { - let s_ptr_0 = src.as_ptr().add(y_src_shift); - let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize); - let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2); - let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3); - - let edge_colors_0 = load_u16_s32_fast::(s_ptr_0); - let edge_colors_1 = load_u16_s32_fast::(s_ptr_1); - let edge_colors_2 = load_u16_s32_fast::(s_ptr_2); - let edge_colors_3 = load_u16_s32_fast::(s_ptr_3); - - store_0 = _mm_mullo_epi32(edge_colors_0, v_edge_count); - store_1 = _mm_mullo_epi32(edge_colors_1, v_edge_count); - store_2 = _mm_mullo_epi32(edge_colors_2, v_edge_count); - store_3 = _mm_mullo_epi32(edge_colors_3, v_edge_count); - } + while yy + 4 <= end_y { + let y = yy; + let y_src_shift = y as usize * src_stride as usize; + let y_dst_shift = y as usize * dst_stride as usize; - unsafe { - for x in 1usize..=half_kernel as usize { - let px = x.min(width as usize - 1) * CN; + let mut store_0: __m128i; + let mut store_1: __m128i; + let mut store_2: __m128i; + let mut store_3: __m128i; - let s_ptr_0 = src.as_ptr().add(y_src_shift + px); - let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + px); - let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2 + px); - let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3 + px); + { + let s_ptr_0 = src.as_ptr().add(y_src_shift); + let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize); + let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2); + let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3); let edge_colors_0 = load_u16_s32_fast::(s_ptr_0); let edge_colors_1 = load_u16_s32_fast::(s_ptr_1); let edge_colors_2 = load_u16_s32_fast::(s_ptr_2); let edge_colors_3 = load_u16_s32_fast::(s_ptr_3); - store_0 = _mm_add_epi32(store_0, edge_colors_0); - store_1 = _mm_add_epi32(store_1, edge_colors_1); - store_2 = _mm_add_epi32(store_2, edge_colors_2); - store_3 = _mm_add_epi32(store_3, edge_colors_3); + store_0 = _mm_mullo_epi32(edge_colors_0, v_edge_count); + store_1 = _mm_mullo_epi32(edge_colors_1, v_edge_count); + store_2 = _mm_mullo_epi32(edge_colors_2, v_edge_count); + store_3 = _mm_mullo_epi32(edge_colors_3, v_edge_count); } - } - for x in 0..width { - let px = x as usize * CN; - - unsafe { - let scale_store_ps0 = _mm_cvtepi32_ps(store_0); - let scale_store_ps1 = _mm_cvtepi32_ps(store_1); - let scale_store_ps2 = _mm_cvtepi32_ps(store_2); - let scale_store_ps3 = _mm_cvtepi32_ps(store_3); - - let r0 = _mm_mul_ps(scale_store_ps0, v_weight); - let r1 = _mm_mul_ps(scale_store_ps1, v_weight); - let r2 = _mm_mul_ps(scale_store_ps2, v_weight); - let r3 = _mm_mul_ps(scale_store_ps3, v_weight); - - let scale_store0 = _mm_cvtps_epi32(r0); - let scale_store1 = _mm_cvtps_epi32(r1); - let scale_store2 = _mm_cvtps_epi32(r2); - let scale_store3 = _mm_cvtps_epi32(r3); - - let bytes_offset_0 = y_dst_shift + px; - let bytes_offset_1 = y_dst_shift + dst_stride as usize + px; - let bytes_offset_2 = y_dst_shift + dst_stride as usize * 2 + px; - let bytes_offset_3 = y_dst_shift + dst_stride as usize * 3 + px; - store_u16_u32::( - unsafe_dst.slice.as_ptr().add(bytes_offset_0) as *mut _, - scale_store0, - ); - store_u16_u32::( - unsafe_dst.slice.as_ptr().add(bytes_offset_1) as *mut _, - scale_store1, - ); - store_u16_u32::( - unsafe_dst.slice.as_ptr().add(bytes_offset_2) as *mut _, - scale_store2, - ); - store_u16_u32::( - unsafe_dst.slice.as_ptr().add(bytes_offset_3) as *mut _, - scale_store3, - ); + { + for x in 1usize..=half_kernel as usize { + let px = x.min(width as usize - 1) * CN; + + let s_ptr_0 = src.as_ptr().add(y_src_shift + px); + let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + px); + let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2 + px); + let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3 + px); + + let edge_colors_0 = load_u16_s32_fast::(s_ptr_0); + let edge_colors_1 = load_u16_s32_fast::(s_ptr_1); + let edge_colors_2 = load_u16_s32_fast::(s_ptr_2); + let edge_colors_3 = load_u16_s32_fast::(s_ptr_3); + + store_0 = _mm_add_epi32(store_0, edge_colors_0); + store_1 = _mm_add_epi32(store_1, edge_colors_1); + store_2 = _mm_add_epi32(store_2, edge_colors_2); + store_3 = _mm_add_epi32(store_3, edge_colors_3); + } } - // subtract previous - unsafe { - let previous_x = (x as i64 - half_kernel as i64).max(0) as usize; - let previous = previous_x * CN; - - let s_ptr_0 = src.as_ptr().add(y_src_shift + previous); - let s_ptr_1 = src - .as_ptr() - .add(y_src_shift + src_stride as usize + previous); - let s_ptr_2 = src - .as_ptr() - .add(y_src_shift + src_stride as usize * 2 + previous); - let s_ptr_3 = src - .as_ptr() - .add(y_src_shift + src_stride as usize * 3 + previous); - - let edge_colors_0 = load_u16_s32_fast::(s_ptr_0); - let edge_colors_1 = load_u16_s32_fast::(s_ptr_1); - let edge_colors_2 = load_u16_s32_fast::(s_ptr_2); - let edge_colors_3 = load_u16_s32_fast::(s_ptr_3); - - store_0 = _mm_sub_epi32(store_0, edge_colors_0); - store_1 = _mm_sub_epi32(store_1, edge_colors_1); - store_2 = _mm_sub_epi32(store_2, edge_colors_2); - store_3 = _mm_sub_epi32(store_3, edge_colors_3); + for x in 0..width { + let px = x as usize * CN; + + { + let scale_store_ps0 = _mm_cvtepi32_ps(store_0); + let scale_store_ps1 = _mm_cvtepi32_ps(store_1); + let scale_store_ps2 = _mm_cvtepi32_ps(store_2); + let scale_store_ps3 = _mm_cvtepi32_ps(store_3); + + let r0 = _mm_mul_ps(scale_store_ps0, v_weight); + let r1 = _mm_mul_ps(scale_store_ps1, v_weight); + let r2 = _mm_mul_ps(scale_store_ps2, v_weight); + let r3 = _mm_mul_ps(scale_store_ps3, v_weight); + + let scale_store0 = _mm_cvtps_epi32(r0); + let scale_store1 = _mm_cvtps_epi32(r1); + let scale_store2 = _mm_cvtps_epi32(r2); + let scale_store3 = _mm_cvtps_epi32(r3); + + let bytes_offset_0 = y_dst_shift + px; + let bytes_offset_1 = y_dst_shift + dst_stride as usize + px; + let bytes_offset_2 = y_dst_shift + dst_stride as usize * 2 + px; + let bytes_offset_3 = y_dst_shift + dst_stride as usize * 3 + px; + store_u16_u32::( + unsafe_dst.slice.as_ptr().add(bytes_offset_0) as *mut _, + scale_store0, + ); + store_u16_u32::( + unsafe_dst.slice.as_ptr().add(bytes_offset_1) as *mut _, + scale_store1, + ); + store_u16_u32::( + unsafe_dst.slice.as_ptr().add(bytes_offset_2) as *mut _, + scale_store2, + ); + store_u16_u32::( + unsafe_dst.slice.as_ptr().add(bytes_offset_3) as *mut _, + scale_store3, + ); + } + + // subtract previous + { + let previous_x = (x as i64 - half_kernel as i64).max(0) as usize; + let previous = previous_x * CN; + + let s_ptr_0 = src.as_ptr().add(y_src_shift + previous); + let s_ptr_1 = src + .as_ptr() + .add(y_src_shift + src_stride as usize + previous); + let s_ptr_2 = src + .as_ptr() + .add(y_src_shift + src_stride as usize * 2 + previous); + let s_ptr_3 = src + .as_ptr() + .add(y_src_shift + src_stride as usize * 3 + previous); + + let edge_colors_0 = load_u16_s32_fast::(s_ptr_0); + let edge_colors_1 = load_u16_s32_fast::(s_ptr_1); + let edge_colors_2 = load_u16_s32_fast::(s_ptr_2); + let edge_colors_3 = load_u16_s32_fast::(s_ptr_3); + + store_0 = _mm_sub_epi32(store_0, edge_colors_0); + store_1 = _mm_sub_epi32(store_1, edge_colors_1); + store_2 = _mm_sub_epi32(store_2, edge_colors_2); + store_3 = _mm_sub_epi32(store_3, edge_colors_3); + } + + // add next + { + let next_x = (x + half_kernel + 1).min(width - 1) as usize; + + let next = next_x * CN; + + let s_ptr_0 = src.as_ptr().add(y_src_shift + next); + let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + next); + let s_ptr_2 = src + .as_ptr() + .add(y_src_shift + src_stride as usize * 2 + next); + let s_ptr_3 = src + .as_ptr() + .add(y_src_shift + src_stride as usize * 3 + next); + + let edge_colors_0 = load_u16_s32_fast::(s_ptr_0); + let edge_colors_1 = load_u16_s32_fast::(s_ptr_1); + let edge_colors_2 = load_u16_s32_fast::(s_ptr_2); + let edge_colors_3 = load_u16_s32_fast::(s_ptr_3); + + store_0 = _mm_add_epi32(store_0, edge_colors_0); + store_1 = _mm_add_epi32(store_1, edge_colors_1); + store_2 = _mm_add_epi32(store_2, edge_colors_2); + store_3 = _mm_add_epi32(store_3, edge_colors_3); + } } - // add next - unsafe { - let next_x = (x + half_kernel + 1).min(width - 1) as usize; - - let next = next_x * CN; - - let s_ptr_0 = src.as_ptr().add(y_src_shift + next); - let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + next); - let s_ptr_2 = src - .as_ptr() - .add(y_src_shift + src_stride as usize * 2 + next); - let s_ptr_3 = src - .as_ptr() - .add(y_src_shift + src_stride as usize * 3 + next); - - let edge_colors_0 = load_u16_s32_fast::(s_ptr_0); - let edge_colors_1 = load_u16_s32_fast::(s_ptr_1); - let edge_colors_2 = load_u16_s32_fast::(s_ptr_2); - let edge_colors_3 = load_u16_s32_fast::(s_ptr_3); - - store_0 = _mm_add_epi32(store_0, edge_colors_0); - store_1 = _mm_add_epi32(store_1, edge_colors_1); - store_2 = _mm_add_epi32(store_2, edge_colors_2); - store_3 = _mm_add_epi32(store_3, edge_colors_3); - } + yy += 4; } - yy += 4; - } - - for y in yy..end_y { - let y_src_shift = y as usize * src_stride as usize; - let y_dst_shift = y as usize * dst_stride as usize; - - let mut store; + for y in yy..end_y { + let y_src_shift = y as usize * src_stride as usize; + let y_dst_shift = y as usize * dst_stride as usize; - unsafe { - let s_ptr = src.as_ptr().add(y_src_shift); - let edge_colors = load_u16_s32_fast::(s_ptr); - store = _mm_mullo_epi32(edge_colors, v_edge_count); - } + let mut store; - unsafe { - for x in 1usize..=half_kernel as usize { - let px = x.min(width as usize - 1) * CN; - let s_ptr = src.as_ptr().add(y_src_shift + px); + { + let s_ptr = src.as_ptr().add(y_src_shift); let edge_colors = load_u16_s32_fast::(s_ptr); - store = _mm_add_epi32(store, edge_colors); - } - } - - for x in 0..width { - let px = x as usize * CN; - - unsafe { - let r0 = _mm_mul_ps(_mm_cvtepi32_ps(store), v_weight); - let bytes_offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.as_ptr().add(bytes_offset) as *mut u16; - store_u16_u32::(ptr, _mm_cvtps_epi32(r0)); + store = _mm_mullo_epi32(edge_colors, v_edge_count); } - // subtract previous - unsafe { - let previous_x = (x as i64 - half_kernel as i64).max(0) as usize; - let previous = previous_x * CN; - let s_ptr = src.as_ptr().add(y_src_shift + previous); - let edge_colors = load_u16_s32_fast::(s_ptr); - store = _mm_sub_epi32(store, edge_colors); + { + for x in 1usize..=half_kernel as usize { + let px = x.min(width as usize - 1) * CN; + let s_ptr = src.as_ptr().add(y_src_shift + px); + let edge_colors = load_u16_s32_fast::(s_ptr); + store = _mm_add_epi32(store, edge_colors); + } } - // add next - unsafe { - let next_x = (x + half_kernel + 1).min(width - 1) as usize; - - let next = next_x * CN; - - let s_ptr = src.as_ptr().add(y_src_shift + next); - let edge_colors = load_u16_s32_fast::(s_ptr); - store = _mm_add_epi32(store, edge_colors); + for x in 0..width { + let px = x as usize * CN; + + { + let r0 = _mm_mul_ps(_mm_cvtepi32_ps(store), v_weight); + let bytes_offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.as_ptr().add(bytes_offset) as *mut u16; + store_u16_u32::(ptr, _mm_cvtps_epi32(r0)); + } + + // subtract previous + { + let previous_x = (x as i64 - half_kernel as i64).max(0) as usize; + let previous = previous_x * CN; + let s_ptr = src.as_ptr().add(y_src_shift + previous); + let edge_colors = load_u16_s32_fast::(s_ptr); + store = _mm_sub_epi32(store, edge_colors); + } + + // add next + { + let next_x = (x + half_kernel + 1).min(width - 1) as usize; + + let next = next_x * CN; + + let s_ptr = src.as_ptr().add(y_src_shift + next); + let edge_colors = load_u16_s32_fast::(s_ptr); + store = _mm_add_epi32(store, edge_colors); + } } } } @@ -299,7 +301,7 @@ pub(crate) fn box_blur_vertical_pass_sse16( } #[target_feature(enable = "sse4.1")] -unsafe fn box_blur_vertical_pass_sse_impl16( +fn box_blur_vertical_pass_sse_impl16( src: &[u16], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -310,46 +312,27 @@ unsafe fn box_blur_vertical_pass_sse_impl16( start_x: u32, end_x: u32, ) { - let kernel_size = radius * 2 + 1; - let edge_count = (kernel_size / 2) + 1; - let v_edge_count = _mm_set1_epi32(edge_count as i32); - - let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); - - let half_kernel = kernel_size / 2; - - let mut cx = start_x; - - while cx + 16 < end_x { - let px = cx as usize; - - let mut store_0: __m128i; - let mut store_1: __m128i; - let mut store_2: __m128i; - let mut store_3: __m128i; + unsafe { + let kernel_size = radius * 2 + 1; + let edge_count = (kernel_size / 2) + 1; + let v_edge_count = _mm_set1_epi32(edge_count as i32); - unsafe { - let s_ptr = src.as_ptr().add(px); - let edge0 = _mm_loadu_si128(s_ptr as *const _); - let edge1 = _mm_loadu_si128(s_ptr.add(8) as *const _); + let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); - let i16_l0 = _mm_unpacklo_epi16(edge0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(edge0, _mm_setzero_si128()); - let i16_l1 = _mm_unpacklo_epi16(edge1, _mm_setzero_si128()); - let i16_h1 = _mm_unpackhi_epi16(edge1, _mm_setzero_si128()); + let half_kernel = kernel_size / 2; - store_0 = _mm_mullo_epi32(i16_l0, v_edge_count); - store_1 = _mm_mullo_epi32(i16_h0, v_edge_count); + let mut cx = start_x; - store_2 = _mm_mullo_epi32(i16_l1, v_edge_count); - store_3 = _mm_mullo_epi32(i16_h1, v_edge_count); - } + while cx + 16 <= end_x { + let px = cx as usize; - unsafe { - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + let mut store_0: __m128i; + let mut store_1: __m128i; + let mut store_2: __m128i; + let mut store_3: __m128i; + { + let s_ptr = src.as_ptr().add(px); let edge0 = _mm_loadu_si128(s_ptr as *const _); let edge1 = _mm_loadu_si128(s_ptr.add(8) as *const _); @@ -358,71 +341,18 @@ unsafe fn box_blur_vertical_pass_sse_impl16( let i16_l1 = _mm_unpacklo_epi16(edge1, _mm_setzero_si128()); let i16_h1 = _mm_unpackhi_epi16(edge1, _mm_setzero_si128()); - store_0 = _mm_add_epi32(store_0, i16_l0); - store_1 = _mm_add_epi32(store_1, i16_h0); + store_0 = _mm_mullo_epi32(i16_l0, v_edge_count); + store_1 = _mm_mullo_epi32(i16_h0, v_edge_count); - store_2 = _mm_add_epi32(store_2, i16_l1); - store_3 = _mm_add_epi32(store_3, i16_h1); + store_2 = _mm_mullo_epi32(i16_l1, v_edge_count); + store_3 = _mm_mullo_epi32(i16_h1, v_edge_count); } - } - - unsafe { - for y in 0..height { - // preload edge pixels - let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; - let previous = - (y as i64 - half_kernel as i64).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y as usize; - - { - let px = cx as usize; - - let store0_ps = _mm_cvtepi32_ps(store_0); - let store1_ps = _mm_cvtepi32_ps(store_1); - let store2_ps = _mm_cvtepi32_ps(store_2); - let store3_ps = _mm_cvtepi32_ps(store_3); - - let r0 = _mm_mul_ps(store0_ps, v_weight); - let r1 = _mm_mul_ps(store1_ps, v_weight); - let r2 = _mm_mul_ps(store2_ps, v_weight); - let r3 = _mm_mul_ps(store3_ps, v_weight); - - let scale_store_0 = _mm_cvtps_epi32(r0); - let scale_store_1 = _mm_cvtps_epi32(r1); - let scale_store_2 = _mm_cvtps_epi32(r2); - let scale_store_3 = _mm_cvtps_epi32(r3); - - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - - let set0 = _mm_packus_epi32(scale_store_0, scale_store_1); - let set1 = _mm_packus_epi32(scale_store_2, scale_store_3); - - _mm_storeu_si128(ptr as *mut _, set0); - _mm_storeu_si128(ptr.add(8) as *mut _, set1); - } - - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge0 = _mm_loadu_si128(s_ptr as *const _); - let edge1 = _mm_loadu_si128(s_ptr.add(8) as *const _); - - let i16_l0 = _mm_unpacklo_epi16(edge0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(edge0, _mm_setzero_si128()); - let i16_l1 = _mm_unpacklo_epi16(edge1, _mm_setzero_si128()); - let i16_h1 = _mm_unpackhi_epi16(edge1, _mm_setzero_si128()); - store_0 = _mm_sub_epi32(store_0, i16_l0); - store_1 = _mm_sub_epi32(store_1, i16_h0); - - store_2 = _mm_sub_epi32(store_2, i16_l1); - store_3 = _mm_sub_epi32(store_3, i16_h1); - } + { + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - // add next - { - let s_ptr = src.as_ptr().add(next + px); let edge0 = _mm_loadu_si128(s_ptr as *const _); let edge1 = _mm_loadu_si128(s_ptr.add(8) as *const _); @@ -438,148 +368,222 @@ unsafe fn box_blur_vertical_pass_sse_impl16( store_3 = _mm_add_epi32(store_3, i16_h1); } } - } - - cx += 16; - } - - while cx + 8 < end_x { - let px = cx as usize; - let mut store_0: __m128i; - let mut store_1: __m128i; - - unsafe { - let s_ptr = src.as_ptr().add(px); - let edge = _mm_loadu_si128(s_ptr as *const _); - - let i16_l0 = _mm_unpacklo_epi16(edge, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(edge, _mm_setzero_si128()); + { + for y in 0..height { + // preload edge pixels + let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; + let previous = + (y as i64 - half_kernel as i64).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y as usize; + + { + let px = cx as usize; + + let store0_ps = _mm_cvtepi32_ps(store_0); + let store1_ps = _mm_cvtepi32_ps(store_1); + let store2_ps = _mm_cvtepi32_ps(store_2); + let store3_ps = _mm_cvtepi32_ps(store_3); + + let r0 = _mm_mul_ps(store0_ps, v_weight); + let r1 = _mm_mul_ps(store1_ps, v_weight); + let r2 = _mm_mul_ps(store2_ps, v_weight); + let r3 = _mm_mul_ps(store3_ps, v_weight); + + let scale_store_0 = _mm_cvtps_epi32(r0); + let scale_store_1 = _mm_cvtps_epi32(r1); + let scale_store_2 = _mm_cvtps_epi32(r2); + let scale_store_3 = _mm_cvtps_epi32(r3); + + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + + let set0 = _mm_packus_epi32(scale_store_0, scale_store_1); + let set1 = _mm_packus_epi32(scale_store_2, scale_store_3); + + _mm_storeu_si128(ptr as *mut _, set0); + _mm_storeu_si128(ptr.add(8) as *mut _, set1); + } + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge0 = _mm_loadu_si128(s_ptr as *const _); + let edge1 = _mm_loadu_si128(s_ptr.add(8) as *const _); + + let i16_l0 = _mm_unpacklo_epi16(edge0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(edge0, _mm_setzero_si128()); + let i16_l1 = _mm_unpacklo_epi16(edge1, _mm_setzero_si128()); + let i16_h1 = _mm_unpackhi_epi16(edge1, _mm_setzero_si128()); + + store_0 = _mm_sub_epi32(store_0, i16_l0); + store_1 = _mm_sub_epi32(store_1, i16_h0); + + store_2 = _mm_sub_epi32(store_2, i16_l1); + store_3 = _mm_sub_epi32(store_3, i16_h1); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge0 = _mm_loadu_si128(s_ptr as *const _); + let edge1 = _mm_loadu_si128(s_ptr.add(8) as *const _); + + let i16_l0 = _mm_unpacklo_epi16(edge0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(edge0, _mm_setzero_si128()); + let i16_l1 = _mm_unpacklo_epi16(edge1, _mm_setzero_si128()); + let i16_h1 = _mm_unpackhi_epi16(edge1, _mm_setzero_si128()); + + store_0 = _mm_add_epi32(store_0, i16_l0); + store_1 = _mm_add_epi32(store_1, i16_h0); + + store_2 = _mm_add_epi32(store_2, i16_l1); + store_3 = _mm_add_epi32(store_3, i16_h1); + } + } + } - store_0 = _mm_mullo_epi32(i16_l0, v_edge_count); - store_1 = _mm_mullo_epi32(i16_h0, v_edge_count); + cx += 16; } - unsafe { - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + while cx + 8 <= end_x { + let px = cx as usize; + + let mut store_0: __m128i; + let mut store_1: __m128i; + { + let s_ptr = src.as_ptr().add(px); let edge = _mm_loadu_si128(s_ptr as *const _); let i16_l0 = _mm_unpacklo_epi16(edge, _mm_setzero_si128()); let i16_h0 = _mm_unpackhi_epi16(edge, _mm_setzero_si128()); - store_0 = _mm_add_epi32(store_0, i16_l0); - store_1 = _mm_add_epi32(store_1, i16_h0); + store_0 = _mm_mullo_epi32(i16_l0, v_edge_count); + store_1 = _mm_mullo_epi32(i16_h0, v_edge_count); } - } - unsafe { - for y in 0..height { - // preload edge pixels - let next = - std::cmp::min(y + half_kernel + 1, height - 1) as usize * src_stride as usize; - let previous = - std::cmp::max(y as i64 - half_kernel as i64, 0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y as usize; + { + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - { - let px = cx as usize; + let edge = _mm_loadu_si128(s_ptr as *const _); - let store0_ps = _mm_cvtepi32_ps(store_0); - let store1_ps = _mm_cvtepi32_ps(store_1); + let i16_l0 = _mm_unpacklo_epi16(edge, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(edge, _mm_setzero_si128()); - let scale_store_0_ps = _mm_mul_ps(store0_ps, v_weight); - let scale_store_1_ps = _mm_mul_ps(store1_ps, v_weight); + store_0 = _mm_add_epi32(store_0, i16_l0); + store_1 = _mm_add_epi32(store_1, i16_h0); + } + } - let scale_store_0 = _mm_cvtps_epi32(scale_store_0_ps); - let scale_store_1 = _mm_cvtps_epi32(scale_store_1_ps); + { + for y in 0..height { + // preload edge pixels + let next = std::cmp::min(y + half_kernel + 1, height - 1) as usize + * src_stride as usize; + let previous = std::cmp::max(y as i64 - half_kernel as i64, 0) as usize + * src_stride as usize; + let y_dst_shift = dst_stride as usize * y as usize; - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + { + let px = cx as usize; - let set0 = _mm_packus_epi32(scale_store_0, scale_store_1); + let store0_ps = _mm_cvtepi32_ps(store_0); + let store1_ps = _mm_cvtepi32_ps(store_1); - _mm_storeu_si128(ptr as *mut _, set0); - } + let scale_store_0_ps = _mm_mul_ps(store0_ps, v_weight); + let scale_store_1_ps = _mm_mul_ps(store1_ps, v_weight); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = _mm_loadu_si128(s_ptr as *const _); + let scale_store_0 = _mm_cvtps_epi32(scale_store_0_ps); + let scale_store_1 = _mm_cvtps_epi32(scale_store_1_ps); - let i16_l0 = _mm_unpacklo_epi16(edge, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(edge, _mm_setzero_si128()); + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - store_0 = _mm_sub_epi32(store_0, i16_l0); - store_1 = _mm_sub_epi32(store_1, i16_h0); - } + let set0 = _mm_packus_epi32(scale_store_0, scale_store_1); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = _mm_loadu_si128(s_ptr as *const _); + _mm_storeu_si128(ptr as *mut _, set0); + } - let i16_l0 = _mm_unpacklo_epi16(edge, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(edge, _mm_setzero_si128()); + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = _mm_loadu_si128(s_ptr as *const _); - store_0 = _mm_add_epi32(store_0, i16_l0); - store_1 = _mm_add_epi32(store_1, i16_h0); - } - } - } + let i16_l0 = _mm_unpacklo_epi16(edge, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(edge, _mm_setzero_si128()); - cx += 8; - } + store_0 = _mm_sub_epi32(store_0, i16_l0); + store_1 = _mm_sub_epi32(store_1, i16_h0); + } - const TAIL_CN: usize = 1; + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = _mm_loadu_si128(s_ptr as *const _); - for x in cx..end_x { - let px = x as usize; + let i16_l0 = _mm_unpacklo_epi16(edge, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(edge, _mm_setzero_si128()); - let mut store; - { - let s_ptr = src.as_ptr().add(px); - let edge_colors = load_u16_s32_fast::(s_ptr); - store = _mm_mullo_epi32(edge_colors, v_edge_count); - } + store_0 = _mm_add_epi32(store_0, i16_l0); + store_1 = _mm_add_epi32(store_1, i16_h0); + } + } + } - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - let edge_colors = load_u16_s32_fast::(s_ptr); - store = _mm_add_epi32(store, edge_colors); + cx += 8; } - for y in 0..height { - // preload edge pixels - let next = - std::cmp::min(y + half_kernel + 1, height - 1) as usize * src_stride as usize; - let previous = - std::cmp::max(y as i64 - half_kernel as i64, 0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y as usize; + const TAIL_CN: usize = 1; + for x in cx..end_x { let px = x as usize; - let scale_store = _mm_mul_ps_epi32(store, v_weight); - - let ptr = unsafe_dst.slice.as_ptr().add(y_dst_shift + px) as *mut u16; - store_u16_u32::(ptr, scale_store); - - // subtract previous + let mut store; { - let s_ptr = src.as_ptr().add(previous + px); + let s_ptr = src.as_ptr().add(px); let edge_colors = load_u16_s32_fast::(s_ptr); - store = _mm_sub_epi32(store, edge_colors); + store = _mm_mullo_epi32(edge_colors, v_edge_count); } - // add next - { - let s_ptr = src.as_ptr().add(next + px); + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); let edge_colors = load_u16_s32_fast::(s_ptr); store = _mm_add_epi32(store, edge_colors); } + + for y in 0..height { + // preload edge pixels + let next = + std::cmp::min(y + half_kernel + 1, height - 1) as usize * src_stride as usize; + let previous = + std::cmp::max(y as i64 - half_kernel as i64, 0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y as usize; + + let px = x as usize; + + let scale_store = _mm_mul_ps_epi32(store, v_weight); + + let ptr = unsafe_dst.slice.as_ptr().add(y_dst_shift + px) as *mut u16; + store_u16_u32::(ptr, scale_store); + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge_colors = load_u16_s32_fast::(s_ptr); + store = _mm_sub_epi32(store, edge_colors); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge_colors = load_u16_s32_fast::(s_ptr); + store = _mm_add_epi32(store, edge_colors); + } + } } } } diff --git a/src/box_filter/sse/rgba8.rs b/src/box_filter/sse/rgba8.rs index 82483856..65be9fb3 100644 --- a/src/box_filter/sse/rgba8.rs +++ b/src/box_filter/sse/rgba8.rs @@ -53,7 +53,7 @@ pub(crate) fn box_blur_horizontal_pass_sse( } #[target_feature(enable = "sse4.1")] -unsafe fn box_blur_horizontal_pass_impl( +fn box_blur_horizontal_pass_impl( src: &[u8], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -63,350 +63,356 @@ unsafe fn box_blur_horizontal_pass_impl( start_y: u32, end_y: u32, ) { - let kernel_size = radius * 2 + 1; - let edge_count = (kernel_size / 2) + 1; - let v_edge_count = _mm_set1_epi32(edge_count as i32); - - let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); - - let half_kernel = kernel_size / 2; - - let mut yy = start_y; - - while yy + 4 < end_y { - let y = yy; - let y_src_shift = y as usize * src_stride as usize; - let y_dst_shift = y as usize * dst_stride as usize; - - let mut store_0: __m128i; - let mut store_1: __m128i; - let mut store_2: __m128i; - let mut store_3: __m128i; - - unsafe { - let s_ptr_0 = src.as_ptr().add(y_src_shift); - let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize); - let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2); - let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3); - - let edge_colors_0 = load_u8_s32_fast::(s_ptr_0); - let edge_colors_1 = load_u8_s32_fast::(s_ptr_1); - let edge_colors_2 = load_u8_s32_fast::(s_ptr_2); - let edge_colors_3 = load_u8_s32_fast::(s_ptr_3); + unsafe { + let kernel_size = radius * 2 + 1; + let edge_count = (kernel_size / 2) + 1; + let v_edge_count = _mm_set1_epi32(edge_count as i32); - store_0 = _mm_madd_epi16(edge_colors_0, v_edge_count); - store_1 = _mm_madd_epi16(edge_colors_1, v_edge_count); - store_2 = _mm_madd_epi16(edge_colors_2, v_edge_count); - store_3 = _mm_madd_epi16(edge_colors_3, v_edge_count); - } + let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); - unsafe { - let mut jx = 1usize; + let half_kernel = kernel_size / 2; - if CN == 4 { - while jx + 4 < half_kernel as usize && jx + 4 < width as usize { - let px = jx.min(width as usize - 1) * CN; + let mut yy = start_y; - let s_ptr_0 = src.as_ptr().add(y_src_shift + px); - let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + px); - let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2 + px); - let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3 + px); + while yy + 4 <= end_y { + let y = yy; + let y_src_shift = y as usize * src_stride as usize; + let y_dst_shift = y as usize * dst_stride as usize; - let sh1 = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let mut store_0: __m128i; + let mut store_1: __m128i; + let mut store_2: __m128i; + let mut store_3: __m128i; - let mut edge_colors_0 = _mm_loadu_si128(s_ptr_0 as *const __m128i); - let mut edge_colors_1 = _mm_loadu_si128(s_ptr_1 as *const __m128i); - let mut edge_colors_2 = _mm_loadu_si128(s_ptr_2 as *const __m128i); - let mut edge_colors_3 = _mm_loadu_si128(s_ptr_3 as *const __m128i); - - edge_colors_0 = _mm_shuffle_epi8(edge_colors_0, sh1); - edge_colors_1 = _mm_shuffle_epi8(edge_colors_1, sh1); - edge_colors_2 = _mm_shuffle_epi8(edge_colors_2, sh1); - edge_colors_3 = _mm_shuffle_epi8(edge_colors_3, sh1); + { + let s_ptr_0 = src.as_ptr().add(y_src_shift); + let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize); + let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2); + let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3); - let mut m0 = _mm_maddubs_epi16(edge_colors_0, _mm_set1_epi8(1)); - let mut m1 = _mm_maddubs_epi16(edge_colors_1, _mm_set1_epi8(1)); - let mut m2 = _mm_maddubs_epi16(edge_colors_2, _mm_set1_epi8(1)); - let mut m3 = _mm_maddubs_epi16(edge_colors_3, _mm_set1_epi8(1)); + let edge_colors_0 = load_u8_s32_fast::(s_ptr_0); + let edge_colors_1 = load_u8_s32_fast::(s_ptr_1); + let edge_colors_2 = load_u8_s32_fast::(s_ptr_2); + let edge_colors_3 = load_u8_s32_fast::(s_ptr_3); - m0 = _mm_madd_epi16(m0, _mm_set1_epi16(1)); - m1 = _mm_madd_epi16(m1, _mm_set1_epi16(1)); - m2 = _mm_madd_epi16(m2, _mm_set1_epi16(1)); - m3 = _mm_madd_epi16(m3, _mm_set1_epi16(1)); + store_0 = _mm_madd_epi16(edge_colors_0, v_edge_count); + store_1 = _mm_madd_epi16(edge_colors_1, v_edge_count); + store_2 = _mm_madd_epi16(edge_colors_2, v_edge_count); + store_3 = _mm_madd_epi16(edge_colors_3, v_edge_count); + } - store_0 = _mm_add_epi32(store_0, m0); - store_1 = _mm_add_epi32(store_1, m1); - store_2 = _mm_add_epi32(store_2, m2); - store_3 = _mm_add_epi32(store_3, m3); + { + let mut jx = 1usize; - jx += 4; + if CN == 4 { + while jx + 4 < half_kernel as usize && jx + 4 < width as usize { + let px = jx.min(width as usize - 1) * CN; + + let s_ptr_0 = src.as_ptr().add(y_src_shift + px); + let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + px); + let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2 + px); + let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3 + px); + + let sh1 = + _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + + let mut edge_colors_0 = _mm_loadu_si128(s_ptr_0 as *const __m128i); + let mut edge_colors_1 = _mm_loadu_si128(s_ptr_1 as *const __m128i); + let mut edge_colors_2 = _mm_loadu_si128(s_ptr_2 as *const __m128i); + let mut edge_colors_3 = _mm_loadu_si128(s_ptr_3 as *const __m128i); + + edge_colors_0 = _mm_shuffle_epi8(edge_colors_0, sh1); + edge_colors_1 = _mm_shuffle_epi8(edge_colors_1, sh1); + edge_colors_2 = _mm_shuffle_epi8(edge_colors_2, sh1); + edge_colors_3 = _mm_shuffle_epi8(edge_colors_3, sh1); + + let mut m0 = _mm_maddubs_epi16(edge_colors_0, _mm_set1_epi8(1)); + let mut m1 = _mm_maddubs_epi16(edge_colors_1, _mm_set1_epi8(1)); + let mut m2 = _mm_maddubs_epi16(edge_colors_2, _mm_set1_epi8(1)); + let mut m3 = _mm_maddubs_epi16(edge_colors_3, _mm_set1_epi8(1)); + + m0 = _mm_madd_epi16(m0, _mm_set1_epi16(1)); + m1 = _mm_madd_epi16(m1, _mm_set1_epi16(1)); + m2 = _mm_madd_epi16(m2, _mm_set1_epi16(1)); + m3 = _mm_madd_epi16(m3, _mm_set1_epi16(1)); + + store_0 = _mm_add_epi32(store_0, m0); + store_1 = _mm_add_epi32(store_1, m1); + store_2 = _mm_add_epi32(store_2, m2); + store_3 = _mm_add_epi32(store_3, m3); + + jx += 4; + } + + while jx + 2 < half_kernel as usize && jx + 2 < width as usize { + let px = jx.min(width as usize - 1) * CN; + + let s_ptr_0 = src.as_ptr().add(y_src_shift + px); + let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + px); + let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2 + px); + let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3 + px); + + let sh1 = + _mm_setr_epi8(0, 4, -1, -1, 1, 5, -1, -1, 2, 6, -1, -1, 3, 7, -1, -1); + + let mut edge_colors_0 = _mm_loadu_si64(s_ptr_0); + let mut edge_colors_1 = _mm_loadu_si64(s_ptr_1); + let mut edge_colors_2 = _mm_loadu_si64(s_ptr_2); + let mut edge_colors_3 = _mm_loadu_si64(s_ptr_3); + + edge_colors_0 = _mm_shuffle_epi8(edge_colors_0, sh1); + edge_colors_1 = _mm_shuffle_epi8(edge_colors_1, sh1); + edge_colors_2 = _mm_shuffle_epi8(edge_colors_2, sh1); + edge_colors_3 = _mm_shuffle_epi8(edge_colors_3, sh1); + + let mut m0 = _mm_maddubs_epi16(edge_colors_0, _mm_set1_epi8(1)); + let mut m1 = _mm_maddubs_epi16(edge_colors_1, _mm_set1_epi8(1)); + let mut m2 = _mm_maddubs_epi16(edge_colors_2, _mm_set1_epi8(1)); + let mut m3 = _mm_maddubs_epi16(edge_colors_3, _mm_set1_epi8(1)); + + m0 = _mm_madd_epi16(m0, _mm_set1_epi16(1)); + m1 = _mm_madd_epi16(m1, _mm_set1_epi16(1)); + m2 = _mm_madd_epi16(m2, _mm_set1_epi16(1)); + m3 = _mm_madd_epi16(m3, _mm_set1_epi16(1)); + + store_0 = _mm_add_epi32(store_0, m0); + store_1 = _mm_add_epi32(store_1, m1); + store_2 = _mm_add_epi32(store_2, m2); + store_3 = _mm_add_epi32(store_3, m3); + + jx += 2; + } } - while jx + 2 < half_kernel as usize && jx + 2 < width as usize { - let px = jx.min(width as usize - 1) * CN; + for x in jx..=half_kernel as usize { + let px = x.min(width as usize - 1) * CN; let s_ptr_0 = src.as_ptr().add(y_src_shift + px); let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + px); let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2 + px); let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3 + px); - let sh1 = _mm_setr_epi8(0, 4, -1, -1, 1, 5, -1, -1, 2, 6, -1, -1, 3, 7, -1, -1); - - let mut edge_colors_0 = _mm_loadu_si64(s_ptr_0); - let mut edge_colors_1 = _mm_loadu_si64(s_ptr_1); - let mut edge_colors_2 = _mm_loadu_si64(s_ptr_2); - let mut edge_colors_3 = _mm_loadu_si64(s_ptr_3); - - edge_colors_0 = _mm_shuffle_epi8(edge_colors_0, sh1); - edge_colors_1 = _mm_shuffle_epi8(edge_colors_1, sh1); - edge_colors_2 = _mm_shuffle_epi8(edge_colors_2, sh1); - edge_colors_3 = _mm_shuffle_epi8(edge_colors_3, sh1); - - let mut m0 = _mm_maddubs_epi16(edge_colors_0, _mm_set1_epi8(1)); - let mut m1 = _mm_maddubs_epi16(edge_colors_1, _mm_set1_epi8(1)); - let mut m2 = _mm_maddubs_epi16(edge_colors_2, _mm_set1_epi8(1)); - let mut m3 = _mm_maddubs_epi16(edge_colors_3, _mm_set1_epi8(1)); + let edge_colors_0 = load_u8_s32_fast::(s_ptr_0); + let edge_colors_1 = load_u8_s32_fast::(s_ptr_1); + let edge_colors_2 = load_u8_s32_fast::(s_ptr_2); + let edge_colors_3 = load_u8_s32_fast::(s_ptr_3); - m0 = _mm_madd_epi16(m0, _mm_set1_epi16(1)); - m1 = _mm_madd_epi16(m1, _mm_set1_epi16(1)); - m2 = _mm_madd_epi16(m2, _mm_set1_epi16(1)); - m3 = _mm_madd_epi16(m3, _mm_set1_epi16(1)); - - store_0 = _mm_add_epi32(store_0, m0); - store_1 = _mm_add_epi32(store_1, m1); - store_2 = _mm_add_epi32(store_2, m2); - store_3 = _mm_add_epi32(store_3, m3); - - jx += 2; + store_0 = _mm_add_epi32(store_0, edge_colors_0); + store_1 = _mm_add_epi32(store_1, edge_colors_1); + store_2 = _mm_add_epi32(store_2, edge_colors_2); + store_3 = _mm_add_epi32(store_3, edge_colors_3); } } - for x in jx..=half_kernel as usize { - let px = x.min(width as usize - 1) * CN; - - let s_ptr_0 = src.as_ptr().add(y_src_shift + px); - let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + px); - let s_ptr_2 = src.as_ptr().add(y_src_shift + src_stride as usize * 2 + px); - let s_ptr_3 = src.as_ptr().add(y_src_shift + src_stride as usize * 3 + px); - - let edge_colors_0 = load_u8_s32_fast::(s_ptr_0); - let edge_colors_1 = load_u8_s32_fast::(s_ptr_1); - let edge_colors_2 = load_u8_s32_fast::(s_ptr_2); - let edge_colors_3 = load_u8_s32_fast::(s_ptr_3); - - store_0 = _mm_add_epi32(store_0, edge_colors_0); - store_1 = _mm_add_epi32(store_1, edge_colors_1); - store_2 = _mm_add_epi32(store_2, edge_colors_2); - store_3 = _mm_add_epi32(store_3, edge_colors_3); - } - } - - for x in 0..width { - let px = x as usize * CN; - - unsafe { - let scale_store_ps0 = _mm_cvtepi32_ps(store_0); - let scale_store_ps1 = _mm_cvtepi32_ps(store_1); - let scale_store_ps2 = _mm_cvtepi32_ps(store_2); - let scale_store_ps3 = _mm_cvtepi32_ps(store_3); - - let r0 = _mm_mul_ps(scale_store_ps0, v_weight); - let r1 = _mm_mul_ps(scale_store_ps1, v_weight); - let r2 = _mm_mul_ps(scale_store_ps2, v_weight); - let r3 = _mm_mul_ps(scale_store_ps3, v_weight); - - let scale_store0 = _mm_cvtps_epi32(r0); - let scale_store1 = _mm_cvtps_epi32(r1); - let scale_store2 = _mm_cvtps_epi32(r2); - let scale_store3 = _mm_cvtps_epi32(r3); - - let px_160 = _mm_packus_epi32(scale_store0, _mm_setzero_si128()); - let px_161 = _mm_packus_epi32(scale_store1, _mm_setzero_si128()); - let px_162 = _mm_packus_epi32(scale_store2, _mm_setzero_si128()); - let px_163 = _mm_packus_epi32(scale_store3, _mm_setzero_si128()); - - let px_80 = _mm_packus_epi16(px_160, _mm_setzero_si128()); - let px_81 = _mm_packus_epi16(px_161, _mm_setzero_si128()); - let px_82 = _mm_packus_epi16(px_162, _mm_setzero_si128()); - let px_83 = _mm_packus_epi16(px_163, _mm_setzero_si128()); - - let bytes_offset_0 = y_dst_shift + px; - let bytes_offset_1 = y_dst_shift + dst_stride as usize + px; - let bytes_offset_2 = y_dst_shift + dst_stride as usize * 2 + px; - let bytes_offset_3 = y_dst_shift + dst_stride as usize * 3 + px; - if CN == 4 { - let dst_ptr_0 = unsafe_dst.slice.as_ptr().add(bytes_offset_0) as *mut u8; - _mm_storeu_si32(dst_ptr_0, px_80); + for x in 0..width { + let px = x as usize * CN; - let dst_ptr_1 = unsafe_dst.slice.as_ptr().add(bytes_offset_1) as *mut u8; - _mm_storeu_si32(dst_ptr_1, px_81); - - let dst_ptr_2 = unsafe_dst.slice.as_ptr().add(bytes_offset_2) as *mut u8; - _mm_storeu_si32(dst_ptr_2, px_82); - - let dst_ptr_3 = unsafe_dst.slice.as_ptr().add(bytes_offset_3) as *mut u8; - _mm_storeu_si32(dst_ptr_3, px_83); - } else { - let dst_ptr_0 = unsafe_dst.slice.as_ptr().add(bytes_offset_0) as *mut u8; - write_u8::(dst_ptr_0, px_80); - - let dst_ptr_1 = unsafe_dst.slice.as_ptr().add(bytes_offset_1) as *mut u8; - write_u8::(dst_ptr_1, px_81); - - let dst_ptr_2 = unsafe_dst.slice.as_ptr().add(bytes_offset_2) as *mut u8; - write_u8::(dst_ptr_2, px_82); - - let dst_ptr_3 = unsafe_dst.slice.as_ptr().add(bytes_offset_3) as *mut u8; - write_u8::(dst_ptr_3, px_83); + { + let scale_store_ps0 = _mm_cvtepi32_ps(store_0); + let scale_store_ps1 = _mm_cvtepi32_ps(store_1); + let scale_store_ps2 = _mm_cvtepi32_ps(store_2); + let scale_store_ps3 = _mm_cvtepi32_ps(store_3); + + let r0 = _mm_mul_ps(scale_store_ps0, v_weight); + let r1 = _mm_mul_ps(scale_store_ps1, v_weight); + let r2 = _mm_mul_ps(scale_store_ps2, v_weight); + let r3 = _mm_mul_ps(scale_store_ps3, v_weight); + + let scale_store0 = _mm_cvtps_epi32(r0); + let scale_store1 = _mm_cvtps_epi32(r1); + let scale_store2 = _mm_cvtps_epi32(r2); + let scale_store3 = _mm_cvtps_epi32(r3); + + let px_160 = _mm_packus_epi32(scale_store0, _mm_setzero_si128()); + let px_161 = _mm_packus_epi32(scale_store1, _mm_setzero_si128()); + let px_162 = _mm_packus_epi32(scale_store2, _mm_setzero_si128()); + let px_163 = _mm_packus_epi32(scale_store3, _mm_setzero_si128()); + + let px_80 = _mm_packus_epi16(px_160, _mm_setzero_si128()); + let px_81 = _mm_packus_epi16(px_161, _mm_setzero_si128()); + let px_82 = _mm_packus_epi16(px_162, _mm_setzero_si128()); + let px_83 = _mm_packus_epi16(px_163, _mm_setzero_si128()); + + let bytes_offset_0 = y_dst_shift + px; + let bytes_offset_1 = y_dst_shift + dst_stride as usize + px; + let bytes_offset_2 = y_dst_shift + dst_stride as usize * 2 + px; + let bytes_offset_3 = y_dst_shift + dst_stride as usize * 3 + px; + if CN == 4 { + let dst_ptr_0 = unsafe_dst.slice.as_ptr().add(bytes_offset_0) as *mut u8; + _mm_storeu_si32(dst_ptr_0, px_80); + + let dst_ptr_1 = unsafe_dst.slice.as_ptr().add(bytes_offset_1) as *mut u8; + _mm_storeu_si32(dst_ptr_1, px_81); + + let dst_ptr_2 = unsafe_dst.slice.as_ptr().add(bytes_offset_2) as *mut u8; + _mm_storeu_si32(dst_ptr_2, px_82); + + let dst_ptr_3 = unsafe_dst.slice.as_ptr().add(bytes_offset_3) as *mut u8; + _mm_storeu_si32(dst_ptr_3, px_83); + } else { + let dst_ptr_0 = unsafe_dst.slice.as_ptr().add(bytes_offset_0) as *mut u8; + write_u8::(dst_ptr_0, px_80); + + let dst_ptr_1 = unsafe_dst.slice.as_ptr().add(bytes_offset_1) as *mut u8; + write_u8::(dst_ptr_1, px_81); + + let dst_ptr_2 = unsafe_dst.slice.as_ptr().add(bytes_offset_2) as *mut u8; + write_u8::(dst_ptr_2, px_82); + + let dst_ptr_3 = unsafe_dst.slice.as_ptr().add(bytes_offset_3) as *mut u8; + write_u8::(dst_ptr_3, px_83); + } } - } - - // subtract previous - unsafe { - let previous_x = (x as i64 - half_kernel as i64).max(0) as usize; - let previous = previous_x * CN; - - let s_ptr_0 = src.as_ptr().add(y_src_shift + previous); - let s_ptr_1 = src - .as_ptr() - .add(y_src_shift + src_stride as usize + previous); - let s_ptr_2 = src - .as_ptr() - .add(y_src_shift + src_stride as usize * 2 + previous); - let s_ptr_3 = src - .as_ptr() - .add(y_src_shift + src_stride as usize * 3 + previous); - let edge_colors_0 = load_u8_s32_fast::(s_ptr_0); - let edge_colors_1 = load_u8_s32_fast::(s_ptr_1); - let edge_colors_2 = load_u8_s32_fast::(s_ptr_2); - let edge_colors_3 = load_u8_s32_fast::(s_ptr_3); + // subtract previous + { + let previous_x = (x as i64 - half_kernel as i64).max(0) as usize; + let previous = previous_x * CN; + + let s_ptr_0 = src.as_ptr().add(y_src_shift + previous); + let s_ptr_1 = src + .as_ptr() + .add(y_src_shift + src_stride as usize + previous); + let s_ptr_2 = src + .as_ptr() + .add(y_src_shift + src_stride as usize * 2 + previous); + let s_ptr_3 = src + .as_ptr() + .add(y_src_shift + src_stride as usize * 3 + previous); + + let edge_colors_0 = load_u8_s32_fast::(s_ptr_0); + let edge_colors_1 = load_u8_s32_fast::(s_ptr_1); + let edge_colors_2 = load_u8_s32_fast::(s_ptr_2); + let edge_colors_3 = load_u8_s32_fast::(s_ptr_3); + + store_0 = _mm_sub_epi32(store_0, edge_colors_0); + store_1 = _mm_sub_epi32(store_1, edge_colors_1); + store_2 = _mm_sub_epi32(store_2, edge_colors_2); + store_3 = _mm_sub_epi32(store_3, edge_colors_3); + } - store_0 = _mm_sub_epi32(store_0, edge_colors_0); - store_1 = _mm_sub_epi32(store_1, edge_colors_1); - store_2 = _mm_sub_epi32(store_2, edge_colors_2); - store_3 = _mm_sub_epi32(store_3, edge_colors_3); + // add next + { + let next_x = (x + half_kernel + 1).min(width - 1) as usize; + + let next = next_x * CN; + + let s_ptr_0 = src.as_ptr().add(y_src_shift + next); + let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + next); + let s_ptr_2 = src + .as_ptr() + .add(y_src_shift + src_stride as usize * 2 + next); + let s_ptr_3 = src + .as_ptr() + .add(y_src_shift + src_stride as usize * 3 + next); + + let edge_colors_0 = load_u8_s32_fast::(s_ptr_0); + let edge_colors_1 = load_u8_s32_fast::(s_ptr_1); + let edge_colors_2 = load_u8_s32_fast::(s_ptr_2); + let edge_colors_3 = load_u8_s32_fast::(s_ptr_3); + + store_0 = _mm_add_epi32(store_0, edge_colors_0); + store_1 = _mm_add_epi32(store_1, edge_colors_1); + store_2 = _mm_add_epi32(store_2, edge_colors_2); + store_3 = _mm_add_epi32(store_3, edge_colors_3); + } } - // add next - unsafe { - let next_x = (x + half_kernel + 1).min(width - 1) as usize; - - let next = next_x * CN; - - let s_ptr_0 = src.as_ptr().add(y_src_shift + next); - let s_ptr_1 = src.as_ptr().add(y_src_shift + src_stride as usize + next); - let s_ptr_2 = src - .as_ptr() - .add(y_src_shift + src_stride as usize * 2 + next); - let s_ptr_3 = src - .as_ptr() - .add(y_src_shift + src_stride as usize * 3 + next); - - let edge_colors_0 = load_u8_s32_fast::(s_ptr_0); - let edge_colors_1 = load_u8_s32_fast::(s_ptr_1); - let edge_colors_2 = load_u8_s32_fast::(s_ptr_2); - let edge_colors_3 = load_u8_s32_fast::(s_ptr_3); - - store_0 = _mm_add_epi32(store_0, edge_colors_0); - store_1 = _mm_add_epi32(store_1, edge_colors_1); - store_2 = _mm_add_epi32(store_2, edge_colors_2); - store_3 = _mm_add_epi32(store_3, edge_colors_3); - } + yy += 4; } - yy += 4; - } + for y in yy..end_y { + let y_src_shift = y as usize * src_stride as usize; + let y_dst_shift = y as usize * dst_stride as usize; - for y in yy..end_y { - let y_src_shift = y as usize * src_stride as usize; - let y_dst_shift = y as usize * dst_stride as usize; + let mut store; - let mut store; + { + let s_ptr = src.as_ptr().add(y_src_shift); + let edge_colors = load_u8_s32_fast::(s_ptr); + store = _mm_madd_epi16(edge_colors, v_edge_count); + } - unsafe { - let s_ptr = src.as_ptr().add(y_src_shift); - let edge_colors = load_u8_s32_fast::(s_ptr); - store = _mm_madd_epi16(edge_colors, v_edge_count); - } + { + let mut jx = 1usize; - unsafe { - let mut jx = 1usize; + if CN == 4 { + while jx + 4 < half_kernel as usize && jx + 4 < width as usize { + let px = jx.min(width as usize - 1) * CN; - if CN == 4 { - while jx + 4 < half_kernel as usize && jx + 4 < width as usize { - let px = jx.min(width as usize - 1) * CN; + let s_ptr_0 = src.as_ptr().add(y_src_shift + px); - let s_ptr_0 = src.as_ptr().add(y_src_shift + px); + let sh1 = + _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); - let sh1 = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + let mut edge_colors_0 = _mm_loadu_si128(s_ptr_0 as *const __m128i); - let mut edge_colors_0 = _mm_loadu_si128(s_ptr_0 as *const __m128i); + edge_colors_0 = _mm_shuffle_epi8(edge_colors_0, sh1); - edge_colors_0 = _mm_shuffle_epi8(edge_colors_0, sh1); + let mut m0 = _mm_maddubs_epi16(edge_colors_0, _mm_set1_epi8(1)); + m0 = _mm_madd_epi16(m0, _mm_set1_epi16(1)); - let mut m0 = _mm_maddubs_epi16(edge_colors_0, _mm_set1_epi8(1)); - m0 = _mm_madd_epi16(m0, _mm_set1_epi16(1)); + store = _mm_add_epi32(store, m0); - store = _mm_add_epi32(store, m0); + jx += 4; + } - jx += 4; - } + while jx + 2 < half_kernel as usize && jx + 2 < width as usize { + let px = jx.min(width as usize - 1) * CN; - while jx + 2 < half_kernel as usize && jx + 2 < width as usize { - let px = jx.min(width as usize - 1) * CN; + let s_ptr_0 = src.as_ptr().add(y_src_shift + px); - let s_ptr_0 = src.as_ptr().add(y_src_shift + px); + let sh1 = + _mm_setr_epi8(0, 4, -1, -1, 1, 5, -1, -1, 2, 6, -1, -1, 3, 7, -1, -1); - let sh1 = _mm_setr_epi8(0, 4, -1, -1, 1, 5, -1, -1, 2, 6, -1, -1, 3, 7, -1, -1); + let mut edge_colors_0 = _mm_loadu_si64(s_ptr_0); + edge_colors_0 = _mm_shuffle_epi8(edge_colors_0, sh1); - let mut edge_colors_0 = _mm_loadu_si64(s_ptr_0); - edge_colors_0 = _mm_shuffle_epi8(edge_colors_0, sh1); + let mut m0 = _mm_maddubs_epi16(edge_colors_0, _mm_set1_epi8(1)); + m0 = _mm_madd_epi16(m0, _mm_set1_epi16(1)); + store = _mm_add_epi32(store, m0); - let mut m0 = _mm_maddubs_epi16(edge_colors_0, _mm_set1_epi8(1)); - m0 = _mm_madd_epi16(m0, _mm_set1_epi16(1)); - store = _mm_add_epi32(store, m0); - - jx += 2; + jx += 2; + } } - } - for x in jx..=half_kernel as usize { - let px = x.min(width as usize - 1) * CN; - let s_ptr = src.as_ptr().add(y_src_shift + px); - let edge_colors = load_u8_s32_fast::(s_ptr); - store = _mm_add_epi32(store, edge_colors); + for x in jx..=half_kernel as usize { + let px = x.min(width as usize - 1) * CN; + let s_ptr = src.as_ptr().add(y_src_shift + px); + let edge_colors = load_u8_s32_fast::(s_ptr); + store = _mm_add_epi32(store, edge_colors); + } } - } - for x in 0..width { - let px = x as usize * CN; + for x in 0..width { + let px = x as usize * CN; - unsafe { - let r0 = _mm_mul_ps(_mm_cvtepi32_ps(store), v_weight); - let bytes_offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.as_ptr().add(bytes_offset) as *mut u8; - store_u8_u32::(ptr, _mm_cvtps_epi32(r0)); - } + { + let r0 = _mm_mul_ps(_mm_cvtepi32_ps(store), v_weight); + let bytes_offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.as_ptr().add(bytes_offset) as *mut u8; + store_u8_u32::(ptr, _mm_cvtps_epi32(r0)); + } - // subtract previous - unsafe { - let previous_x = (x as i64 - half_kernel as i64).max(0) as usize; - let previous = previous_x * CN; - let s_ptr = src.as_ptr().add(y_src_shift + previous); - let edge_colors = load_u8_s32_fast::(s_ptr); - store = _mm_sub_epi32(store, edge_colors); - } + // subtract previous + { + let previous_x = (x as i64 - half_kernel as i64).max(0) as usize; + let previous = previous_x * CN; + let s_ptr = src.as_ptr().add(y_src_shift + previous); + let edge_colors = load_u8_s32_fast::(s_ptr); + store = _mm_sub_epi32(store, edge_colors); + } - // add next - unsafe { - let next_x = (x + half_kernel + 1).min(width - 1) as usize; + // add next + { + let next_x = (x + half_kernel + 1).min(width - 1) as usize; - let next = next_x * CN; + let next = next_x * CN; - let s_ptr = src.as_ptr().add(y_src_shift + next); - let edge_colors = load_u8_s32_fast::(s_ptr); - store = _mm_add_epi32(store, edge_colors); + let s_ptr = src.as_ptr().add(y_src_shift + next); + let edge_colors = load_u8_s32_fast::(s_ptr); + store = _mm_add_epi32(store, edge_colors); + } } } } @@ -431,7 +437,7 @@ pub(crate) fn box_blur_vertical_pass_sse( } #[target_feature(enable = "sse4.1")] -unsafe fn box_blur_vertical_pass_sse_impl( +fn box_blur_vertical_pass_sse_impl( src: &[u8], src_stride: u32, unsafe_dst: &UnsafeSlice, @@ -442,43 +448,43 @@ unsafe fn box_blur_vertical_pass_sse_impl( start_x: u32, end_x: u32, ) { - let kernel_size = radius * 2 + 1; - let edge_count = (kernel_size / 2) + 1; - let v_edge_count = _mm_set1_epi32(edge_count as i32); + unsafe { + let kernel_size = radius * 2 + 1; + let edge_count = (kernel_size / 2) + 1; + let v_edge_count = _mm_set1_epi32(edge_count as i32); - let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); + let v_weight = _mm_set1_ps(1f32 / (radius * 2 + 1) as f32); - let half_kernel = kernel_size / 2; + let half_kernel = kernel_size / 2; - let mut cx = start_x; + let mut cx = start_x; - while cx + 16 < end_x { - let px = cx as usize; + while cx + 16 <= end_x { + let px = cx as usize; - let mut store_0: __m128i; - let mut store_1: __m128i; - let mut store_2: __m128i; - let mut store_3: __m128i; + let mut store_0: __m128i; + let mut store_1: __m128i; + let mut store_2: __m128i; + let mut store_3: __m128i; - unsafe { - let s_ptr = src.as_ptr().add(px); - let edge = _mm_loadu_si128(s_ptr as *const _); - let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); - let hi0 = _mm_unpackhi_epi8(edge, _mm_setzero_si128()); + { + let s_ptr = src.as_ptr().add(px); + let edge = _mm_loadu_si128(s_ptr as *const _); + let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); + let hi0 = _mm_unpackhi_epi8(edge, _mm_setzero_si128()); - let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - let i16_l1 = _mm_unpacklo_epi16(hi0, _mm_setzero_si128()); - let i16_h1 = _mm_unpackhi_epi16(hi0, _mm_setzero_si128()); + let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); + let i16_l1 = _mm_unpacklo_epi16(hi0, _mm_setzero_si128()); + let i16_h1 = _mm_unpackhi_epi16(hi0, _mm_setzero_si128()); - store_0 = _mm_madd_epi16(i16_l0, v_edge_count); - store_1 = _mm_madd_epi16(i16_h0, v_edge_count); + store_0 = _mm_madd_epi16(i16_l0, v_edge_count); + store_1 = _mm_madd_epi16(i16_h0, v_edge_count); - store_2 = _mm_madd_epi16(i16_l1, v_edge_count); - store_3 = _mm_madd_epi16(i16_h1, v_edge_count); - } + store_2 = _mm_madd_epi16(i16_l1, v_edge_count); + store_3 = _mm_madd_epi16(i16_h1, v_edge_count); + } - unsafe { for y in 1..=half_kernel { let y_src_shift = y.min(height - 1) as usize * src_stride as usize; let s_ptr = src.as_ptr().add(y_src_shift + px); @@ -498,9 +504,7 @@ unsafe fn box_blur_vertical_pass_sse_impl( store_2 = _mm_add_epi32(store_2, i16_l1); store_3 = _mm_add_epi32(store_3, i16_h1); } - } - unsafe { for y in 0..height { // preload edge pixels let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; @@ -574,148 +578,149 @@ unsafe fn box_blur_vertical_pass_sse_impl( store_3 = _mm_add_epi32(store_3, i16_h1); } } - } - - cx += 16; - } - - while cx + 8 < end_x { - let px = cx as usize; - - let mut store_0: __m128i; - let mut store_1: __m128i; - unsafe { - let s_ptr = src.as_ptr().add(px); - let edge = _mm_loadu_si64(s_ptr as *const _); - let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); - - let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - - store_0 = _mm_madd_epi16(i16_l0, v_edge_count); - store_1 = _mm_madd_epi16(i16_h0, v_edge_count); + cx += 16; } - unsafe { - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); + while cx + 8 <= end_x { + let px = cx as usize; + + let mut store_0: __m128i; + let mut store_1: __m128i; + { + let s_ptr = src.as_ptr().add(px); let edge = _mm_loadu_si64(s_ptr as *const _); let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - store_0 = _mm_add_epi32(store_0, i16_l0); - store_1 = _mm_add_epi32(store_1, i16_h0); + store_0 = _mm_madd_epi16(i16_l0, v_edge_count); + store_1 = _mm_madd_epi16(i16_h0, v_edge_count); } - } - unsafe { - for y in 0..height { - // preload edge pixels - let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; - let previous = - (y as i64 - half_kernel as i64).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y as usize; + { + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); - let px = cx as usize; + let edge = _mm_loadu_si64(s_ptr as *const _); + let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); - let store0_ps = _mm_cvtepi32_ps(store_0); - let store1_ps = _mm_cvtepi32_ps(store_1); + let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - let scale_store_0_ps = _mm_mul_ps(store0_ps, v_weight); - let scale_store_1_ps = _mm_mul_ps(store1_ps, v_weight); + store_0 = _mm_add_epi32(store_0, i16_l0); + store_1 = _mm_add_epi32(store_1, i16_h0); + } + } - let scale_store_0 = _mm_cvtps_epi32(scale_store_0_ps); - let scale_store_1 = _mm_cvtps_epi32(scale_store_1_ps); + { + for y in 0..height { + // preload edge pixels + let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; + let previous = + (y as i64 - half_kernel as i64).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y as usize; - let offset = y_dst_shift + px; - let ptr = unsafe_dst.slice.get_unchecked(offset).get(); + let px = cx as usize; - let set0 = _mm_packus_epi32(scale_store_0, scale_store_1); + let store0_ps = _mm_cvtepi32_ps(store_0); + let store1_ps = _mm_cvtepi32_ps(store_1); - let full_set = _mm_packus_epi16(set0, _mm_setzero_si128()); - _mm_storeu_si64(ptr as *mut _, full_set); + let scale_store_0_ps = _mm_mul_ps(store0_ps, v_weight); + let scale_store_1_ps = _mm_mul_ps(store1_ps, v_weight); - // subtract previous - { - let s_ptr = src.as_ptr().add(previous + px); - let edge = _mm_loadu_si64(s_ptr as *const _); - let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); + let scale_store_0 = _mm_cvtps_epi32(scale_store_0_ps); + let scale_store_1 = _mm_cvtps_epi32(scale_store_1_ps); - let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); + let offset = y_dst_shift + px; + let ptr = unsafe_dst.slice.get_unchecked(offset).get(); - store_0 = _mm_sub_epi32(store_0, i16_l0); - store_1 = _mm_sub_epi32(store_1, i16_h0); - } + let set0 = _mm_packus_epi32(scale_store_0, scale_store_1); - // add next - { - let s_ptr = src.as_ptr().add(next + px); - let edge = _mm_loadu_si64(s_ptr as *const _); - let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); + let full_set = _mm_packus_epi16(set0, _mm_setzero_si128()); + _mm_storeu_si64(ptr as *mut _, full_set); - let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge = _mm_loadu_si64(s_ptr as *const _); + let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); - store_0 = _mm_add_epi32(store_0, i16_l0); - store_1 = _mm_add_epi32(store_1, i16_h0); - } - } - } + let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - cx += 8; - } + store_0 = _mm_sub_epi32(store_0, i16_l0); + store_1 = _mm_sub_epi32(store_1, i16_h0); + } - const TAIL_CN: usize = 1; + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge = _mm_loadu_si64(s_ptr as *const _); + let lo0 = _mm_unpacklo_epi8(edge, _mm_setzero_si128()); - for x in cx..end_x { - let px = x as usize; + let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - let mut store; - { - let s_ptr = src.as_ptr().add(px); - let edge_colors = load_u8_s32_fast::(s_ptr); - store = _mm_madd_epi16(edge_colors, v_edge_count); - } + store_0 = _mm_add_epi32(store_0, i16_l0); + store_1 = _mm_add_epi32(store_1, i16_h0); + } + } + } - for y in 1..=half_kernel { - let y_src_shift = y.min(height - 1) as usize * src_stride as usize; - let s_ptr = src.as_ptr().add(y_src_shift + px); - let edge_colors = load_u8_s32_fast::(s_ptr); - store = _mm_add_epi32(store, edge_colors); + cx += 8; } - for y in 0..height { - // preload edge pixels - let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; - let previous = (y as i64 - half_kernel as i64).max(0) as usize * src_stride as usize; - let y_dst_shift = dst_stride as usize * y as usize; + const TAIL_CN: usize = 1; + for x in cx..end_x { let px = x as usize; - let scale_store = _mm_mul_ps_epi32(store, v_weight); - - let ptr = unsafe_dst.slice.as_ptr().add(y_dst_shift + px) as *mut u8; - store_u8_u32::(ptr, scale_store); - - // subtract previous + let mut store; { - let s_ptr = src.as_ptr().add(previous + px); + let s_ptr = src.as_ptr().add(px); let edge_colors = load_u8_s32_fast::(s_ptr); - store = _mm_sub_epi32(store, edge_colors); + store = _mm_madd_epi16(edge_colors, v_edge_count); } - // add next - { - let s_ptr = src.as_ptr().add(next + px); + for y in 1..=half_kernel { + let y_src_shift = y.min(height - 1) as usize * src_stride as usize; + let s_ptr = src.as_ptr().add(y_src_shift + px); let edge_colors = load_u8_s32_fast::(s_ptr); store = _mm_add_epi32(store, edge_colors); } + + for y in 0..height { + // preload edge pixels + let next = (y + half_kernel + 1).min(height - 1) as usize * src_stride as usize; + let previous = + (y as i64 - half_kernel as i64).max(0) as usize * src_stride as usize; + let y_dst_shift = dst_stride as usize * y as usize; + + let px = x as usize; + + let scale_store = _mm_mul_ps_epi32(store, v_weight); + + let ptr = unsafe_dst.slice.as_ptr().add(y_dst_shift + px) as *mut u8; + store_u8_u32::(ptr, scale_store); + + // subtract previous + { + let s_ptr = src.as_ptr().add(previous + px); + let edge_colors = load_u8_s32_fast::(s_ptr); + store = _mm_sub_epi32(store, edge_colors); + } + + // add next + { + let s_ptr = src.as_ptr().add(next + px); + let edge_colors = load_u8_s32_fast::(s_ptr); + store = _mm_add_epi32(store, edge_colors); + } + } } } } diff --git a/src/box_filter/sse/rgba_f32.rs b/src/box_filter/sse/rgba_f32.rs index 609c482a..2525393b 100644 --- a/src/box_filter/sse/rgba_f32.rs +++ b/src/box_filter/sse/rgba_f32.rs @@ -79,7 +79,7 @@ impl HorizontalExecutionUnit { let mut yy = start_y; - while yy + 4 < end_y { + while yy + 4 <= end_y { let y = yy; let y_src_shift = y as usize * src_stride as usize; let y_dst_shift = y as usize * dst_stride as usize; diff --git a/src/box_filter/sse/vsum16.rs b/src/box_filter/sse/vsum16.rs index e742dac2..92839e07 100644 --- a/src/box_filter/sse/vsum16.rs +++ b/src/box_filter/sse/vsum16.rs @@ -53,8 +53,6 @@ unsafe fn sse_ring_vertical_row_summ_impl16( let weight = 1. / (radius as f32 * 2. + 1.); let v_weight = _mm_set1_ps(weight); - let chunks = previous_row.chunks_exact(16).len(); - for (((src_next, src_previous), buffer), dst) in next_row .chunks_exact(16) .zip(previous_row.chunks_exact(16)) @@ -119,12 +117,16 @@ unsafe fn sse_ring_vertical_row_summ_impl16( } } + let next_row = next_row.chunks_exact(16).remainder(); + let previous_row = previous_row.chunks_exact(16).remainder(); + let working_row = working_row.chunks_exact_mut(16).into_remainder(); + let dst = dst.chunks_exact_mut(16).into_remainder(); + for (((src_next, src_previous), buffer), dst) in next_row .iter() .zip(previous_row.iter()) .zip(working_row.iter_mut()) .zip(dst.iter_mut()) - .skip(chunks * 16) { let mut weight0 = *buffer; diff --git a/src/box_filter/sse/vsum8.rs b/src/box_filter/sse/vsum8.rs index 488e24a7..cf22c7c7 100644 --- a/src/box_filter/sse/vsum8.rs +++ b/src/box_filter/sse/vsum8.rs @@ -53,8 +53,6 @@ unsafe fn sse_ring_vertical_row_summ_impl( let weight = 1. / (radius as f32 * 2. + 1.); let v_weight = _mm_set1_ps(weight); - let chunks = previous_row.chunks_exact(16).len(); - for (((src_next, src_previous), buffer), dst) in next_row .chunks_exact(16) .zip(previous_row.chunks_exact(16)) @@ -122,12 +120,16 @@ unsafe fn sse_ring_vertical_row_summ_impl( } } + let next_row = next_row.chunks_exact(16).remainder(); + let previous_row = previous_row.chunks_exact(16).remainder(); + let working_row = working_row.chunks_exact_mut(16).into_remainder(); + let dst = dst.chunks_exact_mut(16).into_remainder(); + for (((src_next, src_previous), buffer), dst) in next_row .iter() .zip(previous_row.iter()) .zip(working_row.iter_mut()) .zip(dst.iter_mut()) - .skip(chunks * 16) { let mut weight0 = *buffer; diff --git a/src/box_filter/sse/vsum_f32.rs b/src/box_filter/sse/vsum_f32.rs index 6799e0f7..d84f1826 100644 --- a/src/box_filter/sse/vsum_f32.rs +++ b/src/box_filter/sse/vsum_f32.rs @@ -53,8 +53,6 @@ unsafe fn sse_ring_vertical_row_summ_impl_f32( let weight = 1. / (radius as f32 * 2. + 1.); let v_weight = _mm_set1_ps(weight); - let chunks = previous_row.chunks_exact(16).len(); - for (((src_next, src_previous), buffer), dst) in next_row .chunks_exact(16) .zip(previous_row.chunks_exact(16)) @@ -113,12 +111,16 @@ unsafe fn sse_ring_vertical_row_summ_impl_f32( } } + let next_row = next_row.chunks_exact(16).remainder(); + let previous_row = previous_row.chunks_exact(16).remainder(); + let working_row = working_row.chunks_exact_mut(16).into_remainder(); + let dst = dst.chunks_exact_mut(16).into_remainder(); + for (((src_next, src_previous), buffer), dst) in next_row .iter() .zip(previous_row.iter()) .zip(working_row.iter_mut()) .zip(dst.iter_mut()) - .skip(chunks * 16) { let mut weight0 = *buffer; diff --git a/src/fast_bilateral_filter.rs b/src/fast_bilateral_filter.rs index 704728d7..124fe35e 100644 --- a/src/fast_bilateral_filter.rs +++ b/src/fast_bilateral_filter.rs @@ -29,11 +29,11 @@ use crate::gaussian::gaussian_kernel_1d; use crate::unsafe_slice::UnsafeSlice; use crate::{BlurError, BlurImage, BlurImageMut, FastBlurChannels, ThreadingPolicy}; -use num_traits::real::Real; use num_traits::AsPrimitive; +use num_traits::real::Real; +use rayon::ThreadPool; use rayon::iter::{IndexedParallelIterator, ParallelIterator}; use rayon::prelude::{IntoParallelIterator, ParallelSliceMut}; -use rayon::ThreadPool; use std::fmt::{Debug, Display}; use std::ops::{Add, AddAssign, Div, Index, Mul, MulAssign, Sub}; diff --git a/src/fast_bilateral_image.rs b/src/fast_bilateral_image.rs index 86e3b86b..a7434a73 100644 --- a/src/fast_bilateral_image.rs +++ b/src/fast_bilateral_image.rs @@ -28,8 +28,8 @@ */ use crate::fast_bilateral_filter::fast_bilateral_filter_gray_alpha_impl; use crate::{ - fast_bilateral_filter, fast_bilateral_filter_f32, fast_bilateral_filter_u16, BlurImage, - BlurImageMut, BufferStore, FastBlurChannels, ThreadingPolicy, + BlurImage, BlurImageMut, BufferStore, FastBlurChannels, ThreadingPolicy, fast_bilateral_filter, + fast_bilateral_filter_f32, fast_bilateral_filter_u16, }; use image::{ DynamicImage, GrayAlphaImage, GrayImage, ImageBuffer, Luma, LumaA, Rgb, Rgb32FImage, RgbImage, diff --git a/src/fast_gaussian.rs b/src/fast_gaussian.rs index 1267ee99..21637a09 100644 --- a/src/fast_gaussian.rs +++ b/src/fast_gaussian.rs @@ -204,10 +204,10 @@ fn fg_vertical_pass( f64: PrimitiveCast, { let zero_j: J = PrimitiveCast::cast_(0i32); - let mut buffer_r = Box::new([zero_j; 1024]); - let mut buffer_g = Box::new([zero_j; 1024]); - let mut buffer_b = Box::new([zero_j; 1024]); - let mut buffer_a = Box::new([zero_j; 1024]); + let mut buffer_r = [zero_j; 1024]; + let mut buffer_g = [zero_j; 1024]; + let mut buffer_b = [zero_j; 1024]; + let mut buffer_a = [zero_j; 1024]; let radius_64 = radius as i64; let height_wide = height as i64; let initial: J = PrimitiveCast::cast_(T::get_initial(radius as usize)); @@ -324,10 +324,10 @@ fn fg_horizontal_pass( f64: PrimitiveCast, i64: PrimitiveCast, { - let mut buffer_r = Box::new([0i32.cast_(); 1024]); - let mut buffer_g = Box::new([0i32.cast_(); 1024]); - let mut buffer_b = Box::new([0i32.cast_(); 1024]); - let mut buffer_a = Box::new([0i32.cast_(); 1024]); + let mut buffer_r = [0i32.cast_(); 1024]; + let mut buffer_g = [0i32.cast_(); 1024]; + let mut buffer_b = [0i32.cast_(); 1024]; + let mut buffer_a = [0i32.cast_(); 1024]; let radius_64 = radius as i64; let width_wide = width as i64; let weight: M = (1f64 / (radius as f64 * radius as f64)).cast_(); @@ -517,12 +517,12 @@ impl FastGaussianDispatchProvider for u8 { #[cfg(all(target_arch = "aarch64", feature = "neon"))] { if BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_horizontal = fg_horizontal_pass_neon_u8::; + _dispatcher_horizontal = fg_horizontal_pass_neon_u8::; #[cfg(feature = "rdm")] { if std::arch::is_aarch64_feature_detected!("rdm") { use crate::neon::fg_horizontal_pass_neon_u8_rdm; - _dispatcher_horizontal = fg_horizontal_pass_neon_u8_rdm::; + _dispatcher_horizontal = fg_horizontal_pass_neon_u8_rdm::; } } } @@ -530,7 +530,7 @@ impl FastGaussianDispatchProvider for u8 { #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] { if BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_horizontal = fg_horizontal_pass_wasm_u8::; + _dispatcher_horizontal = fg_horizontal_pass_wasm_u8::; } } #[cfg(all(target_arch = "x86_64", feature = "avx"))] @@ -538,14 +538,14 @@ impl FastGaussianDispatchProvider for u8 { let has_avx = std::arch::is_x86_feature_detected!("avx2"); if has_avx && BASE_RADIUS_I64_CUTOFF > radius { use crate::avx::fg_horizontal_pass_sse_u8; - return fg_horizontal_pass_sse_u8::; + return fg_horizontal_pass_sse_u8::; } } #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] { let is_sse_available = std::arch::is_x86_feature_detected!("sse4.1"); if is_sse_available && BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_horizontal = fg_horizontal_pass_sse_u8::; + _dispatcher_horizontal = fg_horizontal_pass_sse_u8::; } } _dispatcher_horizontal @@ -571,12 +571,12 @@ impl FastGaussianDispatchProvider for u8 { #[cfg(all(target_arch = "aarch64", feature = "neon"))] { if BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_vertical = fg_vertical_pass_neon_u8::; + _dispatcher_vertical = fg_vertical_pass_neon_u8::; #[cfg(feature = "rdm")] { if std::arch::is_aarch64_feature_detected!("rdm") { use crate::neon::fg_vertical_pass_neon_u8_rdm; - _dispatcher_vertical = fg_vertical_pass_neon_u8_rdm::; + _dispatcher_vertical = fg_vertical_pass_neon_u8_rdm::; } } } @@ -584,7 +584,7 @@ impl FastGaussianDispatchProvider for u8 { #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] { if BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_vertical = fg_vertical_pass_wasm_u8::; + _dispatcher_vertical = fg_vertical_pass_wasm_u8::; } } #[cfg(all(target_arch = "x86_64", feature = "avx"))] @@ -592,14 +592,14 @@ impl FastGaussianDispatchProvider for u8 { let has_avx = std::arch::is_x86_feature_detected!("avx2"); if has_avx && BASE_RADIUS_I64_CUTOFF > radius { use crate::avx::fg_vertical_pass_avx_u8; - return fg_vertical_pass_avx_u8::; + return fg_vertical_pass_avx_u8::; } } #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] { let is_sse_available = std::arch::is_x86_feature_detected!("sse4.1"); if is_sse_available && BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_vertical = fg_vertical_pass_sse_u8::; + _dispatcher_vertical = fg_vertical_pass_sse_u8::; } } _dispatcher_vertical @@ -629,19 +629,19 @@ impl FastGaussianDispatchProvider for f32 { let has_avx = std::arch::is_x86_feature_detected!("avx2"); if has_avx { use crate::avx::fg_vertical_pass_avx_f32; - return fg_vertical_pass_avx_f32::; + return fg_vertical_pass_avx_f32::; } } #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] { let is_sse_available = std::arch::is_x86_feature_detected!("sse4.1"); if is_sse_available { - _dispatcher_vertical = fg_vertical_pass_sse_f32::; + _dispatcher_vertical = fg_vertical_pass_sse_f32::; } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _dispatcher_vertical = fg_vertical_pass_neon_f32::; + _dispatcher_vertical = fg_vertical_pass_neon_f32::; } _dispatcher_vertical } @@ -668,19 +668,19 @@ impl FastGaussianDispatchProvider for f32 { let has_avx = std::arch::is_x86_feature_detected!("avx2"); if has_avx { use crate::avx::fg_horizontal_pass_avx_f32; - return fg_horizontal_pass_avx_f32::; + return fg_horizontal_pass_avx_f32::; } } #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] { let is_sse_available = std::arch::is_x86_feature_detected!("sse4.1"); if is_sse_available { - _dispatcher_horizontal = fg_horizontal_pass_sse_f32::; + _dispatcher_horizontal = fg_horizontal_pass_sse_f32::; } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _dispatcher_horizontal = fg_horizontal_pass_neon_f32::; + _dispatcher_horizontal = fg_horizontal_pass_neon_f32::; } _dispatcher_horizontal } @@ -707,7 +707,7 @@ impl FastGaussianDispatchProvider for f16 { }; #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _dispatcher_vertical = fg_vertical_pass_neon_f16::; + _dispatcher_vertical = fg_vertical_pass_neon_f16::; } #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] { @@ -739,14 +739,14 @@ impl FastGaussianDispatchProvider for f16 { }; #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _dispatcher_horizontal = fg_horizontal_pass_neon_f16::; + _dispatcher_horizontal = fg_horizontal_pass_neon_f16::; } #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] { if std::arch::is_x86_feature_detected!("sse4.1") && std::arch::is_x86_feature_detected!("f16c") { - _dispatcher_horizontal = fg_horizontal_pass_sse_f16::; + _dispatcher_horizontal = fg_horizontal_pass_sse_f16::; } } _dispatcher_horizontal diff --git a/src/fast_gaussian_image.rs b/src/fast_gaussian_image.rs index 335be890..8052bf91 100644 --- a/src/fast_gaussian_image.rs +++ b/src/fast_gaussian_image.rs @@ -27,8 +27,8 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::{ - fast_gaussian, fast_gaussian_f32, fast_gaussian_u16, AnisotropicRadius, BlurImageMut, - EdgeMode2D, FastBlurChannels, ThreadingPolicy, + AnisotropicRadius, BlurImageMut, EdgeMode2D, FastBlurChannels, ThreadingPolicy, fast_gaussian, + fast_gaussian_f32, fast_gaussian_u16, }; use image::{ DynamicImage, GrayAlphaImage, GrayImage, ImageBuffer, Luma, LumaA, Rgb, Rgb32FImage, RgbImage, diff --git a/src/fast_gaussian_image_next.rs b/src/fast_gaussian_image_next.rs index 897c64be..8cd5802f 100644 --- a/src/fast_gaussian_image_next.rs +++ b/src/fast_gaussian_image_next.rs @@ -27,8 +27,8 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::{ - fast_gaussian_next, fast_gaussian_next_f32, fast_gaussian_next_u16, AnisotropicRadius, - BlurImageMut, EdgeMode2D, FastBlurChannels, ThreadingPolicy, + AnisotropicRadius, BlurImageMut, EdgeMode2D, FastBlurChannels, ThreadingPolicy, + fast_gaussian_next, fast_gaussian_next_f32, fast_gaussian_next_u16, }; use image::{ DynamicImage, GrayAlphaImage, GrayImage, ImageBuffer, Luma, LumaA, Rgb, Rgb32FImage, RgbImage, diff --git a/src/fast_gaussian_next.rs b/src/fast_gaussian_next.rs index ec0d5d80..bcf1bf27 100644 --- a/src/fast_gaussian_next.rs +++ b/src/fast_gaussian_next.rs @@ -184,10 +184,10 @@ fn fgn_vertical_pass< i32: PrimitiveCast, f64: PrimitiveCast, { - let mut buffer_r = Box::new([0i32.cast_(); 1024]); - let mut buffer_g = Box::new([0i32.cast_(); 1024]); - let mut buffer_b = Box::new([0i32.cast_(); 1024]); - let mut buffer_a = Box::new([0i32.cast_(); 1024]); + let mut buffer_r = [0i32.cast_(); 1024]; + let mut buffer_g = [0i32.cast_(); 1024]; + let mut buffer_b = [0i32.cast_(); 1024]; + let mut buffer_a = [0i32.cast_(); 1024]; let radius_64 = radius as i64; let height_wide = height as i64; let weight: M = (1.0f64 / ((radius as f64) * (radius as f64) * (radius as f64))).cast_(); @@ -325,10 +325,10 @@ fn fgn_horizontal_pass< i32: PrimitiveCast, f64: PrimitiveCast, { - let mut buffer_r = Box::new([0i32.cast_(); 1024]); - let mut buffer_g = Box::new([0i32.cast_(); 1024]); - let mut buffer_b = Box::new([0i32.cast_(); 1024]); - let mut buffer_a = Box::new([0i32.cast_(); 1024]); + let mut buffer_r = [0i32.cast_(); 1024]; + let mut buffer_g = [0i32.cast_(); 1024]; + let mut buffer_b = [0i32.cast_(); 1024]; + let mut buffer_a = [0i32.cast_(); 1024]; let radius_64 = radius as i64; let width_wide = width as i64; let weight: M = (1.0f64 / ((radius as f64) * (radius as f64) * (radius as f64))).cast_(); @@ -581,12 +581,12 @@ impl FastGaussianNextPassProvider for u8 { #[cfg(all(target_arch = "aarch64", feature = "neon"))] { if BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_horizontal = fgn_horizontal_pass_neon_u8::; + _dispatcher_horizontal = fgn_horizontal_pass_neon_u8::; #[cfg(feature = "rdm")] { if std::arch::is_aarch64_feature_detected!("rdm") { use crate::neon::fgn_horizontal_pass_neon_u8_rdm; - _dispatcher_horizontal = fgn_horizontal_pass_neon_u8_rdm::; + _dispatcher_horizontal = fgn_horizontal_pass_neon_u8_rdm::; } } } @@ -595,7 +595,7 @@ impl FastGaussianNextPassProvider for u8 { #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] { if BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_horizontal = fast_gaussian_next_horizontal_pass_wasm_u8::; + _dispatcher_horizontal = fast_gaussian_next_horizontal_pass_wasm_u8::; } } @@ -626,7 +626,7 @@ impl FastGaussianNextPassProvider for u8 { if BASE_RADIUS_I64_CUTOFF > radius && is_avx_available { use crate::avx::fgn_vertical_pass_avx_u8; - return fgn_vertical_pass_avx_u8::; + return fgn_vertical_pass_avx_u8::; } } @@ -635,19 +635,19 @@ impl FastGaussianNextPassProvider for u8 { let is_sse_available = std::arch::is_x86_feature_detected!("sse4.1"); if BASE_RADIUS_I64_CUTOFF > radius && is_sse_available { - _dispatcher_vertical = fgn_vertical_pass_sse_u8::; + _dispatcher_vertical = fgn_vertical_pass_sse_u8::; } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { if BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_vertical = fgn_vertical_pass_neon_u8::; + _dispatcher_vertical = fgn_vertical_pass_neon_u8::; #[cfg(feature = "rdm")] { if std::arch::is_aarch64_feature_detected!("rdm") { use crate::neon::fgn_vertical_pass_neon_u8_rdm; - _dispatcher_vertical = fgn_vertical_pass_neon_u8_rdm::; + _dispatcher_vertical = fgn_vertical_pass_neon_u8_rdm::; } } } @@ -656,7 +656,7 @@ impl FastGaussianNextPassProvider for u8 { #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] { if BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_vertical = fast_gaussian_next_vertical_pass_wasm_u8::; + _dispatcher_vertical = fast_gaussian_next_vertical_pass_wasm_u8::; } } _dispatcher_vertical @@ -694,13 +694,13 @@ impl FastGaussianNextPassProvider for f32 { { let is_sse_available = std::arch::is_x86_feature_detected!("sse4.1"); if is_sse_available { - _dispatcher_horizontal = fgn_horizontal_pass_sse_f32::; + _dispatcher_horizontal = fgn_horizontal_pass_sse_f32::; } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { if BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_horizontal = fgn_horizontal_pass_neon_f32::; + _dispatcher_horizontal = fgn_horizontal_pass_neon_f32::; } } _dispatcher_horizontal @@ -729,20 +729,20 @@ impl FastGaussianNextPassProvider for f32 { if has_avx { use crate::avx::fgn_vertical_pass_avx_f32; - return fgn_vertical_pass_avx_f32::; + return fgn_vertical_pass_avx_f32::; } } #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] { let is_sse_available = std::arch::is_x86_feature_detected!("sse4.1"); if is_sse_available { - _dispatcher_vertical = fgn_vertical_pass_sse_f32::; + _dispatcher_vertical = fgn_vertical_pass_sse_f32::; } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { if BASE_RADIUS_I64_CUTOFF > radius { - _dispatcher_vertical = fgn_vertical_pass_neon_f32::; + _dispatcher_vertical = fgn_vertical_pass_neon_f32::; } } _dispatcher_vertical @@ -773,12 +773,12 @@ impl FastGaussianNextPassProvider for f16 { let is_sse_available = std::arch::is_x86_feature_detected!("sse4.1"); let is_f16c_available = std::arch::is_x86_feature_detected!("f16c"); if is_f16c_available && is_sse_available { - _dispatcher_horizontal = fast_gaussian_next_horizontal_pass_sse_f16::; + _dispatcher_horizontal = fast_gaussian_next_horizontal_pass_sse_f16::; } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _dispatcher_horizontal = fgn_horizontal_pass_neon_f16::; + _dispatcher_horizontal = fgn_horizontal_pass_neon_f16::; } _dispatcher_horizontal } @@ -805,12 +805,12 @@ impl FastGaussianNextPassProvider for f16 { let _is_sse_available = std::arch::is_x86_feature_detected!("sse4.1"); let _is_f16c_available = std::arch::is_x86_feature_detected!("f16c"); if _is_f16c_available && _is_sse_available { - _dispatcher_vertical = fast_gaussian_next_vertical_pass_sse_f16::; + _dispatcher_vertical = fast_gaussian_next_vertical_pass_sse_f16::; } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _dispatcher_vertical = fgn_vertical_pass_neon_f16::; + _dispatcher_vertical = fgn_vertical_pass_neon_f16::; } _dispatcher_vertical } diff --git a/src/filter1d/avx/filter_column.rs b/src/filter1d/avx/filter_column.rs index 76fdf8f0..61faafa5 100644 --- a/src/filter1d/avx/filter_column.rs +++ b/src/filter1d/avx/filter_column.rs @@ -125,7 +125,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, arena: Arena, arena_src: &[&[u8]], @@ -134,192 +134,198 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let image_width = image_size.width * arena.components; - - let length = scanned_kernel.len(); - - let mut cx = 0usize; - - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(0).weight); - - while cx + 128 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let source = _mm256_load_pack_x4(v_src.as_ptr()); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); - let mut k2 = _mm256_mul_epi8_by_ps_x4::(source.2, coeff); - let mut k3 = _mm256_mul_epi8_by_ps_x4::(source.3, coeff); - - for i in 1..length { - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = - _mm256_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source.0, coeff); - k1 = _mm256_mul_add_epi8_by_ps_x4::(k1, v_source.1, coeff); - k2 = _mm256_mul_add_epi8_by_ps_x4::(k2, v_source.2, coeff); - k3 = _mm256_mul_add_epi8_by_ps_x4::(k3, v_source.3, coeff); - } - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x4( - dst_ptr0, - ( - _mm256_pack_ps_x4_epi8(k0), - _mm256_pack_ps_x4_epi8(k1), - _mm256_pack_ps_x4_epi8(k2), - _mm256_pack_ps_x4_epi8(k3), - ), - ); - cx += 128; - } - - while cx + 96 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let source = _mm256_load_pack_x3(v_src.as_ptr()); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); - let mut k2 = _mm256_mul_epi8_by_ps_x4::(source.2, coeff); - - for i in 1..length { - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = - _mm256_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source.0, coeff); - k1 = _mm256_mul_add_epi8_by_ps_x4::(k1, v_source.1, coeff); - k2 = _mm256_mul_add_epi8_by_ps_x4::(k2, v_source.2, coeff); + unsafe { + let image_width = image_size.width * arena.components; + + let length = scanned_kernel.len(); + + let mut cx = 0usize; + + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(0).weight); + + while cx + 128 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let source = _mm256_load_pack_x4(v_src.as_ptr()); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); + let mut k2 = _mm256_mul_epi8_by_ps_x4::(source.2, coeff); + let mut k3 = _mm256_mul_epi8_by_ps_x4::(source.3, coeff); + + for i in 1..length { + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm256_load_pack_x4( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr(), + ); + k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source.0, coeff); + k1 = _mm256_mul_add_epi8_by_ps_x4::(k1, v_source.1, coeff); + k2 = _mm256_mul_add_epi8_by_ps_x4::(k2, v_source.2, coeff); + k3 = _mm256_mul_add_epi8_by_ps_x4::(k3, v_source.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x4( + dst_ptr0, + ( + _mm256_pack_ps_x4_epi8(k0), + _mm256_pack_ps_x4_epi8(k1), + _mm256_pack_ps_x4_epi8(k2), + _mm256_pack_ps_x4_epi8(k3), + ), + ); + cx += 128; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x3( - dst_ptr0, - ( - _mm256_pack_ps_x4_epi8(k0), - _mm256_pack_ps_x4_epi8(k1), - _mm256_pack_ps_x4_epi8(k2), - ), - ); - cx += 96; - } - - while cx + 64 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let source = _mm256_load_pack_x2(v_src.as_ptr()); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); - - for i in 1..length { - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = - _mm256_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source.0, coeff); - k1 = _mm256_mul_add_epi8_by_ps_x4::(k1, v_source.1, coeff); + while cx + 96 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let source = _mm256_load_pack_x3(v_src.as_ptr()); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); + let mut k2 = _mm256_mul_epi8_by_ps_x4::(source.2, coeff); + + for i in 1..length { + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm256_load_pack_x3( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr(), + ); + k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source.0, coeff); + k1 = _mm256_mul_add_epi8_by_ps_x4::(k1, v_source.1, coeff); + k2 = _mm256_mul_add_epi8_by_ps_x4::(k2, v_source.2, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x3( + dst_ptr0, + ( + _mm256_pack_ps_x4_epi8(k0), + _mm256_pack_ps_x4_epi8(k1), + _mm256_pack_ps_x4_epi8(k2), + ), + ); + cx += 96; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x2( - dst_ptr0, - (_mm256_pack_ps_x4_epi8(k0), _mm256_pack_ps_x4_epi8(k1)), - ); - cx += 64; - } - - while cx + 32 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source, coeff); - - for i in 1..length { - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm256_loadu_si256( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, + while cx + 64 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let source = _mm256_load_pack_x2(v_src.as_ptr()); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); + + for i in 1..length { + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm256_load_pack_x2( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr(), + ); + k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source.0, coeff); + k1 = _mm256_mul_add_epi8_by_ps_x4::(k1, v_source.1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x2( + dst_ptr0, + (_mm256_pack_ps_x4_epi8(k0), _mm256_pack_ps_x4_epi8(k1)), ); - k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x4_epi8(k0)); - cx += 32; - } + while cx + 32 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - while cx + 16 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source, coeff); - let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi8_by_ps_x4::(source_0, _mm256_castps256_ps128(coeff)); + for i in 1..length { + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = + _mm256_loadu_si256(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() + as *const __m256i); + k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source, coeff); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - let v_source_0 = _mm_loadu_si128( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - k0 = _mm_mul_add_epi8_by_ps_x4::(k0, v_source_0, _mm_set1_ps(coeff.weight)); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x4_epi8(k0)); + cx += 32; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); - cx += 16; - } + while cx + 16 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - let coeff = *scanned_kernel.get_unchecked(0); + let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi8_by_ps_x4::(source_0, _mm256_castps256_ps128(coeff)); - while cx + 4 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + let v_source_0 = + _mm_loadu_si128(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() + as *const __m128i); + k0 = + _mm_mul_add_epi8_by_ps_x4::(k0, v_source_0, _mm_set1_ps(coeff.weight)); + } - let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff.weight); - let mut k1 = (*v_src.get_unchecked(1) as f32).mul(coeff.weight); - let mut k2 = (*v_src.get_unchecked(2) as f32).mul(coeff.weight); - let mut k3 = (*v_src.get_unchecked(3) as f32).mul(coeff.weight); + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); + cx += 16; + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = mlaf( - k0, - (*arena_src.get_unchecked(i).get_unchecked(cx)) as f32, - coeff.weight, - ); - k1 = mlaf( - k1, - (*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as f32, - coeff.weight, - ); - k2 = mlaf( - k2, - (*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as f32, - coeff.weight, - ); - k3 = mlaf( - k3, - (*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as f32, - coeff.weight, - ); + let coeff = *scanned_kernel.get_unchecked(0); + + while cx + 4 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff.weight); + let mut k1 = (*v_src.get_unchecked(1) as f32).mul(coeff.weight); + let mut k2 = (*v_src.get_unchecked(2) as f32).mul(coeff.weight); + let mut k3 = (*v_src.get_unchecked(3) as f32).mul(coeff.weight); + + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 = mlaf( + k0, + (*arena_src.get_unchecked(i).get_unchecked(cx)) as f32, + coeff.weight, + ); + k1 = mlaf( + k1, + (*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as f32, + coeff.weight, + ); + k2 = mlaf( + k2, + (*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as f32, + coeff.weight, + ); + k3 = mlaf( + k3, + (*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as f32, + coeff.weight, + ); + } + + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } + for x in cx..image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(x..); - for x in cx..image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(x..); + let mut k0 = ((*v_src.get_unchecked(0)) as f32).mul(coeff.weight); - let mut k0 = ((*v_src.get_unchecked(0)) as f32).mul(coeff.weight); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 = mlaf( + k0, + (*arena_src.get_unchecked(i).get_unchecked(x)) as f32, + coeff.weight, + ); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = mlaf( - k0, - (*arena_src.get_unchecked(i).get_unchecked(x)) as f32, - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/avx/filter_column_approx.rs b/src/filter1d/avx/filter_column_approx.rs index 338cc6b2..901729bf 100644 --- a/src/filter1d/avx/filter_column_approx.rs +++ b/src/filter1d/avx/filter_column_approx.rs @@ -35,7 +35,6 @@ use crate::filter1d::avx::sse_utils::{ use crate::filter1d::avx::utils::{ _mm256_mul_add_epi8_by_epi16_x4, _mm256_mul_epi8_by_epi16_x4, _mm256_pack_epi32_x4_epi8, }; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; @@ -48,7 +47,7 @@ pub(crate) fn filter_column_avx_u8_i32_app( dst: &mut [u8], image_size: ImageSize, filter_region: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { filter_column_avx_u8_i32_impl( @@ -63,123 +62,118 @@ pub(crate) fn filter_column_avx_u8_i32_app( } #[target_feature(enable = "avx2")] -unsafe fn filter_column_avx_u8_i32_impl( +fn filter_column_avx_u8_i32_impl( arena: Arena, arena_src: &[&[u8]], dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + v_prepared: &[i32], ) { - let image_width = image_size.width * arena.components; + unsafe { + let image_width = image_size.width * arena.components; - let length = scanned_kernel.len(); + let length = v_prepared.len(); - let mut cx = 0usize; + let mut cx = 0usize; - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.weight.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) - }) - .collect::>(); + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(0)); - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(0)); + while cx + 64 < image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - while cx + 64 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + let source = _mm256_load_pack_x2(v_src.as_ptr()); + let mut k0 = _mm256_mul_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_epi16_x4(source.1, coeff); - let source = _mm256_load_pack_x2(v_src.as_ptr()); - let mut k0 = _mm256_mul_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_epi16_x4(source.1, coeff); + for i in 1..length { + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source = + _mm256_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + k0 = _mm256_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); + k1 = _mm256_mul_add_epi8_by_epi16_x4(k1, v_source.1, coeff); + } - for i in 1..length { - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source = - _mm256_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - k0 = _mm256_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); - k1 = _mm256_mul_add_epi8_by_epi16_x4(k1, v_source.1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x2( + dst_ptr0, + (_mm256_pack_epi32_x4_epi8(k0), _mm256_pack_epi32_x4_epi8(k1)), + ); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x2( - dst_ptr0, - (_mm256_pack_epi32_x4_epi8(k0), _mm256_pack_epi32_x4_epi8(k1)), - ); - cx += 64; - } + while cx + 32 < image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - while cx + 32 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); + let mut k0 = _mm256_mul_epi8_by_epi16_x4(source, coeff); - let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); - let mut k0 = _mm256_mul_epi8_by_epi16_x4(source, coeff); + for i in 1..length { + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source = _mm256_loadu_si256( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, + ); + k0 = _mm256_mul_add_epi8_by_epi16_x4(k0, v_source, coeff); + } - for i in 1..length { - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source = _mm256_loadu_si256( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, - ); - k0 = _mm256_mul_add_epi8_by_epi16_x4(k0, v_source, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_epi32_x4_epi8(k0)); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_epi32_x4_epi8(k0)); - cx += 32; - } + while cx + 16 < image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - while cx + 16 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + let source = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi8_by_epi16_x4(source, _mm256_castsi256_si128(coeff)); - let source = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source, _mm256_castsi256_si128(coeff)); + for i in 1..length { + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source = _mm_loadu_si128( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, + ); + k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source, coeff); + } - for i in 1..length { - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source = _mm_loadu_si128( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i - ); - k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_epi32_x2_epi8(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_epi32_x2_epi8(k0)); - cx += 16; - } + while cx + 4 < image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - while cx + 4 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + let source = _mm_loadu_si32(v_src.as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - let source = _mm_loadu_si32(v_src.as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); + for i in 1..length { + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source = _mm_loadu_si32( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_epi8_by_epi16_x2(k0, v_source, coeff); + } - for i in 1..length { - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source = - _mm_loadu_si32(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _); - k0 = _mm_mul_add_epi8_by_epi16_x2(k0, v_source, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); + cx += 4; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); - cx += 4; - } + let coeff = *v_prepared.get_unchecked(0); - let coeff = *scanned_kernel.get_unchecked(0); + for x in cx..image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(x..); - for x in cx..image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(x..); + let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff); - let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff.weight); + for i in 1..length { + let coeff = *v_prepared.get_unchecked(i); + k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as i32) + .mul(coeff) + .add(k0); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as i32) - .mul(coeff.weight) - .add(k0); + *dst.get_unchecked_mut(x) = k0.to_approx_(); } - - *dst.get_unchecked_mut(x) = k0.to_approx_(); } } diff --git a/src/filter1d/avx/filter_column_complex_u16_f32.rs b/src/filter1d/avx/filter_column_complex_u16_f32.rs index 8648bfca..378cda54 100644 --- a/src/filter1d/avx/filter_column_complex_u16_f32.rs +++ b/src/filter1d/avx/filter_column_complex_u16_f32.rs @@ -29,7 +29,7 @@ use crate::avx::shuffle; use crate::filter1d::arena::Arena; use crate::filter1d::avx::filter_column_complex_u8_f32::{ - _mm256_correct4x64_ps, mq256_complex_mla, mq256_complex_mul, mq_complex_mla, mq_complex_mul, + _mm256_correct4x64_ps, mq_complex_mla, mq_complex_mul, mq256_complex_mla, mq256_complex_mul, }; use crate::img_size::ImageSize; use crate::to_storage::ToStorage; diff --git a/src/filter1d/avx/filter_column_complex_u16_i32.rs b/src/filter1d/avx/filter_column_complex_u16_i32.rs index 1506dabc..2bce260c 100644 --- a/src/filter1d/avx/filter_column_complex_u16_i32.rs +++ b/src/filter1d/avx/filter_column_complex_u16_i32.rs @@ -36,47 +36,57 @@ use num_traits::MulAdd; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm_addsub_epi32(a: __m128i, b: __m128i) -> __m128i { - let c = _mm_sign_epi32(b, _mm_setr_epi32(-1, 1, -1, 1)); - _mm_add_epi32(a, c) +pub(crate) fn _mm_addsub_epi32(a: __m128i, b: __m128i) -> __m128i { + unsafe { + let c = _mm_sign_epi32(b, _mm_setr_epi32(-1, 1, -1, 1)); + _mm_add_epi32(a, c) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_addsub_epi32(a: __m256i, b: __m256i) -> __m256i { - let c = _mm256_sign_epi32(b, _mm256_setr_epi32(-1, 1, -1, 1, -1, 1, -1, 1)); - _mm256_add_epi32(a, c) +pub(crate) fn _mm256_addsub_epi32(a: __m256i, b: __m256i) -> __m256i { + unsafe { + let c = _mm256_sign_epi32(b, _mm256_setr_epi32(-1, 1, -1, 1, -1, 1, -1, 1)); + _mm256_add_epi32(a, c) + } } #[inline(always)] -pub(crate) unsafe fn mq_complex_mla( +pub(crate) fn mq_complex_mla( acc_r: __m128i, r0i0: __m128i, r: __m128i, r_swop: __m128i, ) -> __m128i { - let c0 = _mm_mullo_epi32(r0i0, r); - let c1 = _mm_mullo_epi32(_mm_shuffle_epi32::<{ _shuffle(2, 3, 0, 1) }>(r0i0), r_swop); - _mm_add_epi32(acc_r, _mm_addsub_epi32(c0, c1)) + unsafe { + let c0 = _mm_mullo_epi32(r0i0, r); + let c1 = _mm_mullo_epi32(_mm_shuffle_epi32::<{ _shuffle(2, 3, 0, 1) }>(r0i0), r_swop); + _mm_add_epi32(acc_r, _mm_addsub_epi32(c0, c1)) + } } #[inline(always)] -pub(crate) unsafe fn mq_complex_mul(r0i0: __m128i, r: __m128i, r_swop: __m128i) -> __m128i { - let c0 = _mm_mullo_epi32(r0i0, r); - let c1 = _mm_mullo_epi32(_mm_shuffle_epi32::<{ _shuffle(2, 3, 0, 1) }>(r0i0), r_swop); - _mm_addsub_epi32(c0, c1) +pub(crate) fn mq_complex_mul(r0i0: __m128i, r: __m128i, r_swop: __m128i) -> __m128i { + unsafe { + let c0 = _mm_mullo_epi32(r0i0, r); + let c1 = _mm_mullo_epi32(_mm_shuffle_epi32::<{ _shuffle(2, 3, 0, 1) }>(r0i0), r_swop); + _mm_addsub_epi32(c0, c1) + } } #[inline(always)] -pub(crate) unsafe fn mq256_complex_mla( +pub(crate) fn mq256_complex_mla( acc: __m256i, r0i0: __m256i, r: __m256i, r_swop: __m256i, idx: __m256i, ) -> __m256i { - let c0 = _mm256_mullo_epi32(r0i0, r); - let c1 = _mm256_mullo_epi32(_mm256_permutevar8x32_epi32(r0i0, idx), r_swop); - _mm256_add_epi32(acc, _mm256_addsub_epi32(c0, c1)) + unsafe { + let c0 = _mm256_mullo_epi32(r0i0, r); + let c1 = _mm256_mullo_epi32(_mm256_permutevar8x32_epi32(r0i0, idx), r_swop); + _mm256_add_epi32(acc, _mm256_addsub_epi32(c0, c1)) + } } pub(crate) fn filter_avx_column_complex_u16_i32( @@ -92,7 +102,7 @@ pub(crate) fn filter_avx_column_complex_u16_i32( } #[target_feature(enable = "avx2")] -unsafe fn filter_avx_column_complex_u16_i32_impl( +fn filter_avx_column_complex_u16_i32_impl( arena: Arena, arena_src: &[&[Complex]], dst: &mut [u16], diff --git a/src/filter1d/avx/filter_column_complex_u8_f32.rs b/src/filter1d/avx/filter_column_complex_u8_f32.rs index e2f0f1ca..31c7f8e7 100644 --- a/src/filter1d/avx/filter_column_complex_u8_f32.rs +++ b/src/filter1d/avx/filter_column_complex_u8_f32.rs @@ -34,59 +34,57 @@ use num_complex::Complex; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn mq_complex_mla( - acc_r: __m128, - r0i0: __m128, - r: __m128, - r_swop: __m128, -) -> __m128 { - let c0 = _mm_mul_ps(r0i0, r); - let c1 = _mm_mul_ps( - _mm_shuffle_ps::<{ shuffle(2, 3, 0, 1) }>(r0i0, r0i0), - r_swop, - ); - _mm_add_ps(acc_r, _mm_addsub_ps(c0, c1)) +pub(crate) fn mq_complex_mla(acc_r: __m128, r0i0: __m128, r: __m128, r_swop: __m128) -> __m128 { + unsafe { + let c0 = _mm_mul_ps(r0i0, r); + let c1 = _mm_mul_ps( + _mm_shuffle_ps::<{ shuffle(2, 3, 0, 1) }>(r0i0, r0i0), + r_swop, + ); + _mm_add_ps(acc_r, _mm_addsub_ps(c0, c1)) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_correct4x64_ps(a: __m256) -> __m256 { +pub(crate) fn _mm256_correct4x64_ps(a: __m256) -> __m256 { const M: i32 = shuffle(3, 1, 2, 0); - _mm256_castsi256_ps(_mm256_permute4x64_epi64::(_mm256_castps_si256(a))) + unsafe { _mm256_castsi256_ps(_mm256_permute4x64_epi64::(_mm256_castps_si256(a))) } } #[inline(always)] -pub(crate) unsafe fn mq256_complex_mla( - acc_r: __m256, - r0i0: __m256, - r: __m256, - r_swop: __m256, -) -> __m256 { - let c0 = _mm256_mul_ps(r0i0, r); - let c1 = _mm256_mul_ps( - _mm256_correct4x64_ps(_mm256_shuffle_ps::<{ shuffle(2, 3, 0, 1) }>(r0i0, r0i0)), - r_swop, - ); - _mm256_add_ps(acc_r, _mm256_addsub_ps(c0, c1)) +pub(crate) fn mq256_complex_mla(acc_r: __m256, r0i0: __m256, r: __m256, r_swop: __m256) -> __m256 { + unsafe { + let c0 = _mm256_mul_ps(r0i0, r); + let c1 = _mm256_mul_ps( + _mm256_correct4x64_ps(_mm256_shuffle_ps::<{ shuffle(2, 3, 0, 1) }>(r0i0, r0i0)), + r_swop, + ); + _mm256_add_ps(acc_r, _mm256_addsub_ps(c0, c1)) + } } #[inline(always)] -pub(crate) unsafe fn mq256_complex_mul(r0i0: __m256, r: __m256, r_swop: __m256) -> __m256 { - let c0 = _mm256_mul_ps(r0i0, r); - let c1 = _mm256_mul_ps( - _mm256_correct4x64_ps(_mm256_shuffle_ps::<{ shuffle(2, 3, 0, 1) }>(r0i0, r0i0)), - r_swop, - ); - _mm256_addsub_ps(c0, c1) +pub(crate) fn mq256_complex_mul(r0i0: __m256, r: __m256, r_swop: __m256) -> __m256 { + unsafe { + let c0 = _mm256_mul_ps(r0i0, r); + let c1 = _mm256_mul_ps( + _mm256_correct4x64_ps(_mm256_shuffle_ps::<{ shuffle(2, 3, 0, 1) }>(r0i0, r0i0)), + r_swop, + ); + _mm256_addsub_ps(c0, c1) + } } #[inline(always)] -pub(crate) unsafe fn mq_complex_mul(r0i0: __m128, r: __m128, r_swop: __m128) -> __m128 { - let c0 = _mm_mul_ps(r0i0, r); - let c1 = _mm_mul_ps( - _mm_shuffle_ps::<{ shuffle(2, 3, 0, 1) }>(r0i0, r0i0), - r_swop, - ); - _mm_addsub_ps(c0, c1) +pub(crate) fn mq_complex_mul(r0i0: __m128, r: __m128, r_swop: __m128) -> __m128 { + unsafe { + let c0 = _mm_mul_ps(r0i0, r); + let c1 = _mm_mul_ps( + _mm_shuffle_ps::<{ shuffle(2, 3, 0, 1) }>(r0i0, r0i0), + r_swop, + ); + _mm_addsub_ps(c0, c1) + } } pub(crate) fn filter_avx_column_complex_u8_f32( diff --git a/src/filter1d/avx/filter_column_complex_u8_i32.rs b/src/filter1d/avx/filter_column_complex_u8_i32.rs index 81d58425..96e9319e 100644 --- a/src/filter1d/avx/filter_column_complex_u8_i32.rs +++ b/src/filter1d/avx/filter_column_complex_u8_i32.rs @@ -36,44 +36,50 @@ use num_traits::MulAdd; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn v_interleave_i32x8(a: __m256i, b: __m256i) -> (__m256i, __m256i) { - let xy_l = _mm256_unpacklo_epi32(a, b); - let xy_h = _mm256_unpackhi_epi32(a, b); +pub(crate) fn v_interleave_i32x8(a: __m256i, b: __m256i) -> (__m256i, __m256i) { + unsafe { + let xy_l = _mm256_unpacklo_epi32(a, b); + let xy_h = _mm256_unpackhi_epi32(a, b); - let xy0 = _mm256_permute2x128_si256::<32>(xy_l, xy_h); - let xy1 = _mm256_permute2x128_si256::<49>(xy_l, xy_h); - (xy0, xy1) + let xy0 = _mm256_permute2x128_si256::<32>(xy_l, xy_h); + let xy1 = _mm256_permute2x128_si256::<49>(xy_l, xy_h); + (xy0, xy1) + } } #[inline(always)] -pub(crate) unsafe fn mq_complex_mla( +pub(crate) fn mq_complex_mla( acc: (__m128i, __m128i), r0i0: __m128i, r: __m128i, r_swop: __m128i, ) -> (__m128i, __m128i) { - let xc00 = _mm_madd_epi16(r0i0, r); - let xc10 = _mm_madd_epi16(r0i0, r_swop); + unsafe { + let xc00 = _mm_madd_epi16(r0i0, r); + let xc10 = _mm_madd_epi16(r0i0, r_swop); - let x_r = _mm_unpacklo_epi32(xc00, xc10); - let x_c = _mm_unpackhi_epi32(xc00, xc10); + let x_r = _mm_unpacklo_epi32(xc00, xc10); + let x_c = _mm_unpackhi_epi32(xc00, xc10); - (_mm_add_epi32(acc.0, x_r), _mm_add_epi32(acc.1, x_c)) + (_mm_add_epi32(acc.0, x_r), _mm_add_epi32(acc.1, x_c)) + } } #[inline(always)] -pub(crate) unsafe fn mq256_complex_mla( +pub(crate) fn mq256_complex_mla( acc: (__m256i, __m256i), r0i0: __m256i, r: __m256i, r_swop: __m256i, ) -> (__m256i, __m256i) { - let x_r = _mm256_madd_epi16(r0i0, r); - let x_c = _mm256_madd_epi16(r0i0, r_swop); + unsafe { + let x_r = _mm256_madd_epi16(r0i0, r); + let x_c = _mm256_madd_epi16(r0i0, r_swop); - let (z_r, z_c) = v_interleave_i32x8(x_r, x_c); + let (z_r, z_c) = v_interleave_i32x8(x_r, x_c); - (_mm256_add_epi32(acc.0, z_r), _mm256_add_epi32(acc.1, z_c)) + (_mm256_add_epi32(acc.0, z_r), _mm256_add_epi32(acc.1, z_c)) + } } pub(crate) fn filter_avx_column_complex_u8_i32( @@ -89,7 +95,7 @@ pub(crate) fn filter_avx_column_complex_u8_i32( } #[target_feature(enable = "avx2")] -unsafe fn filter_avx_column_complex_u8_i32_impl( +fn filter_avx_column_complex_u8_i32_impl( arena: Arena, arena_src: &[&[Complex]], dst: &mut [u8], diff --git a/src/filter1d/avx/filter_column_f32.rs b/src/filter1d/avx/filter_column_f32.rs index 8917b178..e89ab329 100644 --- a/src/filter1d/avx/filter_column_f32.rs +++ b/src/filter1d/avx/filter_column_f32.rs @@ -91,7 +91,7 @@ unsafe fn filter_column_avx_f32_f32_impl_fma( } #[target_feature(enable = "avx2")] -unsafe fn filter_column_avx_f32_f32_impl_def( +fn filter_column_avx_f32_f32_impl_def( arena: Arena, arena_src: &[&[f32]], dst: &mut [f32], @@ -115,7 +115,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, arena: Arena, arena_src: &[&[f32]], diff --git a/src/filter1d/avx/filter_column_symm.rs b/src/filter1d/avx/filter_column_symm.rs index 159e40cd..be71bb80 100644 --- a/src/filter1d/avx/filter_column_symm.rs +++ b/src/filter1d/avx/filter_column_symm.rs @@ -81,7 +81,7 @@ pub(crate) fn filter_column_avx_symm_u8_f32( } #[target_feature(enable = "avx2")] -unsafe fn filter_column_avx_symm_u8_f32_impl_def( +fn filter_column_avx_symm_u8_f32_impl_def( arena: Arena, arena_src: &[&[u8]], dst: &mut [u8], @@ -101,7 +101,7 @@ unsafe fn filter_column_avx_symm_u8_f32_impl_def( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_column_avx_symm_u8_f32_impl_fma( +fn filter_column_avx_symm_u8_f32_impl_fma( arena: Arena, arena_src: &[&[u8]], dst: &mut [u8], @@ -125,7 +125,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, arena: Arena, arena_src: &[&[u8]], @@ -134,190 +134,219 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let image_width = image_size.width * arena.components; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let mut cx = 0usize; - - let ref0 = arena_src.get_unchecked(half_len); - - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - - while cx + 96 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source = _mm256_load_pack_x3(v_src.as_ptr()); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); - let mut k2 = _mm256_mul_epi8_by_ps_x4::(source.2, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm256_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - let v_source1 = _mm256_load_pack_x3( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr(), + unsafe { + let image_width = image_size.width * arena.components; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let mut cx = 0usize; + + let ref0 = arena_src.get_unchecked(half_len); + + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + + while cx + 96 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source = _mm256_load_pack_x3(v_src.as_ptr()); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); + let mut k2 = _mm256_mul_epi8_by_ps_x4::(source.2, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_load_pack_x3( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr(), + ); + let v_source1 = _mm256_load_pack_x3( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr(), + ); + k0 = _mm256_mul_add_symm_epi8_by_ps_x4::( + k0, + v_source0.0, + v_source1.0, + coeff, + ); + k1 = _mm256_mul_add_symm_epi8_by_ps_x4::( + k1, + v_source0.1, + v_source1.1, + coeff, + ); + k2 = _mm256_mul_add_symm_epi8_by_ps_x4::( + k2, + v_source0.2, + v_source1.2, + coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x3( + dst_ptr0, + ( + _mm256_pack_ps_x4_epi8(k0), + _mm256_pack_ps_x4_epi8(k1), + _mm256_pack_ps_x4_epi8(k2), + ), ); - k0 = _mm256_mul_add_symm_epi8_by_ps_x4::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epi8_by_ps_x4::(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm256_mul_add_symm_epi8_by_ps_x4::(k2, v_source0.2, v_source1.2, coeff); + cx += 96; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x3( - dst_ptr0, - ( - _mm256_pack_ps_x4_epi8(k0), - _mm256_pack_ps_x4_epi8(k1), - _mm256_pack_ps_x4_epi8(k2), - ), - ); - cx += 96; - } - - while cx + 64 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source = _mm256_load_pack_x2(v_src.as_ptr()); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm256_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - let v_source1 = _mm256_load_pack_x2( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr(), + while cx + 64 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source = _mm256_load_pack_x2(v_src.as_ptr()); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_load_pack_x2( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr(), + ); + let v_source1 = _mm256_load_pack_x2( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr(), + ); + k0 = _mm256_mul_add_symm_epi8_by_ps_x4::( + k0, + v_source0.0, + v_source1.0, + coeff, + ); + k1 = _mm256_mul_add_symm_epi8_by_ps_x4::( + k1, + v_source0.1, + v_source1.1, + coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x2( + dst_ptr0, + (_mm256_pack_ps_x4_epi8(k0), _mm256_pack_ps_x4_epi8(k1)), ); - k0 = _mm256_mul_add_symm_epi8_by_ps_x4::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epi8_by_ps_x4::(k1, v_source0.1, v_source1.1, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x2( - dst_ptr0, - (_mm256_pack_ps_x4_epi8(k0), _mm256_pack_ps_x4_epi8(k1)), - ); - cx += 64; - } - - while cx + 32 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_loadu_si256( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, - ); - let v_source1 = _mm256_loadu_si256( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const __m256i, - ); - k0 = _mm256_mul_add_symm_epi8_by_ps_x4::(k0, v_source0, v_source1, coeff); + while cx + 32 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm256_loadu_si256(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() + as *const __m256i); + let v_source1 = _mm256_loadu_si256( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const __m256i, + ); + k0 = _mm256_mul_add_symm_epi8_by_ps_x4::(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x4_epi8(k0)); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x4_epi8(k0)); - cx += 32; - } - - while cx + 16 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi8_by_ps_x4::(source_0, _mm256_castps256_ps128(coeff)); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source_0 = _mm_loadu_si128( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - let v_source_1 = _mm_loadu_si128( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const __m128i, - ); - k0 = _mm_mul_add_symm_epi8_by_ps_x4::( - k0, - v_source_0, - v_source_1, - _mm_set1_ps(coeff.weight), - ); + while cx + 16 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi8_by_ps_x4::(source_0, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source_0 = + _mm_loadu_si128(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() + as *const __m128i); + let v_source_1 = _mm_loadu_si128( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const __m128i, + ); + k0 = _mm_mul_add_symm_epi8_by_ps_x4::( + k0, + v_source_0, + v_source_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); + cx += 16; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); - cx += 16; - } - - while cx + 4 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source_0 = _mm_loadu_si32(v_src.as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_ps(source_0, _mm256_castps256_ps128(coeff)); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source_0 = _mm_loadu_si32( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source_1 = _mm_loadu_si32( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi8_by_ps::( - k0, - v_source_0, - v_source_1, - _mm_set1_ps(coeff.weight), - ); + while cx + 4 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source_0 = _mm_loadu_si32(v_src.as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_ps(source_0, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source_0 = _mm_loadu_si32( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source_1 = _mm_loadu_si32( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_ps::( + k0, + v_source_0, + v_source_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si32(dst_ptr as *mut _, _mm_pack_ps_epi8(k0)); + cx += 4; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si32(dst_ptr as *mut _, _mm_pack_ps_epi8(k0)); - cx += 4; - } + let coeff = scanned_kernel.get_unchecked(half_len).weight; - let coeff = scanned_kernel.get_unchecked(half_len).weight; + for x in cx..image_width { + let v_src = ref0.get_unchecked(x..); - for x in cx..image_width { - let v_src = ref0.get_unchecked(x..); + let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff); - let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff); + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + k0 = mlaf( + k0, + ((*arena_src.get_unchecked(i).get_unchecked(x)) as f32) + .add(*arena_src.get_unchecked(rollback).get_unchecked(x) as f32), + coeff.weight, + ); + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - k0 = mlaf( - k0, - ((*arena_src.get_unchecked(i).get_unchecked(x)) as f32) - .add(*arena_src.get_unchecked(rollback).get_unchecked(x) as f32), - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/avx/filter_column_symm_approx.rs b/src/filter1d/avx/filter_column_symm_approx.rs index 5995d619..c7de7187 100644 --- a/src/filter1d/avx/filter_column_symm_approx.rs +++ b/src/filter1d/avx/filter_column_symm_approx.rs @@ -30,7 +30,6 @@ use crate::avx::{ _mm256_load_pack_x2, _mm256_load_pack_x4, _mm256_store_pack_x2, _mm256_store_pack_x4, }; use crate::filter1d::arena::Arena; -use crate::filter1d::avx::filter_column_symm_approx_uq0_7::filter_column_avx_symm_u8_uq0_7; use crate::filter1d::avx::sse_utils::{ _mm_mul_add_symm_epi8_by_epi16_x2, _mm_mul_add_symm_epi8_by_epi16_x4, _mm_mul_epi8_by_epi16_x2, _mm_mul_epi8_by_epi16_x4, _mm_pack_epi32_epi8, _mm_pack_epi32_x2_epi8, @@ -38,7 +37,6 @@ use crate::filter1d::avx::sse_utils::{ use crate::filter1d::avx::utils::{ _mm256_mul_add_symm_epi8_by_epi16_x4, _mm256_mul_epi8_by_epi16_x4, _mm256_pack_epi32_x4_epi8, }; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; @@ -51,29 +49,16 @@ pub(crate) fn filter_column_avx_symm_u8_i32_app( dst: &mut [u8], image_size: ImageSize, filter_region: FilterRegion, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { - if scanned_kernel.len() <= 7 { - let is_all_positive = scanned_kernel.iter().all(|&x| x.weight > 0); - if is_all_positive { - return filter_column_avx_symm_u8_uq0_7( - arena, - arena_src, - dst, - image_size, - filter_region, - scanned_kernel, - ); - } - } filter_column_avx_symm_u8_i32_impl( arena, arena_src, dst, image_size, filter_region, - scanned_kernel, + kernel, ); } } @@ -85,27 +70,19 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; - let length = scanned_kernel.len(); + let length = kernel.len(); let half_len = length / 2; let ref0 = arena_src.get_unchecked(half_len); - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.weight.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) - }) - .collect::>(); - let mut cx = 0usize; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(half_len)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(half_len)); while cx + 128 <= image_width { let v_src = ref0.get_unchecked(cx..); @@ -118,7 +95,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm256_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = _mm256_load_pack_x4( @@ -155,7 +132,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm256_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = _mm256_load_pack_x2( @@ -184,7 +161,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm256_loadu_si256( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, ); @@ -210,7 +187,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si128( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -236,7 +213,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si64( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -262,7 +239,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si32( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -280,19 +257,23 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( cx += 4; } - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); for x in cx..image_width { let v_src = ref0.get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(wb as i32); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(wb as i32) .add(k0); } diff --git a/src/filter1d/avx/filter_column_symm_approx_uq0_7.rs b/src/filter1d/avx/filter_column_symm_approx_uq0_7.rs index ccc3b8f9..2aacbaae 100644 --- a/src/filter1d/avx/filter_column_symm_approx_uq0_7.rs +++ b/src/filter1d/avx/filter_column_symm_approx_uq0_7.rs @@ -28,7 +28,6 @@ */ use crate::avx::{_mm256_load_pack_x2, _mm256_store_pack_x2}; use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::img_size::ImageSize; use std::arch::x86_64::*; @@ -40,34 +39,16 @@ pub(crate) fn filter_column_avx_symm_u8_uq0_7( dst: &mut [u8], image_size: ImageSize, filter_region: FilterRegion, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8).min(i8::MAX as i32) as i8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } filter_column_avx_symm_u8_i32_impl( arena, arena_src, dst, image_size, filter_region, - &shifted, + kernel, ); } } @@ -79,31 +60,23 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[i8], + kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; - let length = scanned_kernel.len(); + let length = kernel.len(); let half_len = length / 2; let ref0 = arena_src.get_unchecked(half_len); - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[0], z[0], z[0]]) - }) - .collect::>(); - let mut cx = 0usize; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(half_len)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(half_len)); let rnd = _mm256_set1_epi16(1 << 6); - while cx + 64 < image_width { + while cx + 64 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm256_load_pack_x2(v_src.as_ptr()); @@ -138,7 +111,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm256_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = _mm256_load_pack_x2( @@ -178,7 +151,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( cx += 64; } - while cx + 32 < image_width { + while cx + 32 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); @@ -193,7 +166,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm256_loadu_si256( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, ); @@ -221,7 +194,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( cx += 32; } - while cx + 16 < image_width { + while cx + 16 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); @@ -242,7 +215,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si128( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -270,7 +243,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( cx += 16; } - while cx + 8 < image_width { + while cx + 8 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm_loadu_si64(v_src.as_ptr() as *const _); @@ -284,7 +257,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si64( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -310,19 +283,23 @@ unsafe fn filter_column_avx_symm_u8_i32_impl( cx += 8; } - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); for x in cx..image_width { let v_src = ref0.get_unchecked(x..); + let cb = coeff.to_ne_bytes(); + let wb = i8::from_ne_bytes([cb[0]]); - let mut k0 = ((*v_src.get_unchecked(0)) as u16).mul(coeff as u16); + let mut k0 = ((*v_src.get_unchecked(0)) as u16).mul(wb as u16); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); + let cb = coeff.to_ne_bytes(); + let wb = i8::from_ne_bytes([cb[0]]); let rollback = length - i - 1; k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as u16) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as u16) - .mul(coeff as u16) + .mul(wb as u16) .add(k0); } diff --git a/src/filter1d/avx/filter_column_symm_approx_x2.rs b/src/filter1d/avx/filter_column_symm_approx_x2.rs index 68b24a23..933eb413 100644 --- a/src/filter1d/avx/filter_column_symm_approx_x2.rs +++ b/src/filter1d/avx/filter_column_symm_approx_x2.rs @@ -36,7 +36,6 @@ use crate::filter1d::avx::utils::{ _mm256_mul_add_symm_epi8_by_epi16_x4, _mm256_mul_epi8_by_epi16_x4, _mm256_pack_epi32_x4_epi8, }; use crate::filter1d::filter_1d_column_handler::FilterBrows; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; use std::arch::x86_64::*; @@ -48,43 +47,28 @@ pub(crate) fn filter_column_avx_symm_u8_i32_app_x2( dst: &mut [u8], image_size: ImageSize, dst_stride: usize, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { - filter_column_avx_symm_u8_i32_impl_x2( - arena, - brows, - dst, - image_size, - dst_stride, - scanned_kernel, - ); + filter_column_avx_symm_u8_i32_impl_x2(arena, brows, dst, image_size, dst_stride, kernel); } } #[target_feature(enable = "avx2")] -unsafe fn filter_column_avx_symm_u8_i32_impl_x2( +fn filter_column_avx_symm_u8_i32_impl_x2( arena: Arena, brows: FilterBrows, dst: &mut [u8], image_size: ImageSize, dst_stride: usize, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; - let length = scanned_kernel.len(); + let length = kernel.len(); let half_len = length / 2; - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.weight.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) - }) - .collect::>(); - let brows0 = brows.brows[0]; let brows1 = brows.brows[1]; @@ -95,7 +79,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( let ref0 = brows0.get_unchecked(half_len); let ref1 = brows1.get_unchecked(half_len); - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(half_len)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(half_len)); while cx + 64 <= image_width { let v_src0 = ref0.get_unchecked(cx..); @@ -112,7 +96,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(i)); let v_source0_0 = _mm256_load_pack_x2(brows0.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1_0 = _mm256_load_pack_x2( @@ -165,7 +149,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(i)); let v_source0_0 = _mm256_loadu_si256( brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, @@ -205,7 +189,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0_0 = _mm_loadu_si128( brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -243,7 +227,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0_0 = _mm_loadu_si64( brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -281,7 +265,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0_0 = _mm_loadu_si32( brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -308,25 +292,29 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( } for x in cx..image_width { - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); let v_src0 = ref0.get_unchecked(x..); let v_src1 = ref1.get_unchecked(x..); - let mut k0 = ((*v_src0.get_unchecked(0)) as i32).mul(coeff.weight); - let mut k1 = ((*v_src1.get_unchecked(0)) as i32).mul(coeff.weight); + let mut k0 = ((*v_src0.get_unchecked(0)) as i32).mul(wb as i32); + let mut k1 = ((*v_src1.get_unchecked(0)) as i32).mul(wb as i32); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); k0 = ((*brows0.get_unchecked(i).get_unchecked(x)) as i32) .add((*brows0.get_unchecked(rollback).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(wb as i32) .add(k0); k1 = ((*brows1.get_unchecked(i).get_unchecked(x)) as i32) .add((*brows1.get_unchecked(rollback).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(wb as i32) .add(k1); } diff --git a/src/filter1d/avx/filter_column_symm_f32.rs b/src/filter1d/avx/filter_column_symm_f32.rs index 250db8b9..d3f4e430 100644 --- a/src/filter1d/avx/filter_column_symm_f32.rs +++ b/src/filter1d/avx/filter_column_symm_f32.rs @@ -71,7 +71,7 @@ pub(crate) fn filter_column_avx_symm_f32_f32( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_column_avx_symm_f32_f32_impl_fma( +fn filter_column_avx_symm_f32_f32_impl_fma( arena: Arena, arena_src: &[&[f32]], dst: &mut [f32], @@ -91,7 +91,7 @@ unsafe fn filter_column_avx_symm_f32_f32_impl_fma( } #[target_feature(enable = "avx2")] -unsafe fn filter_column_avx_symm_f32_f32_impl_def( +fn filter_column_avx_symm_f32_f32_impl_def( arena: Arena, arena_src: &[&[f32]], dst: &mut [f32], @@ -115,7 +115,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, arena: Arena, arena_src: &[&[f32]], diff --git a/src/filter1d/avx/filter_column_symm_f32_f64.rs b/src/filter1d/avx/filter_column_symm_f32_f64.rs index e5e9de14..33fa6587 100644 --- a/src/filter1d/avx/filter_column_symm_f32_f64.rs +++ b/src/filter1d/avx/filter_column_symm_f32_f64.rs @@ -68,7 +68,7 @@ pub(crate) fn filter_column_avx_symm_f32_f64( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_column_avx_symm_f32_f64_impl_fma( +fn filter_column_avx_symm_f32_f64_impl_fma( arena: Arena, arena_src: &[&[f32]], dst: &mut [f32], @@ -88,7 +88,7 @@ unsafe fn filter_column_avx_symm_f32_f64_impl_fma( } #[target_feature(enable = "avx2")] -unsafe fn filter_column_avx_symm_f32_f64_impl_def( +fn filter_column_avx_symm_f32_f64_impl_def( arena: Arena, arena_src: &[&[f32]], dst: &mut [f32], @@ -112,7 +112,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, arena: Arena, arena_src: &[&[f32]], @@ -133,7 +133,7 @@ impl ExecutionUnit { let mut cx = 0usize; - while cx + 16 < dst_stride { + while cx + 16 <= dst_stride { let v_src = ref0.get_unchecked(cx..); let source = _mm256_load_pack_ps_x2(v_src.as_ptr()); @@ -209,7 +209,7 @@ impl ExecutionUnit { cx += 16; } - while cx + 8 < dst_stride { + while cx + 8 <= dst_stride { let v_src = ref0.get_unchecked(cx..); let source = _mm256_loadu_ps(v_src.as_ptr()); @@ -258,15 +258,15 @@ impl ExecutionUnit { cx += 8; } - while cx + 4 < dst_stride { + while cx + 4 <= dst_stride { let v_src = ref0.get_unchecked(cx..); let source_0 = _mm_loadu_ps(v_src.as_ptr()); - let mut k0 = _mm_mul_pd(_mm_cvtps_pd(source_0), _mm256_castpd256_pd128(coeff)); + let mut k0 = _mm256_mul_pd(_mm256_cvtps_pd(source_0), coeff); for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set_pd1(scanned_kernel.get_unchecked(i).weight); + let coeff = _mm256_set1_pd(scanned_kernel.get_unchecked(i).weight); let v_source_0 = _mm_loadu_ps(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source_1 = _mm_loadu_ps( @@ -275,15 +275,15 @@ impl ExecutionUnit { .get_unchecked(cx..) .as_ptr(), ); - k0 = _mm_opt_fmla_pd::( + k0 = _mm256_opt_fmla_pd::( k0, - _mm_add_pd(_mm_cvtps_pd(v_source_0), _mm_cvtps_pd(v_source_1)), + _mm256_add_pd(_mm256_cvtps_pd(v_source_0), _mm256_cvtps_pd(v_source_1)), coeff, ); } let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_ps(dst_ptr, _mm_cvtpd_ps(k0)); + _mm_storeu_ps(dst_ptr, _mm256_cvtpd_ps(k0)); cx += 4; } diff --git a/src/filter1d/avx/filter_column_symm_u16.rs b/src/filter1d/avx/filter_column_symm_u16.rs index 3ecb9dd1..0b15f9e1 100644 --- a/src/filter1d/avx/filter_column_symm_u16.rs +++ b/src/filter1d/avx/filter_column_symm_u16.rs @@ -82,7 +82,7 @@ pub(crate) fn filter_column_avx_symm_u16_f32( } #[target_feature(enable = "avx2")] -unsafe fn filter_column_avx_symm_u16_f32_impl_def( +fn filter_column_avx_symm_u16_f32_impl_def( arena: Arena, arena_src: &[&[u16]], dst: &mut [u16], @@ -102,7 +102,7 @@ unsafe fn filter_column_avx_symm_u16_f32_impl_def( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_column_avx_symm_u16_f32_impl_fma( +fn filter_column_avx_symm_u16_f32_impl_fma( arena: Arena, arena_src: &[&[u16]], dst: &mut [u16], @@ -126,7 +126,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, arena: Arena, arena_src: &[&[u16]], @@ -135,232 +135,279 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let image_width = image_size.width * arena.components; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let ref0 = arena_src.get_unchecked(half_len); - - let mut cx = 0usize; - - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - - while cx + 64 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source = _mm256_load_pack_x4(v_src.as_ptr() as *const _); - let mut k0 = _mm256_mul_epi16_by_ps_x2::(source.0, coeff); - let mut k1 = _mm256_mul_epi16_by_ps_x2::(source.1, coeff); - let mut k2 = _mm256_mul_epi16_by_ps_x2::(source.2, coeff); - let mut k3 = _mm256_mul_epi16_by_ps_x2::(source.3, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_load_pack_x4( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source1 = _mm256_load_pack_x4( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const _, + unsafe { + let image_width = image_size.width * arena.components; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let ref0 = arena_src.get_unchecked(half_len); + + let mut cx = 0usize; + + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + + while cx + 64 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source = _mm256_load_pack_x4(v_src.as_ptr() as *const _); + let mut k0 = _mm256_mul_epi16_by_ps_x2::(source.0, coeff); + let mut k1 = _mm256_mul_epi16_by_ps_x2::(source.1, coeff); + let mut k2 = _mm256_mul_epi16_by_ps_x2::(source.2, coeff); + let mut k3 = _mm256_mul_epi16_by_ps_x2::(source.3, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_load_pack_x4( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source1 = _mm256_load_pack_x4( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k0, + v_source0.0, + v_source1.0, + coeff, + ); + k1 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k1, + v_source0.1, + v_source1.1, + coeff, + ); + k2 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k2, + v_source0.2, + v_source1.2, + coeff, + ); + k3 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k3, + v_source0.3, + v_source1.3, + coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x4( + dst_ptr0 as *mut _, + ( + _mm256_pack_ps_x2_epi16(k0), + _mm256_pack_ps_x2_epi16(k1), + _mm256_pack_ps_x2_epi16(k2), + _mm256_pack_ps_x2_epi16(k3), + ), ); - k0 = _mm256_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm256_mul_add_symm_epi16_by_ps_x2::(k2, v_source0.2, v_source1.2, coeff); - k3 = _mm256_mul_add_symm_epi16_by_ps_x2::(k3, v_source0.3, v_source1.3, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x4( - dst_ptr0 as *mut _, - ( - _mm256_pack_ps_x2_epi16(k0), - _mm256_pack_ps_x2_epi16(k1), - _mm256_pack_ps_x2_epi16(k2), - _mm256_pack_ps_x2_epi16(k3), - ), - ); - cx += 64; - } - - while cx + 48 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source = _mm256_load_pack_x3(v_src.as_ptr() as *const _); - let mut k0 = _mm256_mul_epi16_by_ps_x2::(source.0, coeff); - let mut k1 = _mm256_mul_epi16_by_ps_x2::(source.1, coeff); - let mut k2 = _mm256_mul_epi16_by_ps_x2::(source.2, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_load_pack_x3( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source1 = _mm256_load_pack_x3( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const _, + while cx + 48 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source = _mm256_load_pack_x3(v_src.as_ptr() as *const _); + let mut k0 = _mm256_mul_epi16_by_ps_x2::(source.0, coeff); + let mut k1 = _mm256_mul_epi16_by_ps_x2::(source.1, coeff); + let mut k2 = _mm256_mul_epi16_by_ps_x2::(source.2, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_load_pack_x3( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source1 = _mm256_load_pack_x3( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k0, + v_source0.0, + v_source1.0, + coeff, + ); + k1 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k1, + v_source0.1, + v_source1.1, + coeff, + ); + k2 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k2, + v_source0.2, + v_source1.2, + coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x3( + dst_ptr0 as *mut _, + ( + _mm256_pack_ps_x2_epi16(k0), + _mm256_pack_ps_x2_epi16(k1), + _mm256_pack_ps_x2_epi16(k2), + ), ); - k0 = _mm256_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm256_mul_add_symm_epi16_by_ps_x2::(k2, v_source0.2, v_source1.2, coeff); + cx += 48; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x3( - dst_ptr0 as *mut _, - ( - _mm256_pack_ps_x2_epi16(k0), - _mm256_pack_ps_x2_epi16(k1), - _mm256_pack_ps_x2_epi16(k2), - ), - ); - cx += 48; - } - - while cx + 32 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source = _mm256_load_pack_x2(v_src.as_ptr() as *const _); - let mut k0 = _mm256_mul_epi16_by_ps_x2::(source.0, coeff); - let mut k1 = _mm256_mul_epi16_by_ps_x2::(source.1, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_load_pack_x2( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source1 = _mm256_load_pack_x2( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const _, + while cx + 32 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source = _mm256_load_pack_x2(v_src.as_ptr() as *const _); + let mut k0 = _mm256_mul_epi16_by_ps_x2::(source.0, coeff); + let mut k1 = _mm256_mul_epi16_by_ps_x2::(source.1, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_load_pack_x2( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source1 = _mm256_load_pack_x2( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k0, + v_source0.0, + v_source1.0, + coeff, + ); + k1 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k1, + v_source0.1, + v_source1.1, + coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x2( + dst_ptr0 as *mut _, + (_mm256_pack_ps_x2_epi16(k0), _mm256_pack_ps_x2_epi16(k1)), ); - k0 = _mm256_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x2( - dst_ptr0 as *mut _, - (_mm256_pack_ps_x2_epi16(k0), _mm256_pack_ps_x2_epi16(k1)), - ); - cx += 32; - } - - while cx + 16 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); - let mut k0 = _mm256_mul_epi16_by_ps_x2::(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_loadu_si256( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, - ); - let v_source1 = _mm256_loadu_si256( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const __m256i, - ); - k0 = _mm256_mul_add_symm_epi16_by_ps_x2::(k0, v_source0, v_source1, coeff); + while cx + 16 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); + let mut k0 = _mm256_mul_epi16_by_ps_x2::(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm256_loadu_si256(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() + as *const __m256i); + let v_source1 = _mm256_loadu_si256( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const __m256i, + ); + k0 = _mm256_mul_add_symm_epi16_by_ps_x2::(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x2_epi16(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x2_epi16(k0)); - cx += 16; - } - - while cx + 8 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi16_by_ps_x2::(source_0, _mm256_castps256_ps128(coeff)); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source_0 = _mm_loadu_si128( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - let v_source_1 = _mm_loadu_si128( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const __m128i, - ); - k0 = _mm_mul_add_symm_epi16_by_ps_x2::( - k0, - v_source_0, - v_source_1, - _mm_set1_ps(coeff.weight), - ); + while cx + 8 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi16_by_ps_x2::(source_0, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source_0 = + _mm_loadu_si128(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() + as *const __m128i); + let v_source_1 = _mm_loadu_si128( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const __m128i, + ); + k0 = _mm_mul_add_symm_epi16_by_ps_x2::( + k0, + v_source_0, + v_source_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x2_epi16(k0)); + cx += 8; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x2_epi16(k0)); - cx += 8; - } - - while cx + 4 < image_width { - let v_src = ref0.get_unchecked(cx..); - - let source_0 = _mm_loadu_si64(v_src.as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps::(source_0, _mm256_castps256_ps128(coeff)); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source_0 = _mm_loadu_si64( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source_1 = _mm_loadu_si64( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi16_by_ps::( - k0, - v_source_0, - v_source_1, - _mm_set1_ps(coeff.weight), - ); + while cx + 4 <= image_width { + let v_src = ref0.get_unchecked(cx..); + + let source_0 = _mm_loadu_si64(v_src.as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps::(source_0, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source_0 = _mm_loadu_si64( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source_1 = _mm_loadu_si64( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps::( + k0, + v_source_0, + v_source_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr as *mut _, _mm_pack_ps_epi16(k0)); + cx += 4; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr as *mut _, _mm_pack_ps_epi16(k0)); - cx += 4; - } + let coeff = scanned_kernel.get_unchecked(half_len).weight; - let coeff = scanned_kernel.get_unchecked(half_len).weight; + for x in cx..image_width { + let v_src = ref0.get_unchecked(x..); - for x in cx..image_width { - let v_src = ref0.get_unchecked(x..); + let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff); - let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff); + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + k0 = mlaf( + k0, + ((*arena_src.get_unchecked(i).get_unchecked(x)) as f32) + .add(*arena_src.get_unchecked(rollback).get_unchecked(x) as f32), + coeff.weight, + ); + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - k0 = mlaf( - k0, - ((*arena_src.get_unchecked(i).get_unchecked(x)) as f32) - .add(*arena_src.get_unchecked(rollback).get_unchecked(x) as f32), - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/avx/filter_column_symm_u16_x2.rs b/src/filter1d/avx/filter_column_symm_u16_x2.rs index 912519c1..62235ef5 100644 --- a/src/filter1d/avx/filter_column_symm_u16_x2.rs +++ b/src/filter1d/avx/filter_column_symm_u16_x2.rs @@ -79,7 +79,7 @@ pub(crate) fn filter_column_avx_symm_u16_f32_x2( } #[target_feature(enable = "avx2")] -unsafe fn filter_column_avx_symm_u16_f32_impl_def_x2( +fn filter_column_avx_symm_u16_f32_impl_def_x2( arena: Arena, brows: FilterBrows, dst: &mut [u16], @@ -92,7 +92,7 @@ unsafe fn filter_column_avx_symm_u16_f32_impl_def_x2( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_column_avx_symm_u16_f32_impl_fma_x2( +fn filter_column_avx_symm_u16_f32_impl_fma_x2( arena: Arena, brows: FilterBrows, dst: &mut [u16], @@ -109,7 +109,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, arena: Arena, brows: FilterBrows, @@ -118,257 +118,273 @@ impl ExecutionUnit { dst_stride: usize, scanned_kernel: &[ScanPoint1d], ) { - let image_width = image_size.width * arena.components; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let mut cx = 0usize; - - let (dst0, dst1) = dst.split_at_mut(dst_stride); - - let brows0 = brows.brows[0]; - let brows1 = brows.brows[1]; - - let ref0 = brows0.get_unchecked(half_len); - let ref1 = brows1.get_unchecked(half_len); - - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - - while cx + 32 < image_width { - let v_src0 = ref0.get_unchecked(cx..); - let v_src1 = ref1.get_unchecked(cx..); - - let source0 = _mm256_load_pack_x2(v_src0.as_ptr() as *const _); - let source1 = _mm256_load_pack_x2(v_src1.as_ptr() as *const _); - - let mut k0_0 = _mm256_mul_epi16_by_ps_x2::(source0.0, coeff); - let mut k1_0 = _mm256_mul_epi16_by_ps_x2::(source0.1, coeff); - - let mut k0_1 = _mm256_mul_epi16_by_ps_x2::(source1.0, coeff); - let mut k1_1 = _mm256_mul_epi16_by_ps_x2::(source1.1, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0_0 = _mm256_load_pack_x2( - brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source1_0 = _mm256_load_pack_x2( - brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, - ); - - let v_source0_1 = _mm256_load_pack_x2( - brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source1_1 = _mm256_load_pack_x2( - brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, - ); - - k0_0 = _mm256_mul_add_symm_epi16_by_ps_x2::( - k0_0, - v_source0_0.0, - v_source1_0.0, - coeff, - ); - k1_0 = _mm256_mul_add_symm_epi16_by_ps_x2::( - k1_0, - v_source0_0.1, - v_source1_0.1, - coeff, + unsafe { + let image_width = image_size.width * arena.components; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let mut cx = 0usize; + + let (dst0, dst1) = dst.split_at_mut(dst_stride); + + let brows0 = brows.brows[0]; + let brows1 = brows.brows[1]; + + let ref0 = brows0.get_unchecked(half_len); + let ref1 = brows1.get_unchecked(half_len); + + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + + while cx + 32 < image_width { + let v_src0 = ref0.get_unchecked(cx..); + let v_src1 = ref1.get_unchecked(cx..); + + let source0 = _mm256_load_pack_x2(v_src0.as_ptr() as *const _); + let source1 = _mm256_load_pack_x2(v_src1.as_ptr() as *const _); + + let mut k0_0 = _mm256_mul_epi16_by_ps_x2::(source0.0, coeff); + let mut k1_0 = _mm256_mul_epi16_by_ps_x2::(source0.1, coeff); + + let mut k0_1 = _mm256_mul_epi16_by_ps_x2::(source1.0, coeff); + let mut k1_1 = _mm256_mul_epi16_by_ps_x2::(source1.1, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0_0 = _mm256_load_pack_x2( + brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source1_0 = _mm256_load_pack_x2( + brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, + ); + + let v_source0_1 = _mm256_load_pack_x2( + brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source1_1 = _mm256_load_pack_x2( + brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, + ); + + k0_0 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k0_0, + v_source0_0.0, + v_source1_0.0, + coeff, + ); + k1_0 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k1_0, + v_source0_0.1, + v_source1_0.1, + coeff, + ); + + k0_1 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k0_1, + v_source0_1.0, + v_source1_1.0, + coeff, + ); + k1_1 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k1_1, + v_source0_1.1, + v_source1_1.1, + coeff, + ); + } + + let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); + let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); + + _mm256_store_pack_x2( + dst_ptr0 as *mut _, + (_mm256_pack_ps_x2_epi16(k0_0), _mm256_pack_ps_x2_epi16(k1_0)), ); - k0_1 = _mm256_mul_add_symm_epi16_by_ps_x2::( - k0_1, - v_source0_1.0, - v_source1_1.0, - coeff, - ); - k1_1 = _mm256_mul_add_symm_epi16_by_ps_x2::( - k1_1, - v_source0_1.1, - v_source1_1.1, - coeff, + _mm256_store_pack_x2( + dst_ptr1 as *mut _, + (_mm256_pack_ps_x2_epi16(k0_1), _mm256_pack_ps_x2_epi16(k1_1)), ); + cx += 32; } - let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); - let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); - - _mm256_store_pack_x2( - dst_ptr0 as *mut _, - (_mm256_pack_ps_x2_epi16(k0_0), _mm256_pack_ps_x2_epi16(k1_0)), - ); - - _mm256_store_pack_x2( - dst_ptr1 as *mut _, - (_mm256_pack_ps_x2_epi16(k0_1), _mm256_pack_ps_x2_epi16(k1_1)), - ); - cx += 32; - } - - while cx + 16 < image_width { - let v_src0 = ref0.get_unchecked(cx..); - let v_src1 = ref1.get_unchecked(cx..); - - let source0 = _mm256_loadu_si256(v_src0.as_ptr() as *const __m256i); - let source1 = _mm256_loadu_si256(v_src1.as_ptr() as *const __m256i); - - let mut k0 = _mm256_mul_epi16_by_ps_x2::(source0, coeff); - let mut k1 = _mm256_mul_epi16_by_ps_x2::(source1, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0_0 = _mm256_loadu_si256( - brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, - ); - let v_source1_0 = - _mm256_loadu_si256(brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() - as *const __m256i); - let v_source0_1 = _mm256_loadu_si256( - brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, - ); - let v_source1_1 = - _mm256_loadu_si256(brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() - as *const __m256i); - k0 = _mm256_mul_add_symm_epi16_by_ps_x2::(k0, v_source0_0, v_source1_0, coeff); - k1 = _mm256_mul_add_symm_epi16_by_ps_x2::(k1, v_source0_1, v_source1_1, coeff); + while cx + 16 < image_width { + let v_src0 = ref0.get_unchecked(cx..); + let v_src1 = ref1.get_unchecked(cx..); + + let source0 = _mm256_loadu_si256(v_src0.as_ptr() as *const __m256i); + let source1 = _mm256_loadu_si256(v_src1.as_ptr() as *const __m256i); + + let mut k0 = _mm256_mul_epi16_by_ps_x2::(source0, coeff); + let mut k1 = _mm256_mul_epi16_by_ps_x2::(source1, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0_0 = _mm256_loadu_si256( + brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, + ); + let v_source1_0 = _mm256_loadu_si256( + brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() + as *const __m256i, + ); + let v_source0_1 = _mm256_loadu_si256( + brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, + ); + let v_source1_1 = _mm256_loadu_si256( + brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() + as *const __m256i, + ); + k0 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k0, + v_source0_0, + v_source1_0, + coeff, + ); + k1 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k1, + v_source0_1, + v_source1_1, + coeff, + ); + } + + let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); + let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); + + _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x2_epi16(k0)); + _mm256_storeu_si256(dst_ptr1 as *mut __m256i, _mm256_pack_ps_x2_epi16(k1)); + + cx += 16; } - let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); - let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); - - _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x2_epi16(k0)); - _mm256_storeu_si256(dst_ptr1 as *mut __m256i, _mm256_pack_ps_x2_epi16(k1)); - - cx += 16; - } - - while cx + 8 < image_width { - let v_src0 = ref0.get_unchecked(cx..); - let v_src1 = ref1.get_unchecked(cx..); - - let source_0 = _mm_loadu_si128(v_src0.as_ptr() as *const __m128i); - let source_1 = _mm_loadu_si128(v_src1.as_ptr() as *const __m128i); - - let mut k0 = _mm_mul_epi16_by_ps_x2::(source_0, _mm256_castps256_ps128(coeff)); - let mut k1 = _mm_mul_epi16_by_ps_x2::(source_1, _mm256_castps256_ps128(coeff)); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source0_0 = _mm_loadu_si128( - brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - let v_source1_0 = - _mm_loadu_si128(brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() - as *const __m128i); - let v_source0_1 = _mm_loadu_si128( - brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - let v_source1_1 = - _mm_loadu_si128(brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() - as *const __m128i); - k0 = _mm_mul_add_symm_epi16_by_ps_x2::( - k0, - v_source0_0, - v_source1_0, - _mm_set1_ps(coeff.weight), - ); - k1 = _mm_mul_add_symm_epi16_by_ps_x2::( - k1, - v_source0_1, - v_source1_1, - _mm_set1_ps(coeff.weight), - ); + while cx + 8 < image_width { + let v_src0 = ref0.get_unchecked(cx..); + let v_src1 = ref1.get_unchecked(cx..); + + let source_0 = _mm_loadu_si128(v_src0.as_ptr() as *const __m128i); + let source_1 = _mm_loadu_si128(v_src1.as_ptr() as *const __m128i); + + let mut k0 = _mm_mul_epi16_by_ps_x2::(source_0, _mm256_castps256_ps128(coeff)); + let mut k1 = _mm_mul_epi16_by_ps_x2::(source_1, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source0_0 = _mm_loadu_si128( + brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, + ); + let v_source1_0 = _mm_loadu_si128( + brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() + as *const __m128i, + ); + let v_source0_1 = _mm_loadu_si128( + brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, + ); + let v_source1_1 = _mm_loadu_si128( + brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() + as *const __m128i, + ); + k0 = _mm_mul_add_symm_epi16_by_ps_x2::( + k0, + v_source0_0, + v_source1_0, + _mm_set1_ps(coeff.weight), + ); + k1 = _mm_mul_add_symm_epi16_by_ps_x2::( + k1, + v_source0_1, + v_source1_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); + let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); + + _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_ps_x2_epi16(k0)); + _mm_storeu_si128(dst_ptr1 as *mut __m128i, _mm_pack_ps_x2_epi16(k1)); + + cx += 8; } - let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); - let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); - - _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_ps_x2_epi16(k0)); - _mm_storeu_si128(dst_ptr1 as *mut __m128i, _mm_pack_ps_x2_epi16(k1)); - - cx += 8; - } - - while cx + 4 < image_width { - let v_src0 = ref0.get_unchecked(cx..); - let v_src1 = ref1.get_unchecked(cx..); - - let source_0 = _mm_loadu_si64(v_src0.as_ptr() as *const _); - let source_1 = _mm_loadu_si64(v_src1.as_ptr() as *const _); - - let mut k0 = _mm_mul_epi16_by_ps::(source_0, _mm256_castps256_ps128(coeff)); - let mut k1 = _mm_mul_epi16_by_ps::(source_1, _mm256_castps256_ps128(coeff)); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source0_0 = _mm_loadu_si64( - brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source1_0 = _mm_loadu_si64( - brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source0_1 = _mm_loadu_si64( - brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source1_1 = _mm_loadu_si64( - brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi16_by_ps::( - k0, - v_source0_0, - v_source1_0, - _mm_set1_ps(coeff.weight), - ); - k1 = _mm_mul_add_symm_epi16_by_ps::( - k1, - v_source0_1, - v_source1_1, - _mm_set1_ps(coeff.weight), - ); + while cx + 4 < image_width { + let v_src0 = ref0.get_unchecked(cx..); + let v_src1 = ref1.get_unchecked(cx..); + + let source_0 = _mm_loadu_si64(v_src0.as_ptr() as *const _); + let source_1 = _mm_loadu_si64(v_src1.as_ptr() as *const _); + + let mut k0 = _mm_mul_epi16_by_ps::(source_0, _mm256_castps256_ps128(coeff)); + let mut k1 = _mm_mul_epi16_by_ps::(source_1, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source0_0 = _mm_loadu_si64( + brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source1_0 = _mm_loadu_si64( + brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source0_1 = _mm_loadu_si64( + brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source1_1 = _mm_loadu_si64( + brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps::( + k0, + v_source0_0, + v_source1_0, + _mm_set1_ps(coeff.weight), + ); + k1 = _mm_mul_add_symm_epi16_by_ps::( + k1, + v_source0_1, + v_source1_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); + let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); + + _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_ps_epi16(k0)); + _mm_storeu_si64(dst_ptr1 as *mut _, _mm_pack_ps_epi16(k1)); + + cx += 4; } - let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); - let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); - - _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_ps_epi16(k0)); - _mm_storeu_si64(dst_ptr1 as *mut _, _mm_pack_ps_epi16(k1)); - - cx += 4; - } - - let coeff = scanned_kernel.get_unchecked(half_len).weight; - - for x in cx..image_width { - let v_src0 = ref0.get_unchecked(x..); - let v_src1 = ref1.get_unchecked(x..); - - let mut k0 = (*v_src0.get_unchecked(0) as f32).mul(coeff); - let mut k1 = (*v_src1.get_unchecked(0) as f32).mul(coeff); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - k0 = mlaf( - k0, - ((*brows0.get_unchecked(i).get_unchecked(x)) as f32) - .add(*brows0.get_unchecked(rollback).get_unchecked(x) as f32), - coeff.weight, - ); - k1 = mlaf( - k1, - ((*brows1.get_unchecked(i).get_unchecked(x)) as f32) - .add(*brows1.get_unchecked(rollback).get_unchecked(x) as f32), - coeff.weight, - ); + let coeff = scanned_kernel.get_unchecked(half_len).weight; + + for x in cx..image_width { + let v_src0 = ref0.get_unchecked(x..); + let v_src1 = ref1.get_unchecked(x..); + + let mut k0 = (*v_src0.get_unchecked(0) as f32).mul(coeff); + let mut k1 = (*v_src1.get_unchecked(0) as f32).mul(coeff); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + k0 = mlaf( + k0, + ((*brows0.get_unchecked(i).get_unchecked(x)) as f32) + .add(*brows0.get_unchecked(rollback).get_unchecked(x) as f32), + coeff.weight, + ); + k1 = mlaf( + k1, + ((*brows1.get_unchecked(i).get_unchecked(x)) as f32) + .add(*brows1.get_unchecked(rollback).get_unchecked(x) as f32), + coeff.weight, + ); + } + + *dst0.get_unchecked_mut(x) = k0.to_(); + *dst1.get_unchecked_mut(x) = k1.to_(); } - - *dst0.get_unchecked_mut(x) = k0.to_(); - *dst1.get_unchecked_mut(x) = k1.to_(); } } } diff --git a/src/filter1d/avx/filter_column_symm_uq15_u16.rs b/src/filter1d/avx/filter_column_symm_uq15_u16.rs index a0a39c26..2efe07d4 100644 --- a/src/filter1d/avx/filter_column_symm_uq15_u16.rs +++ b/src/filter1d/avx/filter_column_symm_uq15_u16.rs @@ -36,7 +36,6 @@ use crate::filter1d::avx::sse_utils::{ use crate::filter1d::avx::utils::{ _mm256_mul_add_symm_epu16_by_epu16_x4, _mm256_mul_epu16_widen, _mm256_pack_epi32_x4_epu16, }; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; @@ -49,7 +48,7 @@ pub(crate) fn filter_column_avx_symm_uq15_u16( dst: &mut [u16], image_size: ImageSize, filter_region: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[u32], ) { unsafe { filter_column_avx_symm_uq15_u16_impl( @@ -70,7 +69,7 @@ unsafe fn filter_column_avx_symm_uq15_u16_impl( dst: &mut [u16], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[u32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -80,7 +79,7 @@ unsafe fn filter_column_avx_symm_uq15_u16_impl( let ref0 = arena_src.get_unchecked(half_len); - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(half_len).weight as i32); + let coeff = _mm256_set1_epi32(*scanned_kernel.get_unchecked(half_len) as i32); let mut cx = 0usize; @@ -95,7 +94,7 @@ unsafe fn filter_column_avx_symm_uq15_u16_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm256_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0 = _mm256_load_pack_x4( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -133,7 +132,7 @@ unsafe fn filter_column_avx_symm_uq15_u16_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm256_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0 = _mm256_load_pack_x2( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -166,7 +165,7 @@ unsafe fn filter_column_avx_symm_uq15_u16_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm256_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0 = _mm256_loadu_si256( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, ); @@ -192,7 +191,7 @@ unsafe fn filter_column_avx_symm_uq15_u16_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0 = _mm_loadu_si128( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -221,7 +220,7 @@ unsafe fn filter_column_avx_symm_uq15_u16_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0 = _mm_loadu_si64( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -260,14 +259,14 @@ unsafe fn filter_column_avx_symm_uq15_u16_impl( for x in cx..image_width { let v_src = ref0.get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as u32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as u32).mul(coeff); for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as u32) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as u32) - .mul(coeff.weight) + .mul(coeff) .add(k0); } diff --git a/src/filter1d/avx/filter_column_symm_uq15_u16_x2.rs b/src/filter1d/avx/filter_column_symm_uq15_u16_x2.rs index 1ae2635d..e0dbc91b 100644 --- a/src/filter1d/avx/filter_column_symm_uq15_u16_x2.rs +++ b/src/filter1d/avx/filter_column_symm_uq15_u16_x2.rs @@ -35,7 +35,6 @@ use crate::filter1d::avx::utils::{ _mm256_mul_add_symm_epu16_by_epu16_x4, _mm256_mul_epu16_widen, _mm256_pack_epi32_x4_epu16, }; use crate::filter1d::filter_1d_column_handler::FilterBrows; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; use std::arch::x86_64::*; @@ -47,7 +46,7 @@ pub(crate) fn filter_column_avx_symm_uq15_u16_x2( dst: &mut [u16], image_size: ImageSize, dst_stride: usize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[u32], ) { unsafe { filter_column_avx_symm_u8_i32_impl_x2( @@ -68,7 +67,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( dst: &mut [u16], image_size: ImageSize, dst_stride: usize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[u32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -84,7 +83,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( let ref0 = brows0.get_unchecked(half_len); let ref1 = brows1.get_unchecked(half_len); - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(half_len).weight as i32); + let coeff = _mm256_set1_epi32(*scanned_kernel.get_unchecked(half_len) as i32); let mut cx = 0usize; @@ -103,7 +102,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm256_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0_0 = _mm256_load_pack_x2( brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -174,7 +173,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm256_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0_0 = _mm256_loadu_si256( brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, @@ -213,7 +212,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0_0 = _mm_loadu_si128( brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -257,7 +256,7 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0_0 = _mm_loadu_si64( brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -323,20 +322,20 @@ unsafe fn filter_column_avx_symm_u8_i32_impl_x2( let v_src0 = ref0.get_unchecked(x..); let v_src1 = ref1.get_unchecked(x..); - let mut k0 = ((*v_src0.get_unchecked(0)) as u32).mul(coeff.weight); - let mut k1 = ((*v_src1.get_unchecked(0)) as u32).mul(coeff.weight); + let mut k0 = ((*v_src0.get_unchecked(0)) as u32).mul(coeff); + let mut k1 = ((*v_src1.get_unchecked(0)) as u32).mul(coeff); for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; k0 = ((*brows0.get_unchecked(i).get_unchecked(x)) as u32) .add((*brows0.get_unchecked(rollback).get_unchecked(x)) as u32) - .mul(coeff.weight) + .mul(coeff) .add(k0); k1 = ((*brows1.get_unchecked(i).get_unchecked(x)) as u32) .add((*brows1.get_unchecked(rollback).get_unchecked(x)) as u32) - .mul(coeff.weight) + .mul(coeff) .add(k1); } diff --git a/src/filter1d/avx/filter_column_symm_x2.rs b/src/filter1d/avx/filter_column_symm_x2.rs index c217e839..08ec1dc0 100644 --- a/src/filter1d/avx/filter_column_symm_x2.rs +++ b/src/filter1d/avx/filter_column_symm_x2.rs @@ -78,7 +78,7 @@ pub(crate) fn filter_column_avx_symm_u8_f32_x2( } #[target_feature(enable = "avx2")] -unsafe fn filter_column_avx_symm_u8_f32_impl_def_x2( +fn filter_column_avx_symm_u8_f32_impl_def_x2( arena: Arena, brows: FilterBrows, dst: &mut [u8], @@ -91,7 +91,7 @@ unsafe fn filter_column_avx_symm_u8_f32_impl_def_x2( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_column_avx_symm_u8_f32_impl_fma_x2( +fn filter_column_avx_symm_u8_f32_impl_fma_x2( arena: Arena, brows: FilterBrows, dst: &mut [u8], @@ -108,7 +108,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, arena: Arena, brows: FilterBrows, @@ -117,186 +117,202 @@ impl ExecutionUnit { dst_stride: usize, scanned_kernel: &[ScanPoint1d], ) { - let image_width = image_size.width * arena.components; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let mut cx = 0usize; - - let (dst0, dst1) = dst.split_at_mut(dst_stride); - - let brows0 = brows.brows[0]; - let brows1 = brows.brows[1]; - - let ref0 = brows0.get_unchecked(half_len); - let ref1 = brows1.get_unchecked(half_len); - - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - - while cx + 32 < image_width { - let shifted_src0 = ref0.get_unchecked(cx..); - let shifted_src1 = ref1.get_unchecked(cx..); - - let source0 = _mm256_loadu_si256(shifted_src0.as_ptr() as *const __m256i); - let source1 = _mm256_loadu_si256(shifted_src1.as_ptr() as *const __m256i); - - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source0, coeff); - let mut k1 = _mm256_mul_epi8_by_ps_x4::(source1, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0_0 = _mm256_loadu_si256( - brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, - ); - let v_source1_0 = - _mm256_loadu_si256(brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() - as *const __m256i); - let v_source0_1 = _mm256_loadu_si256( - brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, - ); - let v_source1_1 = - _mm256_loadu_si256(brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() - as *const __m256i); - k0 = _mm256_mul_add_symm_epi8_by_ps_x4::(k0, v_source0_0, v_source1_0, coeff); - k1 = _mm256_mul_add_symm_epi8_by_ps_x4::(k1, v_source0_1, v_source1_1, coeff); + unsafe { + let image_width = image_size.width * arena.components; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let mut cx = 0usize; + + let (dst0, dst1) = dst.split_at_mut(dst_stride); + + let brows0 = brows.brows[0]; + let brows1 = brows.brows[1]; + + let ref0 = brows0.get_unchecked(half_len); + let ref1 = brows1.get_unchecked(half_len); + + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + + while cx + 32 < image_width { + let shifted_src0 = ref0.get_unchecked(cx..); + let shifted_src1 = ref1.get_unchecked(cx..); + + let source0 = _mm256_loadu_si256(shifted_src0.as_ptr() as *const __m256i); + let source1 = _mm256_loadu_si256(shifted_src1.as_ptr() as *const __m256i); + + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source0, coeff); + let mut k1 = _mm256_mul_epi8_by_ps_x4::(source1, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0_0 = _mm256_loadu_si256( + brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, + ); + let v_source1_0 = _mm256_loadu_si256( + brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() + as *const __m256i, + ); + let v_source0_1 = _mm256_loadu_si256( + brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, + ); + let v_source1_1 = _mm256_loadu_si256( + brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() + as *const __m256i, + ); + k0 = _mm256_mul_add_symm_epi8_by_ps_x4::( + k0, + v_source0_0, + v_source1_0, + coeff, + ); + k1 = _mm256_mul_add_symm_epi8_by_ps_x4::( + k1, + v_source0_1, + v_source1_1, + coeff, + ); + } + + let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); + let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); + + _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x4_epi8(k0)); + _mm256_storeu_si256(dst_ptr1 as *mut __m256i, _mm256_pack_ps_x4_epi8(k1)); + + cx += 32; } - let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); - let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); - - _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x4_epi8(k0)); - _mm256_storeu_si256(dst_ptr1 as *mut __m256i, _mm256_pack_ps_x4_epi8(k1)); - - cx += 32; - } - - while cx + 16 < image_width { - let shifted_src0 = ref0.get_unchecked(cx..); - let shifted_src1 = ref1.get_unchecked(cx..); - - let source_0 = _mm_loadu_si128(shifted_src0.as_ptr() as *const __m128i); - let source_1 = _mm_loadu_si128(shifted_src1.as_ptr() as *const __m128i); - - let mut k0 = _mm_mul_epi8_by_ps_x4::(source_0, _mm256_castps256_ps128(coeff)); - let mut k1 = _mm_mul_epi8_by_ps_x4::(source_1, _mm256_castps256_ps128(coeff)); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source_0_0 = _mm_loadu_si128( - brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - let v_source_1_0 = - _mm_loadu_si128(brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() - as *const __m128i); - let v_source_0_1 = _mm_loadu_si128( - brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - let v_source_1_1 = - _mm_loadu_si128(brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() - as *const __m128i); - k0 = _mm_mul_add_symm_epi8_by_ps_x4::( - k0, - v_source_0_0, - v_source_1_0, - _mm_set1_ps(coeff.weight), - ); - k1 = _mm_mul_add_symm_epi8_by_ps_x4::( - k1, - v_source_0_1, - v_source_1_1, - _mm_set1_ps(coeff.weight), - ); + while cx + 16 < image_width { + let shifted_src0 = ref0.get_unchecked(cx..); + let shifted_src1 = ref1.get_unchecked(cx..); + + let source_0 = _mm_loadu_si128(shifted_src0.as_ptr() as *const __m128i); + let source_1 = _mm_loadu_si128(shifted_src1.as_ptr() as *const __m128i); + + let mut k0 = _mm_mul_epi8_by_ps_x4::(source_0, _mm256_castps256_ps128(coeff)); + let mut k1 = _mm_mul_epi8_by_ps_x4::(source_1, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source_0_0 = _mm_loadu_si128( + brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, + ); + let v_source_1_0 = _mm_loadu_si128( + brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() + as *const __m128i, + ); + let v_source_0_1 = _mm_loadu_si128( + brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, + ); + let v_source_1_1 = _mm_loadu_si128( + brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() + as *const __m128i, + ); + k0 = _mm_mul_add_symm_epi8_by_ps_x4::( + k0, + v_source_0_0, + v_source_1_0, + _mm_set1_ps(coeff.weight), + ); + k1 = _mm_mul_add_symm_epi8_by_ps_x4::( + k1, + v_source_0_1, + v_source_1_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); + let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); + + _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); + _mm_storeu_si128(dst_ptr1 as *mut __m128i, _mm_pack_ps_x4_epi8(k1)); + + cx += 16; } - let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); - let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); - - _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); - _mm_storeu_si128(dst_ptr1 as *mut __m128i, _mm_pack_ps_x4_epi8(k1)); - - cx += 16; - } - - while cx + 4 < image_width { - let shifted_src0 = ref0.get_unchecked(cx..); - let shifted_src1 = ref1.get_unchecked(cx..); - - let source_0 = _mm_loadu_si32(shifted_src0.as_ptr() as *const _); - let source_1 = _mm_loadu_si32(shifted_src1.as_ptr() as *const _); - - let mut k0 = _mm_mul_epi8_by_ps(source_0, _mm256_castps256_ps128(coeff)); - let mut k1 = _mm_mul_epi8_by_ps(source_1, _mm256_castps256_ps128(coeff)); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source_0_0 = _mm_loadu_si32( - brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source_1_0 = _mm_loadu_si32( - brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source_0_1 = _mm_loadu_si32( - brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source_1_1 = _mm_loadu_si32( - brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi8_by_ps::( - k0, - v_source_0_0, - v_source_1_0, - _mm_set1_ps(coeff.weight), - ); - k1 = _mm_mul_add_symm_epi8_by_ps::( - k1, - v_source_0_1, - v_source_1_1, - _mm_set1_ps(coeff.weight), - ); + while cx + 4 < image_width { + let shifted_src0 = ref0.get_unchecked(cx..); + let shifted_src1 = ref1.get_unchecked(cx..); + + let source_0 = _mm_loadu_si32(shifted_src0.as_ptr() as *const _); + let source_1 = _mm_loadu_si32(shifted_src1.as_ptr() as *const _); + + let mut k0 = _mm_mul_epi8_by_ps(source_0, _mm256_castps256_ps128(coeff)); + let mut k1 = _mm_mul_epi8_by_ps(source_1, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source_0_0 = _mm_loadu_si32( + brows0.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source_1_0 = _mm_loadu_si32( + brows0.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source_0_1 = _mm_loadu_si32( + brows1.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source_1_1 = _mm_loadu_si32( + brows1.get_unchecked(rollback).get_unchecked(cx..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_ps::( + k0, + v_source_0_0, + v_source_1_0, + _mm_set1_ps(coeff.weight), + ); + k1 = _mm_mul_add_symm_epi8_by_ps::( + k1, + v_source_0_1, + v_source_1_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); + let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); + + _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_ps_epi8(k0)); + _mm_storeu_si32(dst_ptr1 as *mut _, _mm_pack_ps_epi8(k1)); + + cx += 4; } - let dst_ptr0 = dst0.get_unchecked_mut(cx..).as_mut_ptr(); - let dst_ptr1 = dst1.get_unchecked_mut(cx..).as_mut_ptr(); - - _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_ps_epi8(k0)); - _mm_storeu_si32(dst_ptr1 as *mut _, _mm_pack_ps_epi8(k1)); - - cx += 4; - } - - let coeff = scanned_kernel.get_unchecked(half_len).weight; - - for x in cx..image_width { - let v_src0 = ref0.get_unchecked(x..); - let v_src1 = ref1.get_unchecked(x..); - - let mut k0 = (*v_src0.get_unchecked(0) as f32).mul(coeff); - let mut k1 = (*v_src1.get_unchecked(0) as f32).mul(coeff); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - k0 = mlaf( - k0, - ((*brows0.get_unchecked(i).get_unchecked(x)) as f32) - .add(*brows0.get_unchecked(rollback).get_unchecked(x) as f32), - coeff.weight, - ); - - k1 = mlaf( - k1, - ((*brows1.get_unchecked(i).get_unchecked(x)) as f32) - .add(*brows1.get_unchecked(rollback).get_unchecked(x) as f32), - coeff.weight, - ); + let coeff = scanned_kernel.get_unchecked(half_len).weight; + + for x in cx..image_width { + let v_src0 = ref0.get_unchecked(x..); + let v_src1 = ref1.get_unchecked(x..); + + let mut k0 = (*v_src0.get_unchecked(0) as f32).mul(coeff); + let mut k1 = (*v_src1.get_unchecked(0) as f32).mul(coeff); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + k0 = mlaf( + k0, + ((*brows0.get_unchecked(i).get_unchecked(x)) as f32) + .add(*brows0.get_unchecked(rollback).get_unchecked(x) as f32), + coeff.weight, + ); + + k1 = mlaf( + k1, + ((*brows1.get_unchecked(i).get_unchecked(x)) as f32) + .add(*brows1.get_unchecked(rollback).get_unchecked(x) as f32), + coeff.weight, + ); + } + + *dst0.get_unchecked_mut(x) = k0.to_(); + *dst1.get_unchecked_mut(x) = k0.to_(); } - - *dst0.get_unchecked_mut(x) = k0.to_(); - *dst1.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/avx/filter_row.rs b/src/filter1d/avx/filter_row.rs index 3e3ec0db..d4e5c43a 100644 --- a/src/filter1d/avx/filter_row.rs +++ b/src/filter1d/avx/filter_row.rs @@ -80,7 +80,7 @@ pub(crate) fn filter_row_avx_u8_f32( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_row_avx_u8_f32_fma( +fn filter_row_avx_u8_f32_fma( arena: Arena, arena_src: &[u8], dst: &mut [u8], @@ -100,7 +100,7 @@ unsafe fn filter_row_avx_u8_f32_fma( } #[target_feature(enable = "avx2")] -unsafe fn filter_row_avx_u8_f32_def( +fn filter_row_avx_u8_f32_def( arena: Arena, arena_src: &[u8], dst: &mut [u8], @@ -124,7 +124,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, _: Arena, arena_src: &[u8], @@ -133,154 +133,158 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let src = arena_src; - - let local_src = src; - - let length = scanned_kernel.iter().len(); - - let max_width = image_size.width * N; - - let mut cx = 0usize; - - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(0).weight); - - while cx + 128 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm256_load_pack_x4(shifted_src.as_ptr()); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); - let mut k2 = _mm256_mul_epi8_by_ps_x4::(source.2, coeff); - let mut k3 = _mm256_mul_epi8_by_ps_x4::(source.3, coeff); - - for i in 1..length { - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm256_load_pack_x4(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source.0, coeff); - k1 = _mm256_mul_add_epi8_by_ps_x4::(k1, v_source.1, coeff); - k2 = _mm256_mul_add_epi8_by_ps_x4::(k2, v_source.2, coeff); - k3 = _mm256_mul_add_epi8_by_ps_x4::(k3, v_source.3, coeff); - } - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x4( - dst_ptr0, - ( - _mm256_pack_ps_x4_epi8(k0), - _mm256_pack_ps_x4_epi8(k1), - _mm256_pack_ps_x4_epi8(k2), - _mm256_pack_ps_x4_epi8(k3), - ), - ); - cx += 128; - } - - while cx + 64 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm256_load_pack_x2(shifted_src.as_ptr()); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); - - for i in 1..length { - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm256_load_pack_x2(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source.0, coeff); - k1 = _mm256_mul_add_epi8_by_ps_x4::(k1, v_source.1, coeff); + unsafe { + let src = arena_src; + + let local_src = src; + + let length = scanned_kernel.iter().len(); + + let max_width = image_size.width * N; + + let mut cx = 0usize; + + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(0).weight); + + while cx + 128 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = _mm256_load_pack_x4(shifted_src.as_ptr()); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); + let mut k2 = _mm256_mul_epi8_by_ps_x4::(source.2, coeff); + let mut k3 = _mm256_mul_epi8_by_ps_x4::(source.3, coeff); + + for i in 1..length { + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm256_load_pack_x4(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source.0, coeff); + k1 = _mm256_mul_add_epi8_by_ps_x4::(k1, v_source.1, coeff); + k2 = _mm256_mul_add_epi8_by_ps_x4::(k2, v_source.2, coeff); + k3 = _mm256_mul_add_epi8_by_ps_x4::(k3, v_source.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x4( + dst_ptr0, + ( + _mm256_pack_ps_x4_epi8(k0), + _mm256_pack_ps_x4_epi8(k1), + _mm256_pack_ps_x4_epi8(k2), + _mm256_pack_ps_x4_epi8(k3), + ), + ); + cx += 128; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x2( - dst_ptr0, - (_mm256_pack_ps_x4_epi8(k0), _mm256_pack_ps_x4_epi8(k1)), - ); - cx += 64; - } + while cx + 64 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm256_load_pack_x2(shifted_src.as_ptr()); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); - let source = _mm256_loadu_si256(shifted_src.as_ptr() as *const __m256i); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source, coeff); + for i in 1..length { + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm256_load_pack_x2(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source.0, coeff); + k1 = _mm256_mul_add_epi8_by_ps_x4::(k1, v_source.1, coeff); + } - for i in 1..length { - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm256_loadu_si256( - shifted_src.get_unchecked(i * N..).as_ptr() as *const __m256i + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x2( + dst_ptr0, + (_mm256_pack_ps_x4_epi8(k0), _mm256_pack_ps_x4_epi8(k1)), ); - k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x4_epi8(k0)); - cx += 32; - } + while cx + 32 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm256_loadu_si256(shifted_src.as_ptr() as *const __m256i); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source, coeff); - let source_0 = _mm_loadu_si128(shifted_src.as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi8_by_ps_x4::(source_0, _mm256_castps256_ps128(coeff)); + for i in 1..length { + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm256_loadu_si256( + shifted_src.get_unchecked(i * N..).as_ptr() as *const __m256i + ); + k0 = _mm256_mul_add_epi8_by_ps_x4::(k0, v_source, coeff); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - let v_source_0 = - _mm_loadu_si128(shifted_src.get_unchecked(i * N..).as_ptr() as *const __m128i); - k0 = _mm_mul_add_epi8_by_ps_x4::(k0, v_source_0, _mm_set1_ps(coeff.weight)); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_pack_ps_x4_epi8(k0)); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); - cx += 16; - } + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - let coeff = *scanned_kernel.get_unchecked(0); + let source_0 = _mm_loadu_si128(shifted_src.as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi8_by_ps_x4::(source_0, _mm256_castps256_ps128(coeff)); - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + let v_source_0 = _mm_loadu_si128( + shifted_src.get_unchecked(i * N..).as_ptr() as *const __m128i + ); + k0 = + _mm_mul_add_epi8_by_ps_x4::(k0, v_source_0, _mm_set1_ps(coeff.weight)); + } - let mut k0 = ((*shifted_src.get_unchecked(0)) as f32).mul(coeff.weight); - let mut k1 = ((*shifted_src.get_unchecked(1)) as f32).mul(coeff.weight); - let mut k2 = ((*shifted_src.get_unchecked(2)) as f32).mul(coeff.weight); - let mut k3 = ((*shifted_src.get_unchecked(3)) as f32).mul(coeff.weight); - - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = mlaf(k0, (*shifted_src.get_unchecked(i * N)) as f32, coeff.weight); - k1 = mlaf( - k1, - (*shifted_src.get_unchecked(i * N + 1)) as f32, - coeff.weight, - ); - k2 = mlaf( - k2, - (*shifted_src.get_unchecked(i * N + 2)) as f32, - coeff.weight, - ); - k3 = mlaf( - k3, - (*shifted_src.get_unchecked(i * N + 3)) as f32, - coeff.weight, - ); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); + cx += 16; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } + let coeff = *scanned_kernel.get_unchecked(0); + + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let mut k0 = ((*shifted_src.get_unchecked(0)) as f32).mul(coeff.weight); + let mut k1 = ((*shifted_src.get_unchecked(1)) as f32).mul(coeff.weight); + let mut k2 = ((*shifted_src.get_unchecked(2)) as f32).mul(coeff.weight); + let mut k3 = ((*shifted_src.get_unchecked(3)) as f32).mul(coeff.weight); + + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 = mlaf(k0, (*shifted_src.get_unchecked(i * N)) as f32, coeff.weight); + k1 = mlaf( + k1, + (*shifted_src.get_unchecked(i * N + 1)) as f32, + coeff.weight, + ); + k2 = mlaf( + k2, + (*shifted_src.get_unchecked(i * N + 2)) as f32, + coeff.weight, + ); + k3 = mlaf( + k3, + (*shifted_src.get_unchecked(i * N + 3)) as f32, + coeff.weight, + ); + } + + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; + } - for x in cx..max_width { - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = ((*shifted_src.get_unchecked(0)) as f32).mul(coeff.weight); + for x in cx..max_width { + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = ((*shifted_src.get_unchecked(0)) as f32).mul(coeff.weight); - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = mlaf(k0, (*shifted_src.get_unchecked(i * N)) as f32, coeff.weight); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 = mlaf(k0, (*shifted_src.get_unchecked(i * N)) as f32, coeff.weight); + } + *dst.get_unchecked_mut(x) = k0.to_(); } - *dst.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/avx/filter_row_approx.rs b/src/filter1d/avx/filter_row_approx.rs index 5bd67f29..0b97c0a4 100644 --- a/src/filter1d/avx/filter_row_approx.rs +++ b/src/filter1d/avx/filter_row_approx.rs @@ -62,7 +62,7 @@ pub(crate) fn filter_row_avx_u8_i32_app( } #[target_feature(enable = "avx2")] -unsafe fn filter_row_neon_u8_i32_impl( +fn filter_row_neon_u8_i32_impl( _: Arena, arena_src: &[u8], dst: &mut [u8], @@ -70,131 +70,133 @@ unsafe fn filter_row_neon_u8_i32_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let src = arena_src; + unsafe { + let src = arena_src; + + let v_prepared = scanned_kernel + .iter() + .map(|&x| { + let z = x.weight.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) + }) + .collect::>(); - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.weight.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) - }) - .collect::>(); + let length = scanned_kernel.len(); - let length = scanned_kernel.len(); + let local_src = src; - let local_src = src; + let max_width = image_size.width * N; - let max_width = image_size.width * N; + let mut cx = 0usize; - let mut cx = 0usize; + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(0)); - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(0)); + while cx + 64 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 64 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm256_load_pack_x2(shifted_src.as_ptr()); + let mut k0 = _mm256_mul_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_epi16_x4(source.1, coeff); - let source = _mm256_load_pack_x2(shifted_src.as_ptr()); - let mut k0 = _mm256_mul_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_epi16_x4(source.1, coeff); + for i in 1..length { + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source = _mm256_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); + k0 = _mm256_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); + k1 = _mm256_mul_add_epi8_by_epi16_x4(k1, v_source.1, coeff); + } - for i in 1..length { - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source = _mm256_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); - k0 = _mm256_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); - k1 = _mm256_mul_add_epi8_by_epi16_x4(k1, v_source.1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x2( + dst_ptr0, + (_mm256_pack_epi32_x4_epi8(k0), _mm256_pack_epi32_x4_epi8(k1)), + ); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x2( - dst_ptr0, - (_mm256_pack_epi32_x4_epi8(k0), _mm256_pack_epi32_x4_epi8(k1)), - ); - cx += 64; - } + while cx + 32 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm256_loadu_si256(shifted_src.as_ptr() as *const _); + let mut k0 = _mm256_mul_epi8_by_epi16_x4(source, coeff); - let source = _mm256_loadu_si256(shifted_src.as_ptr() as *const _); - let mut k0 = _mm256_mul_epi8_by_epi16_x4(source, coeff); + for i in 1..length { + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source = + _mm256_loadu_si256(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + k0 = _mm256_mul_add_epi8_by_epi16_x4(k0, v_source, coeff); + } - for i in 1..length { - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source = - _mm256_loadu_si256(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - k0 = _mm256_mul_add_epi8_by_epi16_x4(k0, v_source, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_epi32_x4_epi8(k0)); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_epi32_x4_epi8(k0)); - cx += 32; - } + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm_loadu_si128(shifted_src.as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_epi16_x4(source, _mm256_castsi256_si128(coeff)); - let source = _mm_loadu_si128(shifted_src.as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source, _mm256_castsi256_si128(coeff)); + for i in 1..length { + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source, coeff); + } - for i in 1..length { - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epi8(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epi8(k0)); - cx += 16; - } + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm_loadu_si64(shifted_src.as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - let source = _mm_loadu_si64(shifted_src.as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); + for i in 1..length { + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source = + _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + k0 = _mm_mul_add_epi8_by_epi16_x2(k0, v_source, coeff); + } - for i in 1..length { - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_epi8_by_epi16_x2(k0, v_source, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); + cx += 8; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); - cx += 8; - } + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm_loadu_si32(shifted_src.as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - let source = _mm_loadu_si32(shifted_src.as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); + for i in 1..length { + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source = + _mm_loadu_si32(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + k0 = _mm_mul_add_epi8_by_epi16_x2(k0, v_source, coeff); + } - for i in 1..length { - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source = - _mm_loadu_si32(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_epi8_by_epi16_x2(k0, v_source, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); + cx += 4; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); - cx += 4; - } + for x in cx..max_width { + let coeff = *scanned_kernel.get_unchecked(0); + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(0) as i32 * coeff.weight; - for x in cx..max_width { - let coeff = *scanned_kernel.get_unchecked(0); - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(0) as i32 * coeff.weight; + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); + k0 += (*shifted_src.get_unchecked(i * N) as i32) * coeff.weight; + } - k0 += (*shifted_src.get_unchecked(i * N) as i32) * coeff.weight; + *dst.get_unchecked_mut(x) = k0.to_approx_(); } - - *dst.get_unchecked_mut(x) = k0.to_approx_(); } } diff --git a/src/filter1d/avx/filter_row_complex_u8_i32.rs b/src/filter1d/avx/filter_row_complex_u8_i32.rs index 13d6bab6..d46501ef 100644 --- a/src/filter1d/avx/filter_row_complex_u8_i32.rs +++ b/src/filter1d/avx/filter_row_complex_u8_i32.rs @@ -45,43 +45,51 @@ pub(crate) fn filter_avx_row_complex_u8_i32( } #[inline(always)] -pub(crate) unsafe fn widen_mul(a: __m128i, b: __m128i) -> (__m128i, __m128i) { - let lo = _mm_mullo_epi16(a, b); - let hi = _mm_mulhi_epi16(a, b); - (_mm_unpacklo_epi16(lo, hi), _mm_unpackhi_epi16(lo, hi)) +pub(crate) fn widen_mul(a: __m128i, b: __m128i) -> (__m128i, __m128i) { + unsafe { + let lo = _mm_mullo_epi16(a, b); + let hi = _mm_mulhi_epi16(a, b); + (_mm_unpacklo_epi16(lo, hi), _mm_unpackhi_epi16(lo, hi)) + } } #[inline(always)] -pub(crate) unsafe fn widen_mul256(a: __m256i, b: __m256i) -> (__m256i, __m256i) { - let lo = _mm256_mullo_epi16(a, b); - let hi = _mm256_mulhi_epi16(a, b); - let c_lo = _mm256_unpacklo_epi16(lo, hi); - let c_hi = _mm256_unpackhi_epi16(lo, hi); - - let xy0 = _mm256_permute2x128_si256::<32>(c_lo, c_hi); - let xy1 = _mm256_permute2x128_si256::<49>(c_lo, c_hi); - (xy0, xy1) +pub(crate) fn widen_mul256(a: __m256i, b: __m256i) -> (__m256i, __m256i) { + unsafe { + let lo = _mm256_mullo_epi16(a, b); + let hi = _mm256_mulhi_epi16(a, b); + let c_lo = _mm256_unpacklo_epi16(lo, hi); + let c_hi = _mm256_unpackhi_epi16(lo, hi); + + let xy0 = _mm256_permute2x128_si256::<32>(c_lo, c_hi); + let xy1 = _mm256_permute2x128_si256::<49>(c_lo, c_hi); + (xy0, xy1) + } } #[inline(always)] -unsafe fn v_interleave_i16x8(a: __m256i, b: __m256i) -> (__m256i, __m256i) { - let xy_l = _mm256_unpacklo_epi16(a, b); - let xy_h = _mm256_unpacklo_epi16(a, b); +fn v_interleave_i16x8(a: __m256i, b: __m256i) -> (__m256i, __m256i) { + unsafe { + let xy_l = _mm256_unpacklo_epi16(a, b); + let xy_h = _mm256_unpacklo_epi16(a, b); - let xy0 = _mm256_permute2x128_si256::<32>(xy_l, xy_h); - let xy1 = _mm256_permute2x128_si256::<49>(xy_l, xy_h); - (xy0, xy1) + let xy0 = _mm256_permute2x128_si256::<32>(xy_l, xy_h); + let xy1 = _mm256_permute2x128_si256::<49>(xy_l, xy_h); + (xy0, xy1) + } } #[inline(always)] -pub(crate) unsafe fn widen_mul_lo(a: __m128i, b: __m128i) -> __m128i { - let lo = _mm_mullo_epi16(a, b); - let hi = _mm_mulhi_epi16(a, b); - _mm_unpacklo_epi16(lo, hi) +pub(crate) fn widen_mul_lo(a: __m128i, b: __m128i) -> __m128i { + unsafe { + let lo = _mm_mullo_epi16(a, b); + let hi = _mm_mulhi_epi16(a, b); + _mm_unpacklo_epi16(lo, hi) + } } #[target_feature(enable = "avx2")] -unsafe fn filter_avx_row_complex_u8_i32_impl( +fn filter_avx_row_complex_u8_i32_impl( arena: Arena, arena_src: &[u8], dst: &mut [Complex], diff --git a/src/filter1d/avx/filter_row_f32.rs b/src/filter1d/avx/filter_row_f32.rs index 38e60fec..8dc3ec48 100644 --- a/src/filter1d/avx/filter_row_f32.rs +++ b/src/filter1d/avx/filter_row_f32.rs @@ -74,7 +74,7 @@ pub(crate) fn filter_row_avx_f32_f32( } #[target_feature(enable = "avx2")] -unsafe fn filter_row_avx_f32_f32_def( +fn filter_row_avx_f32_f32_def( arena: Arena, arena_src: &[f32], dst: &mut [f32], @@ -82,21 +82,19 @@ unsafe fn filter_row_avx_f32_f32_def( filter_region: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - unsafe { - let unit = ExecutionUnit::::default(); - unit.pass( - arena, - arena_src, - dst, - image_size, - filter_region, - scanned_kernel, - ); - } + let unit = ExecutionUnit::::default(); + unit.pass( + arena, + arena_src, + dst, + image_size, + filter_region, + scanned_kernel, + ); } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_row_avx_f32_f32_fma( +fn filter_row_avx_f32_f32_fma( arena: Arena, arena_src: &[f32], dst: &mut [f32], @@ -104,17 +102,15 @@ unsafe fn filter_row_avx_f32_f32_fma( filter_region: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - unsafe { - let unit = ExecutionUnit::::default(); - unit.pass( - arena, - arena_src, - dst, - image_size, - filter_region, - scanned_kernel, - ); - } + let unit = ExecutionUnit::::default(); + unit.pass( + arena, + arena_src, + dst, + image_size, + filter_region, + scanned_kernel, + ); } #[derive(Copy, Clone, Default)] @@ -122,7 +118,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, _: Arena, arena_src: &[f32], @@ -131,111 +127,115 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let src = arena_src; + unsafe { + let src = arena_src; + + let local_src = src; - let local_src = src; + let length = scanned_kernel.len(); - let length = scanned_kernel.len(); + let mut cx = 0usize; - let mut cx = 0usize; + let max_width = image_size.width * N; - let max_width = image_size.width * N; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(0).weight); - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(0).weight); + while cx + 32 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm256_load_pack_ps_x4(shifted_src.as_ptr()); + let mut k0 = _mm256_mul_ps(source.0, coeff); + let mut k1 = _mm256_mul_ps(source.1, coeff); + let mut k2 = _mm256_mul_ps(source.2, coeff); + let mut k3 = _mm256_mul_ps(source.3, coeff); - let source = _mm256_load_pack_ps_x4(shifted_src.as_ptr()); - let mut k0 = _mm256_mul_ps(source.0, coeff); - let mut k1 = _mm256_mul_ps(source.1, coeff); - let mut k2 = _mm256_mul_ps(source.2, coeff); - let mut k3 = _mm256_mul_ps(source.3, coeff); + for i in 1..length { + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = + _mm256_load_pack_ps_x4(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm256_opt_fmlaf_ps::(k0, v_source.0, coeff); + k1 = _mm256_opt_fmlaf_ps::(k1, v_source.1, coeff); + k2 = _mm256_opt_fmlaf_ps::(k2, v_source.2, coeff); + k3 = _mm256_opt_fmlaf_ps::(k3, v_source.3, coeff); + } - for i in 1..length { - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm256_load_pack_ps_x4(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm256_opt_fmlaf_ps::(k0, v_source.0, coeff); - k1 = _mm256_opt_fmlaf_ps::(k1, v_source.1, coeff); - k2 = _mm256_opt_fmlaf_ps::(k2, v_source.2, coeff); - k3 = _mm256_opt_fmlaf_ps::(k3, v_source.3, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_ps_x4(dst_ptr0, (k0, k1, k2, k3)); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_ps_x4(dst_ptr0, (k0, k1, k2, k3)); - cx += 32; - } + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm256_load_pack_ps_x2(shifted_src.as_ptr()); + let mut k0 = _mm256_mul_ps(source.0, coeff); + let mut k1 = _mm256_mul_ps(source.1, coeff); - let source = _mm256_load_pack_ps_x2(shifted_src.as_ptr()); - let mut k0 = _mm256_mul_ps(source.0, coeff); - let mut k1 = _mm256_mul_ps(source.1, coeff); + for i in 1..length { + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = + _mm256_load_pack_ps_x2(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm256_opt_fmlaf_ps::(k0, v_source.0, coeff); + k1 = _mm256_opt_fmlaf_ps::(k1, v_source.1, coeff); + } - for i in 1..length { - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm256_load_pack_ps_x2(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm256_opt_fmlaf_ps::(k0, v_source.0, coeff); - k1 = _mm256_opt_fmlaf_ps::(k1, v_source.1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_ps_x2(dst_ptr0, (k0, k1)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_ps_x2(dst_ptr0, (k0, k1)); - cx += 16; - } + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm256_loadu_ps(shifted_src.as_ptr()); + let mut k0 = _mm256_mul_ps(source, coeff); - let source = _mm256_loadu_ps(shifted_src.as_ptr()); - let mut k0 = _mm256_mul_ps(source, coeff); + for i in 1..length { + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm256_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm256_opt_fmlaf_ps::(k0, v_source, coeff); + } - for i in 1..length { - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm256_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm256_opt_fmlaf_ps::(k0, v_source, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_ps(dst_ptr0, k0); + cx += 8; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_ps(dst_ptr0, k0); - cx += 8; - } + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source_0 = _mm_loadu_ps(shifted_src.as_ptr()); + let mut k0 = _mm_mul_ps(source_0, _mm256_castps256_ps128(coeff)); - let source_0 = _mm_loadu_ps(shifted_src.as_ptr()); - let mut k0 = _mm_mul_ps(source_0, _mm256_castps256_ps128(coeff)); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + let v_source_0 = _mm_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_opt_fmlaf_ps::(k0, v_source_0, _mm_set1_ps(coeff.weight)); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - let v_source_0 = _mm_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_opt_fmlaf_ps::(k0, v_source_0, _mm_set1_ps(coeff.weight)); + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_ps(dst_ptr, k0); + cx += 4; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_ps(dst_ptr, k0); - cx += 4; - } + let coeff = *scanned_kernel.get_unchecked(0); - let coeff = *scanned_kernel.get_unchecked(0); + while cx < max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source_0 = _mm_load_ss(shifted_src.as_ptr()); + let mut k0 = _mm_mul_ps(source_0, _mm_set1_ps(coeff.weight)); - let source_0 = _mm_load_ss(shifted_src.as_ptr()); - let mut k0 = _mm_mul_ps(source_0, _mm_set1_ps(coeff.weight)); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + let v_source_0 = _mm_load_ss(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_opt_fmlaf_ps::(k0, v_source_0, _mm_set1_ps(coeff.weight)); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - let v_source_0 = _mm_load_ss(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_opt_fmlaf_ps::(k0, v_source_0, _mm_set1_ps(coeff.weight)); + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_ss(dst_ptr, k0); + cx += 1; } - - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_ss(dst_ptr, k0); - cx += 1; } } } diff --git a/src/filter1d/avx/filter_row_f32_f64_symm.rs b/src/filter1d/avx/filter_row_f32_f64_symm.rs index 30707aa5..baa4a33a 100644 --- a/src/filter1d/avx/filter_row_f32_f64_symm.rs +++ b/src/filter1d/avx/filter_row_f32_f64_symm.rs @@ -68,7 +68,7 @@ pub(crate) fn filter_row_avx_f32_f64_symm( } #[target_feature(enable = "avx2")] -unsafe fn filter_row_avx_f32_f64_def_symm( +fn filter_row_avx_f32_f64_def_symm( arena: Arena, arena_src: &[f32], dst: &mut [f32], @@ -76,21 +76,19 @@ unsafe fn filter_row_avx_f32_f64_def_symm( filter_region: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - unsafe { - let unit = ExecutionUnit::::default(); - unit.pass( - arena, - arena_src, - dst, - image_size, - filter_region, - scanned_kernel, - ); - } + let unit = ExecutionUnit::::default(); + unit.pass( + arena, + arena_src, + dst, + image_size, + filter_region, + scanned_kernel, + ); } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_row_avx_f32_f64_fma_symm( +fn filter_row_avx_f32_f64_fma_symm( arena: Arena, arena_src: &[f32], dst: &mut [f32], @@ -98,17 +96,15 @@ unsafe fn filter_row_avx_f32_f64_fma_symm( filter_region: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - unsafe { - let unit = ExecutionUnit::::default(); - unit.pass( - arena, - arena_src, - dst, - image_size, - filter_region, - scanned_kernel, - ); - } + let unit = ExecutionUnit::::default(); + unit.pass( + arena, + arena_src, + dst, + image_size, + filter_region, + scanned_kernel, + ); } #[derive(Debug, Default, Copy, Clone)] @@ -116,7 +112,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, _: Arena, arena_src: &[f32], @@ -125,176 +121,185 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let src = arena_src; - - let local_src = src; - - let length = scanned_kernel.len(); - - let mut cx = 0usize; - - let max_width = image_size.width * N; - - let half_len = length / 2; - - let coeff = _mm256_set1_pd(scanned_kernel.get_unchecked(half_len).weight); - - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm256_load_pack_ps_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm256_mul_pd(_mm256_cvtps_pd(_mm256_castps256_ps128(source.0)), coeff); - let mut k1 = - _mm256_mul_pd(_mm256_cvtps_pd(_mm256_extractf128_ps::<1>(source.0)), coeff); - let mut k2 = _mm256_mul_pd(_mm256_cvtps_pd(_mm256_castps256_ps128(source.1)), coeff); - let mut k3 = - _mm256_mul_pd(_mm256_cvtps_pd(_mm256_extractf128_ps::<1>(source.1)), coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_pd(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_load_pack_ps_x2(shifted_src.get_unchecked(i * N..).as_ptr()); - let v_source1 = - _mm256_load_pack_ps_x2(shifted_src.get_unchecked(rollback * N..).as_ptr()); - k0 = _mm256_opt_fmla_pd::( - k0, - _mm256_add_pd( - _mm256_cvtps_pd(_mm256_castps256_ps128(v_source0.0)), - _mm256_cvtps_pd(_mm256_castps256_ps128(v_source1.0)), - ), - coeff, - ); - k1 = _mm256_opt_fmla_pd::( - k1, - _mm256_add_pd( - _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source0.0)), - _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source1.0)), - ), - coeff, - ); - - k2 = _mm256_opt_fmla_pd::( - k2, - _mm256_add_pd( - _mm256_cvtps_pd(_mm256_castps256_ps128(v_source0.1)), - _mm256_cvtps_pd(_mm256_castps256_ps128(v_source1.1)), - ), - coeff, - ); - k3 = _mm256_opt_fmla_pd::( - k3, - _mm256_add_pd( - _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source0.1)), - _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source1.1)), + unsafe { + let src = arena_src; + + let local_src = src; + + let length = scanned_kernel.len(); + + let mut cx = 0usize; + + let max_width = image_size.width * N; + + let half_len = length / 2; + + let coeff = _mm256_set1_pd(scanned_kernel.get_unchecked(half_len).weight); + + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm256_load_pack_ps_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = + _mm256_mul_pd(_mm256_cvtps_pd(_mm256_castps256_ps128(source.0)), coeff); + let mut k1 = + _mm256_mul_pd(_mm256_cvtps_pd(_mm256_extractf128_ps::<1>(source.0)), coeff); + let mut k2 = + _mm256_mul_pd(_mm256_cvtps_pd(_mm256_castps256_ps128(source.1)), coeff); + let mut k3 = + _mm256_mul_pd(_mm256_cvtps_pd(_mm256_extractf128_ps::<1>(source.1)), coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_pd(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm256_load_pack_ps_x2(shifted_src.get_unchecked(i * N..).as_ptr()); + let v_source1 = + _mm256_load_pack_ps_x2(shifted_src.get_unchecked(rollback * N..).as_ptr()); + k0 = _mm256_opt_fmla_pd::( + k0, + _mm256_add_pd( + _mm256_cvtps_pd(_mm256_castps256_ps128(v_source0.0)), + _mm256_cvtps_pd(_mm256_castps256_ps128(v_source1.0)), + ), + coeff, + ); + k1 = _mm256_opt_fmla_pd::( + k1, + _mm256_add_pd( + _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source0.0)), + _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source1.0)), + ), + coeff, + ); + + k2 = _mm256_opt_fmla_pd::( + k2, + _mm256_add_pd( + _mm256_cvtps_pd(_mm256_castps256_ps128(v_source0.1)), + _mm256_cvtps_pd(_mm256_castps256_ps128(v_source1.1)), + ), + coeff, + ); + k3 = _mm256_opt_fmla_pd::( + k3, + _mm256_add_pd( + _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source0.1)), + _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source1.1)), + ), + coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + + let z0 = _mm256_cvtpd_ps(k0); + let z1 = _mm256_cvtpd_ps(k1); + let z2 = _mm256_cvtpd_ps(k2); + let z3 = _mm256_cvtpd_ps(k3); + + _mm256_store_pack_ps_x2( + dst_ptr0, + ( + _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(z0), z1), + _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(z2), z3), ), - coeff, ); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - - let z0 = _mm256_cvtpd_ps(k0); - let z1 = _mm256_cvtpd_ps(k1); - let z2 = _mm256_cvtpd_ps(k2); - let z3 = _mm256_cvtpd_ps(k3); - - _mm256_store_pack_ps_x2( - dst_ptr0, - ( + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = _mm256_loadu_ps(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm256_mul_pd(_mm256_cvtps_pd(_mm256_castps256_ps128(source)), coeff); + let mut k1 = + _mm256_mul_pd(_mm256_cvtps_pd(_mm256_extractf128_ps::<1>(source)), coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_pd(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); + let v_source1 = + _mm256_loadu_ps(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm256_opt_fmla_pd::( + k0, + _mm256_add_pd( + _mm256_cvtps_pd(_mm256_castps256_ps128(v_source0)), + _mm256_cvtps_pd(_mm256_castps256_ps128(v_source1)), + ), + coeff, + ); + k1 = _mm256_opt_fmla_pd::( + k1, + _mm256_add_pd( + _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source0)), + _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source1)), + ), + coeff, + ); + } + + let z0 = _mm256_cvtpd_ps(k0); + let z1 = _mm256_cvtpd_ps(k1); + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_ps( + dst_ptr0, _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(z0), z1), - _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(z2), z3), - ), - ); - cx += 16; - } - - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm256_loadu_ps(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm256_mul_pd(_mm256_cvtps_pd(_mm256_castps256_ps128(source)), coeff); - let mut k1 = _mm256_mul_pd(_mm256_cvtps_pd(_mm256_extractf128_ps::<1>(source)), coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_pd(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); - let v_source1 = - _mm256_loadu_ps(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm256_opt_fmla_pd::( - k0, - _mm256_add_pd( - _mm256_cvtps_pd(_mm256_castps256_ps128(v_source0)), - _mm256_cvtps_pd(_mm256_castps256_ps128(v_source1)), - ), - coeff, - ); - k1 = _mm256_opt_fmla_pd::( - k1, - _mm256_add_pd( - _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source0)), - _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(v_source1)), - ), - coeff, ); + cx += 8; } - let z0 = _mm256_cvtpd_ps(k0); - let z1 = _mm256_cvtpd_ps(k1); - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_ps( - dst_ptr0, - _mm256_insertf128_ps::<1>(_mm256_castps128_ps256(z0), z1), - ); - cx += 8; - } - - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source_0 = _mm_loadu_ps(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm_mul_pd(_mm_cvtps_pd(source_0), _mm256_castpd256_pd128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set_pd1(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm_loadu_ps(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = _mm_loadu_ps(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm_opt_fmla_pd::( - k0, - _mm_add_pd(_mm_cvtps_pd(v_source0), _mm_cvtps_pd(v_source1)), - coeff, - ); + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source_0 = _mm_loadu_ps(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm256_mul_pd(_mm256_cvtps_pd(source_0), coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_pd(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm_loadu_ps(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm_loadu_ps(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm256_opt_fmla_pd::( + k0, + _mm256_add_pd(_mm256_cvtps_pd(v_source0), _mm256_cvtps_pd(v_source1)), + coeff, + ); + } + + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_ps(dst_ptr, _mm256_cvtpd_ps(k0)); + cx += 4; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_ps(dst_ptr, _mm_cvtpd_ps(k0)); - cx += 4; - } - - while cx < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source_0 = _mm_load_ss(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm_mul_pd(_mm_cvtps_pd(source_0), _mm256_castpd256_pd128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set_pd1(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm_load_ss(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = _mm_load_ss(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm_opt_fmla_pd::( - k0, - _mm_add_pd(_mm_cvtps_pd(v_source0), _mm_cvtps_pd(v_source1)), - coeff, - ); + while cx < max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source_0 = _mm_load_ss(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm_mul_pd(_mm_cvtps_pd(source_0), _mm256_castpd256_pd128(coeff)); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set_pd1(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm_load_ss(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm_load_ss(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm_opt_fmla_pd::( + k0, + _mm_add_pd(_mm_cvtps_pd(v_source0), _mm_cvtps_pd(v_source1)), + coeff, + ); + } + + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_ss(dst_ptr, _mm_cvtpd_ps(k0)); + + cx += 1; } - - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_ss(dst_ptr, _mm_cvtpd_ps(k0)); - - cx += 1; } } } diff --git a/src/filter1d/avx/filter_row_f32_symm.rs b/src/filter1d/avx/filter_row_f32_symm.rs index 33211793..ae5c6e7f 100644 --- a/src/filter1d/avx/filter_row_f32_symm.rs +++ b/src/filter1d/avx/filter_row_f32_symm.rs @@ -71,7 +71,7 @@ pub(crate) fn filter_row_avx_f32_f32_symm( } #[target_feature(enable = "avx2")] -unsafe fn filter_row_avx_f32_f32_def_symm( +fn filter_row_avx_f32_f32_def_symm( arena: Arena, arena_src: &[f32], dst: &mut [f32], @@ -79,21 +79,19 @@ unsafe fn filter_row_avx_f32_f32_def_symm( filter_region: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - unsafe { - let unit = ExecutionUnit::::default(); - unit.pass( - arena, - arena_src, - dst, - image_size, - filter_region, - scanned_kernel, - ); - } + let unit = ExecutionUnit::::default(); + unit.pass( + arena, + arena_src, + dst, + image_size, + filter_region, + scanned_kernel, + ); } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_row_avx_f32_f32_fma_symm( +fn filter_row_avx_f32_f32_fma_symm( arena: Arena, arena_src: &[f32], dst: &mut [f32], @@ -101,17 +99,15 @@ unsafe fn filter_row_avx_f32_f32_fma_symm( filter_region: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - unsafe { - let unit = ExecutionUnit::::default(); - unit.pass( - arena, - arena_src, - dst, - image_size, - filter_region, - scanned_kernel, - ); - } + let unit = ExecutionUnit::::default(); + unit.pass( + arena, + arena_src, + dst, + image_size, + filter_region, + scanned_kernel, + ); } #[derive(Debug, Default, Copy, Clone)] @@ -119,7 +115,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, _: Arena, arena_src: &[f32], @@ -128,124 +124,156 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let src = arena_src; + unsafe { + let src = arena_src; + + let local_src = src; - let local_src = src; + let length = scanned_kernel.len(); - let length = scanned_kernel.len(); + let mut cx = 0usize; - let mut cx = 0usize; + let max_width = image_size.width * N; - let max_width = image_size.width * N; + let half_len = length / 2; - let half_len = length / 2; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + while cx + 32 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = + _mm256_load_pack_ps_x4(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm256_mul_ps(source.0, coeff); + let mut k1 = _mm256_mul_ps(source.1, coeff); + let mut k2 = _mm256_mul_ps(source.2, coeff); + let mut k3 = _mm256_mul_ps(source.3, coeff); - let source = _mm256_load_pack_ps_x4(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm256_mul_ps(source.0, coeff); - let mut k1 = _mm256_mul_ps(source.1, coeff); - let mut k2 = _mm256_mul_ps(source.2, coeff); - let mut k3 = _mm256_mul_ps(source.3, coeff); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm256_load_pack_ps_x4(shifted_src.get_unchecked(i * N..).as_ptr()); + let v_source1 = + _mm256_load_pack_ps_x4(shifted_src.get_unchecked(rollback * N..).as_ptr()); + k0 = _mm256_opt_fmlaf_ps::( + k0, + _mm256_add_ps(v_source0.0, v_source1.0), + coeff, + ); + k1 = _mm256_opt_fmlaf_ps::( + k1, + _mm256_add_ps(v_source0.1, v_source1.1), + coeff, + ); + k2 = _mm256_opt_fmlaf_ps::( + k2, + _mm256_add_ps(v_source0.2, v_source1.2), + coeff, + ); + k3 = _mm256_opt_fmlaf_ps::( + k3, + _mm256_add_ps(v_source0.3, v_source1.3), + coeff, + ); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_load_pack_ps_x4(shifted_src.get_unchecked(i * N..).as_ptr()); - let v_source1 = - _mm256_load_pack_ps_x4(shifted_src.get_unchecked(rollback * N..).as_ptr()); - k0 = _mm256_opt_fmlaf_ps::(k0, _mm256_add_ps(v_source0.0, v_source1.0), coeff); - k1 = _mm256_opt_fmlaf_ps::(k1, _mm256_add_ps(v_source0.1, v_source1.1), coeff); - k2 = _mm256_opt_fmlaf_ps::(k2, _mm256_add_ps(v_source0.2, v_source1.2), coeff); - k3 = _mm256_opt_fmlaf_ps::(k3, _mm256_add_ps(v_source0.3, v_source1.3), coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_ps_x4(dst_ptr0, (k0, k1, k2, k3)); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_ps_x4(dst_ptr0, (k0, k1, k2, k3)); - cx += 32; - } + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm256_load_pack_ps_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm256_mul_ps(source.0, coeff); - let mut k1 = _mm256_mul_ps(source.1, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_load_pack_ps_x2(shifted_src.get_unchecked(i * N..).as_ptr()); - let v_source1 = - _mm256_load_pack_ps_x2(shifted_src.get_unchecked(rollback * N..).as_ptr()); - k0 = _mm256_opt_fmlaf_ps::(k0, _mm256_add_ps(v_source0.0, v_source1.0), coeff); - k1 = _mm256_opt_fmlaf_ps::(k1, _mm256_add_ps(v_source0.1, v_source1.1), coeff); + let source = + _mm256_load_pack_ps_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm256_mul_ps(source.0, coeff); + let mut k1 = _mm256_mul_ps(source.1, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm256_load_pack_ps_x2(shifted_src.get_unchecked(i * N..).as_ptr()); + let v_source1 = + _mm256_load_pack_ps_x2(shifted_src.get_unchecked(rollback * N..).as_ptr()); + k0 = _mm256_opt_fmlaf_ps::( + k0, + _mm256_add_ps(v_source0.0, v_source1.0), + coeff, + ); + k1 = _mm256_opt_fmlaf_ps::( + k1, + _mm256_add_ps(v_source0.1, v_source1.1), + coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_ps_x2(dst_ptr0, (k0, k1)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_ps_x2(dst_ptr0, (k0, k1)); - cx += 16; - } + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm256_loadu_ps(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm256_mul_ps(source, coeff); - let source = _mm256_loadu_ps(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm256_mul_ps(source, coeff); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); + let v_source1 = + _mm256_loadu_ps(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm256_opt_fmlaf_ps::(k0, _mm256_add_ps(v_source0, v_source1), coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); - let v_source1 = - _mm256_loadu_ps(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm256_opt_fmlaf_ps::(k0, _mm256_add_ps(v_source0, v_source1), coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_ps(dst_ptr0, k0); + cx += 8; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_ps(dst_ptr0, k0); - cx += 8; - } + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source_0 = _mm_loadu_ps(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm_mul_ps(source_0, _mm256_castps256_ps128(coeff)); - let source_0 = _mm_loadu_ps(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm_mul_ps(source_0, _mm256_castps256_ps128(coeff)); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set_ps1(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm_loadu_ps(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm_loadu_ps(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm_opt_fmlaf_ps::(k0, _mm_add_ps(v_source0, v_source1), coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set_ps1(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm_loadu_ps(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = _mm_loadu_ps(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm_opt_fmlaf_ps::(k0, _mm_add_ps(v_source0, v_source1), coeff); + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_ps(dst_ptr, k0); + cx += 4; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_ps(dst_ptr, k0); - cx += 4; - } + while cx < max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source_0 = _mm_load_ss(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm_mul_ps(source_0, _mm256_castps256_ps128(coeff)); - let source_0 = _mm_load_ss(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm_mul_ps(source_0, _mm256_castps256_ps128(coeff)); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set_ps1(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm_load_ss(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm_load_ss(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm_opt_fmlaf_ps::(k0, _mm_add_ps(v_source0, v_source1), coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set_ps1(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm_load_ss(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = _mm_load_ss(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm_opt_fmlaf_ps::(k0, _mm_add_ps(v_source0, v_source1), coeff); + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_ss(dst_ptr, k0); + cx += 1; } - - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_ss(dst_ptr, k0); - cx += 1; } } } diff --git a/src/filter1d/avx/filter_row_symm.rs b/src/filter1d/avx/filter_row_symm.rs index 2ea1b9c4..bba57255 100644 --- a/src/filter1d/avx/filter_row_symm.rs +++ b/src/filter1d/avx/filter_row_symm.rs @@ -79,7 +79,7 @@ pub(crate) fn filter_row_avx_symm_u8_f32( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_row_avx_symm_u8_f32_fma( +fn filter_row_avx_symm_u8_f32_fma( arena: Arena, arena_src: &[u8], dst: &mut [u8], @@ -99,7 +99,7 @@ unsafe fn filter_row_avx_symm_u8_f32_fma( } #[target_feature(enable = "avx2")] -unsafe fn filter_row_avx_u8_f32_def( +fn filter_row_avx_u8_f32_def( arena: Arena, arena_src: &[u8], dst: &mut [u8], @@ -123,7 +123,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, _: Arena, arena_src: &[u8], @@ -132,156 +132,172 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let width = image_size.width; - - let src = &arena_src; - - let max_width = width * N; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let local_src = src; - - let mut cx = 0usize; - - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - - while cx + 64 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm256_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm256_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = - _mm256_load_pack_x2(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm256_mul_add_symm_epi8_by_ps_x4::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epi8_by_ps_x4::(k1, v_source0.1, v_source1.1, coeff); - } - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x2( - dst_ptr0, - (_mm256_pack_ps_x4_epi8(k0), _mm256_pack_ps_x4_epi8(k1)), - ); - cx += 64; - } - - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm256_loadu_si256(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm256_mul_epi8_by_ps_x4::(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm256_loadu_si256(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm256_loadu_si256( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + unsafe { + let width = image_size.width; + + let src = &arena_src; + + let max_width = width * N; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let local_src = src; + + let mut cx = 0usize; + + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + + while cx + 64 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm256_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_ps_x4::(source.1, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm256_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm256_load_pack_x2(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm256_mul_add_symm_epi8_by_ps_x4::( + k0, + v_source0.0, + v_source1.0, + coeff, + ); + k1 = _mm256_mul_add_symm_epi8_by_ps_x4::( + k1, + v_source0.1, + v_source1.1, + coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x2( + dst_ptr0, + (_mm256_pack_ps_x4_epi8(k0), _mm256_pack_ps_x4_epi8(k1)), ); - k0 = _mm256_mul_add_symm_epi8_by_ps_x4::(k0, v_source0, v_source1, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_ps_x4_epi8(k0)); - cx += 32; - } - - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + while cx + 32 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_ps_x4::(source, _mm256_castps256_ps128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si128( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm256_loadu_si256( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ ); - k0 = _mm_mul_add_symm_epi8_by_ps_x4::(k0, v_source0, v_source1, coeff); + let mut k0 = _mm256_mul_epi8_by_ps_x4::(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_loadu_si256( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm256_loadu_si256( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epi8_by_ps_x4::(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_ps_x4_epi8(k0)); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_ps_x4_epi8(k0)); - cx += 16; - } - - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_ps_x2(source, _mm256_castps256_ps128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si64( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi8_by_ps_x2::(k0, v_source0, v_source1, coeff); + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_ps_x4::(source, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_ps_x4::(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_ps_x4_epi8(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_ps_x2_epi8(k0)); - cx += 8; - } - - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si32(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_ps(source, _mm256_castps256_ps128(coeff)); + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_ps_x2(source, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_ps_x2::(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_ps_x2_epi8(k0)); + cx += 8; + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_loadu_si32(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si32( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi8_by_ps::(k0, v_source0, v_source1, coeff); + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si32(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_ps(source, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_loadu_si32(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si32( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_ps::(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_ps_epi8(k0)); + cx += 4; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_ps_epi8(k0)); - cx += 4; - } + for x in cx..max_width { + let coeff = *scanned_kernel.get_unchecked(half_len); + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as f32 * coeff.weight; - for x in cx..max_width { - let coeff = *scanned_kernel.get_unchecked(half_len); - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as f32 * coeff.weight; + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + k0 = mlaf( + k0, + *shifted_src.get_unchecked(i * N) as f32 + + *shifted_src.get_unchecked(rollback * N) as f32, + coeff.weight, + ); + } - k0 = mlaf( - k0, - *shifted_src.get_unchecked(i * N) as f32 - + *shifted_src.get_unchecked(rollback * N) as f32, - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/avx/filter_row_symm_approx.rs b/src/filter1d/avx/filter_row_symm_approx.rs index 9aa94adc..b50685dd 100644 --- a/src/filter1d/avx/filter_row_symm_approx.rs +++ b/src/filter1d/avx/filter_row_symm_approx.rs @@ -62,7 +62,7 @@ pub(crate) fn filter_row_avx_symm_u8_i32_app( } #[target_feature(enable = "avx2")] -unsafe fn filter_row_avx_symm_u8_i32_impl( +fn filter_row_avx_symm_u8_i32_impl( arena: Arena, arena_src: &[u8], dst: &mut [u8], @@ -70,155 +70,162 @@ unsafe fn filter_row_avx_symm_u8_i32_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let width = image_size.width; - - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.weight.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) - }) - .collect::>(); - - let src = arena_src; - - let length = scanned_kernel.len(); - let half_len = length / 2; + unsafe { + let width = image_size.width; - let local_src = src; + let v_prepared = scanned_kernel + .iter() + .map(|&x| { + let z = x.weight.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) + }) + .collect::>(); - let mut cx = 0usize; + let src = arena_src; - let max_width = width * arena.components; + let length = scanned_kernel.len(); + let half_len = length / 2; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(half_len)); + let local_src = src; - while cx + 64 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let mut cx = 0usize; - let source = _mm256_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm256_mul_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm256_mul_epi8_by_epi16_x4(source.1, coeff); + let max_width = width * arena.components; - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = _mm256_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = - _mm256_load_pack_x2(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm256_mul_add_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); - } + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(half_len)); - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x2( - dst_ptr0, - (_mm256_pack_epi32_x4_epi8(k0), _mm256_pack_epi32_x4_epi8(k1)), - ); - cx += 64; - } + while cx + 64 < max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm256_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm256_mul_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm256_mul_epi8_by_epi16_x4(source.1, coeff); - let source = - _mm256_loadu_si256(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm256_mul_epi8_by_epi16_x4(source, coeff); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source0 = _mm256_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm256_load_pack_x2(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm256_mul_add_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm256_mul_add_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm256_loadu_si256(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm256_loadu_si256( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x2( + dst_ptr0, + (_mm256_pack_epi32_x4_epi8(k0), _mm256_pack_epi32_x4_epi8(k1)), ); - k0 = _mm256_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_epi32_x4_epi8(k0)); - cx += 32; - } - - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source, _mm256_castsi256_si128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si128(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + while cx + 32 < max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm256_loadu_si256(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm256_mul_epi8_by_epi16_x4(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source0 = + _mm256_loadu_si256(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm256_loadu_si256( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_epi32_x4_epi8(k0)); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epi8(k0)); - cx += 16; - } - - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si64(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + while cx + 16 < max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_epi16_x4(source, _mm256_castsi256_si128(coeff)); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epi8(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); - cx += 8; - } - - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_loadu_si32(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); + while cx + 8 < max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); + cx += 8; + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si32(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si32(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + while cx + 4 < max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si32(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source0 = + _mm_loadu_si32(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si32( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); + cx += 4; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); - cx += 4; - } + for x in cx..max_width { + let coeff = *scanned_kernel.get_unchecked(half_len); + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; - for x in cx..max_width { - let coeff = *scanned_kernel.get_unchecked(half_len); - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + k0 += (*shifted_src.get_unchecked(i * N) as i16 + + *shifted_src.get_unchecked(rollback * N) as i16) as i32 + * coeff.weight; + } - k0 += (*shifted_src.get_unchecked(i * N) as i16 - + *shifted_src.get_unchecked(rollback * N) as i16) as i32 - * coeff.weight; + *dst.get_unchecked_mut(x) = k0.to_approx_(); } - - *dst.get_unchecked_mut(x) = k0.to_approx_(); } } diff --git a/src/filter1d/avx/filter_row_symm_u16.rs b/src/filter1d/avx/filter_row_symm_u16.rs index fd7b6f5c..fc08848b 100644 --- a/src/filter1d/avx/filter_row_symm_u16.rs +++ b/src/filter1d/avx/filter_row_symm_u16.rs @@ -79,7 +79,7 @@ pub(crate) fn filter_row_avx_symm_u16_f32( } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn filter_row_avx_symm_u16_f32_fma( +fn filter_row_avx_symm_u16_f32_fma( arena: Arena, arena_src: &[u16], dst: &mut [u16], @@ -99,7 +99,7 @@ unsafe fn filter_row_avx_symm_u16_f32_fma( } #[target_feature(enable = "avx2")] -unsafe fn filter_row_avx_u16_f32_def( +fn filter_row_avx_u16_f32_def( arena: Arena, arena_src: &[u16], dst: &mut [u16], @@ -123,7 +123,7 @@ struct RowSymmExecutionUnit {} impl RowSymmExecutionUnit { #[inline(always)] - unsafe fn pass( + fn pass( &self, _: Arena, arena_src: &[u16], @@ -132,175 +132,213 @@ impl RowSymmExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let width = image_size.width; + unsafe { + let width = image_size.width; - let src = &arena_src; + let src = &arena_src; - let max_width = width * N; + let max_width = width * N; - let length = scanned_kernel.len(); - let half_len = length / 2; + let length = scanned_kernel.len(); + let half_len = length / 2; - let local_src = src; + let local_src = src; - let mut cx = 0usize; + let mut cx = 0usize; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - while cx + 64 <= max_width { - let shifted_src = local_src.get_unchecked(cx..); + while cx + 64 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - let source = - _mm256_load_pack_x4(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm256_mul_epi16_by_ps_x2::(source.0, coeff); - let mut k1 = _mm256_mul_epi16_by_ps_x2::(source.1, coeff); - let mut k2 = _mm256_mul_epi16_by_ps_x2::(source.2, coeff); - let mut k3 = _mm256_mul_epi16_by_ps_x2::(source.3, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm256_load_pack_x4(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm256_load_pack_x4( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm256_load_pack_x4( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, ); - k0 = _mm256_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm256_mul_add_symm_epi16_by_ps_x2::(k2, v_source0.2, v_source1.2, coeff); - k3 = _mm256_mul_add_symm_epi16_by_ps_x2::(k3, v_source0.3, v_source1.3, coeff); - } - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x4( - dst_ptr0 as *mut _, - ( - _mm256_pack_ps_x2_epi16(k0), - _mm256_pack_ps_x2_epi16(k1), - _mm256_pack_ps_x2_epi16(k2), - _mm256_pack_ps_x2_epi16(k3), - ), - ); - cx += 64; - } - - while cx + 32 <= max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm256_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm256_mul_epi16_by_ps_x2::(source.0, coeff); - let mut k1 = _mm256_mul_epi16_by_ps_x2::(source.1, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm256_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm256_load_pack_x2( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let mut k0 = _mm256_mul_epi16_by_ps_x2::(source.0, coeff); + let mut k1 = _mm256_mul_epi16_by_ps_x2::(source.1, coeff); + let mut k2 = _mm256_mul_epi16_by_ps_x2::(source.2, coeff); + let mut k3 = _mm256_mul_epi16_by_ps_x2::(source.3, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_load_pack_x4( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm256_load_pack_x4( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k0, + v_source0.0, + v_source1.0, + coeff, + ); + k1 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k1, + v_source0.1, + v_source1.1, + coeff, + ); + k2 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k2, + v_source0.2, + v_source1.2, + coeff, + ); + k3 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k3, + v_source0.3, + v_source1.3, + coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x4( + dst_ptr0 as *mut _, + ( + _mm256_pack_ps_x2_epi16(k0), + _mm256_pack_ps_x2_epi16(k1), + _mm256_pack_ps_x2_epi16(k2), + _mm256_pack_ps_x2_epi16(k3), + ), ); - k0 = _mm256_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x2( - dst_ptr0 as *mut _, - (_mm256_pack_ps_x2_epi16(k0), _mm256_pack_ps_x2_epi16(k1)), - ); - cx += 32; - } - - while cx + 16 <= max_width { - let shifted_src = local_src.get_unchecked(cx..); + while cx + 32 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - let source = - _mm256_loadu_si256(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm256_mul_epi16_by_ps_x2::(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm256_loadu_si256(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm256_loadu_si256( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm256_load_pack_x2( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, + ); + let mut k0 = _mm256_mul_epi16_by_ps_x2::(source.0, coeff); + let mut k1 = _mm256_mul_epi16_by_ps_x2::(source.1, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_load_pack_x2( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm256_load_pack_x2( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k0, + v_source0.0, + v_source1.0, + coeff, + ); + k1 = _mm256_mul_add_symm_epi16_by_ps_x2::( + k1, + v_source0.1, + v_source1.1, + coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x2( + dst_ptr0 as *mut _, + (_mm256_pack_ps_x2_epi16(k0), _mm256_pack_ps_x2_epi16(k1)), ); - k0 = _mm256_mul_add_symm_epi16_by_ps_x2::(k0, v_source0, v_source1, coeff); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_ps_x2_epi16(k0)); - cx += 16; - } - - while cx + 8 <= max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps_x2::(source, _mm256_castps256_ps128(coeff)); + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si128( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm256_loadu_si256( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ ); - k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0, v_source1, coeff); + let mut k0 = _mm256_mul_epi16_by_ps_x2::(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm256_loadu_si256( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm256_loadu_si256( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epi16_by_ps_x2::(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_ps_x2_epi16(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_ps_x2_epi16(k0)); - cx += 8; - } - - while cx + 4 <= max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps::(source, _mm256_castps256_ps128(coeff)); + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps_x2::(source, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_ps_x2_epi16(k0)); + cx += 8; + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si64( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi16_by_ps::(k0, v_source0, v_source1, coeff); + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps::(source, _mm256_castps256_ps128(coeff)); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps::(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_ps_epi16(k0)); + cx += 4; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_ps_epi16(k0)); - cx += 4; - } + let coeff = *scanned_kernel.get_unchecked(half_len); - let coeff = *scanned_kernel.get_unchecked(half_len); + for x in cx..max_width { + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as f32 * coeff.weight; - for x in cx..max_width { - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as f32 * coeff.weight; + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + k0 = mlaf( + k0, + *shifted_src.get_unchecked(i * N) as f32 + + *shifted_src.get_unchecked(rollback * N) as f32, + coeff.weight, + ); + } - k0 = mlaf( - k0, - *shifted_src.get_unchecked(i * N) as f32 - + *shifted_src.get_unchecked(rollback * N) as f32, - coeff.weight, - ); + use crate::to_storage::ToStorage; + *dst.get_unchecked_mut(x) = k0.to_(); } - - use crate::to_storage::ToStorage; - *dst.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/avx/filter_row_symm_uq15_u16.rs b/src/filter1d/avx/filter_row_symm_uq15_u16.rs index ada73605..07265733 100644 --- a/src/filter1d/avx/filter_row_symm_uq15_u16.rs +++ b/src/filter1d/avx/filter_row_symm_uq15_u16.rs @@ -63,7 +63,7 @@ pub(crate) fn filter_row_avx_symm_uq15_u16( } #[target_feature(enable = "avx2")] -unsafe fn filter_row_avx_symm_uq15_u16_impl( +fn filter_row_avx_symm_uq15_u16_impl( arena: Arena, arena_src: &[u16], dst: &mut [u16], @@ -71,190 +71,195 @@ unsafe fn filter_row_avx_symm_uq15_u16_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let width = image_size.width; - - let src = arena_src; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let local_src = src; - - let mut cx = 0usize; - - let max_width = width * arena.components; - - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(half_len).weight as i32); - - while cx + 64 <= max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm256_load_pack_x4(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm256_mul_epu16_widen(source.0, coeff); - let mut k1 = _mm256_mul_epu16_widen(source.1, coeff); - let mut k2 = _mm256_mul_epu16_widen(source.2, coeff); - let mut k3 = _mm256_mul_epu16_widen(source.3, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = - _mm256_load_pack_x4(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm256_load_pack_x4( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + unsafe { + let width = image_size.width; + + let src = arena_src; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let local_src = src; + + let mut cx = 0usize; + + let max_width = width * arena.components; + + let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(half_len).weight as i32); + + while cx + 64 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm256_load_pack_x4(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm256_mul_epu16_widen(source.0, coeff); + let mut k1 = _mm256_mul_epu16_widen(source.1, coeff); + let mut k2 = _mm256_mul_epu16_widen(source.2, coeff); + let mut k3 = _mm256_mul_epu16_widen(source.3, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = + _mm256_load_pack_x4(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm256_load_pack_x4( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epu16_by_epu16_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm256_mul_add_symm_epu16_by_epu16_x4(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm256_mul_add_symm_epu16_by_epu16_x4(k2, v_source0.2, v_source1.2, coeff); + k3 = _mm256_mul_add_symm_epu16_by_epu16_x4(k3, v_source0.3, v_source1.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x4( + dst_ptr0 as *mut _, + ( + _mm256_pack_epi32_x4_epu16(k0), + _mm256_pack_epi32_x4_epu16(k1), + _mm256_pack_epi32_x4_epu16(k2), + _mm256_pack_epi32_x4_epu16(k3), + ), ); - k0 = _mm256_mul_add_symm_epu16_by_epu16_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epu16_by_epu16_x4(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm256_mul_add_symm_epu16_by_epu16_x4(k2, v_source0.2, v_source1.2, coeff); - k3 = _mm256_mul_add_symm_epu16_by_epu16_x4(k3, v_source0.3, v_source1.3, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x4( - dst_ptr0 as *mut _, - ( - _mm256_pack_epi32_x4_epu16(k0), - _mm256_pack_epi32_x4_epu16(k1), - _mm256_pack_epi32_x4_epu16(k2), - _mm256_pack_epi32_x4_epu16(k3), - ), - ); - cx += 64; - } - - while cx + 32 <= max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm256_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm256_mul_epu16_widen(source.0, coeff); - let mut k1 = _mm256_mul_epu16_widen(source.1, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = - _mm256_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm256_load_pack_x2( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + while cx + 32 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm256_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm256_mul_epu16_widen(source.0, coeff); + let mut k1 = _mm256_mul_epu16_widen(source.1, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = + _mm256_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm256_load_pack_x2( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epu16_by_epu16_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm256_mul_add_symm_epu16_by_epu16_x4(k1, v_source0.1, v_source1.1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_store_pack_x2( + dst_ptr0 as *mut _, + ( + _mm256_pack_epi32_x4_epu16(k0), + _mm256_pack_epi32_x4_epu16(k1), + ), ); - k0 = _mm256_mul_add_symm_epu16_by_epu16_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm256_mul_add_symm_epu16_by_epu16_x4(k1, v_source0.1, v_source1.1, coeff); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_store_pack_x2( - dst_ptr0 as *mut _, - ( - _mm256_pack_epi32_x4_epu16(k0), - _mm256_pack_epi32_x4_epu16(k1), - ), - ); - cx += 32; - } - - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm256_loadu_si256(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm256_mul_epu16_widen(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = - _mm256_loadu_si256(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm256_loadu_si256( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm256_mul_add_symm_epu16_by_epu16_x4(k0, v_source0, v_source1, coeff); + while cx + 16 < max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm256_loadu_si256(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm256_mul_epu16_widen(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = + _mm256_loadu_si256(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm256_loadu_si256( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epu16_by_epu16_x4(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_epi32_x4_epu16(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_epi32_x4_epu16(k0)); - cx += 16; - } - - while cx + 8 <= max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epu16_widen(source, _mm256_castsi256_si128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si128(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epu16_by_epu16_x4(k0, v_source0, v_source1, coeff); + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epu16_widen(source, _mm256_castsi256_si128(coeff)); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epu16_by_epu16_x4(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epu16(k0)); + cx += 8; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epu16(k0)); - cx += 8; - } - - while cx + 4 <= max_width { - let shifted_src = local_src.get_unchecked(cx..); + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - let source = _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mullo_epi32( - _mm_unpacklo_epi16(source, _mm_setzero_si128()), - _mm256_castsi256_si128(coeff), - ); + let source = + _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mullo_epi32( + _mm_unpacklo_epi16(source, _mm_setzero_si128()), + _mm256_castsi256_si128(coeff), + ); - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si64(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_add_epi32( - k0, - _mm_mullo_epi32( - _mm_add_epi32( - _mm_unpacklo_epi16(v_source0, _mm_setzero_si128()), - _mm_unpacklo_epi16(v_source1, _mm_setzero_si128()), + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_add_epi32( + k0, + _mm_mullo_epi32( + _mm_add_epi32( + _mm_unpacklo_epi16(v_source0, _mm_setzero_si128()), + _mm_unpacklo_epi16(v_source1, _mm_setzero_si128()), + ), + coeff, ), - coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + + let rnd = _mm_set1_epi32((1 << 14) - 1); + _mm_storeu_si64( + dst_ptr0 as *mut _, + _mm_packus_epi32( + _mm_srli_epi32::<15>(_mm_add_epi32(k0, rnd)), + _mm_setzero_si128(), ), ); + cx += 4; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + let coeff = *scanned_kernel.get_unchecked(half_len); - let rnd = _mm_set1_epi32((1 << 14) - 1); - _mm_storeu_si64( - dst_ptr0 as *mut _, - _mm_packus_epi32( - _mm_srli_epi32::<15>(_mm_add_epi32(k0, rnd)), - _mm_setzero_si128(), - ), - ); - cx += 4; - } + for x in cx..max_width { + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as u32 * coeff.weight; - let coeff = *scanned_kernel.get_unchecked(half_len); + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; - for x in cx..max_width { - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as u32 * coeff.weight; + k0 += (*shifted_src.get_unchecked(i * N) as u32 + + *shifted_src.get_unchecked(rollback * N) as u32) + * coeff.weight; + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - - k0 += (*shifted_src.get_unchecked(i * N) as u32 - + *shifted_src.get_unchecked(rollback * N) as u32) - * coeff.weight; + *dst.get_unchecked_mut(x) = k0.to_approx_(); } - - *dst.get_unchecked_mut(x) = k0.to_approx_(); } } diff --git a/src/filter1d/avx/mod.rs b/src/filter1d/avx/mod.rs index 25a3b7da..f163cd94 100644 --- a/src/filter1d/avx/mod.rs +++ b/src/filter1d/avx/mod.rs @@ -69,13 +69,14 @@ mod utils; pub(crate) use filter_column::filter_column_avx_u8_f32; pub(crate) use filter_column_approx::filter_column_avx_u8_i32_app; -pub(crate) use filter_column_complex_u16_f32::filter_avx_column_complex_u16_f32; -pub(crate) use filter_column_complex_u16_i32::filter_avx_column_complex_u16_i32; pub(crate) use filter_column_complex_u8_f32::filter_avx_column_complex_u8_f32; pub(crate) use filter_column_complex_u8_i32::filter_avx_column_complex_u8_i32; +pub(crate) use filter_column_complex_u16_f32::filter_avx_column_complex_u16_f32; +pub(crate) use filter_column_complex_u16_i32::filter_avx_column_complex_u16_i32; pub(crate) use filter_column_f32::filter_column_avx_f32_f32; pub(crate) use filter_column_symm::filter_column_avx_symm_u8_f32; pub(crate) use filter_column_symm_approx::filter_column_avx_symm_u8_i32_app; +pub(crate) use filter_column_symm_approx_uq0_7::filter_column_avx_symm_u8_uq0_7; pub(crate) use filter_column_symm_approx_x2::filter_column_avx_symm_u8_i32_app_x2; pub(crate) use filter_column_symm_f32::filter_column_avx_symm_f32_f32; pub(crate) use filter_column_symm_f32_f64::filter_column_avx_symm_f32_f64; @@ -87,10 +88,10 @@ pub(crate) use filter_column_symm_x2::filter_column_avx_symm_u8_f32_x2; pub(crate) use filter_row::filter_row_avx_u8_f32; pub(crate) use filter_row_approx::filter_row_avx_u8_i32_app; pub(crate) use filter_row_complex_f32_f32::filter_avx_row_complex_f32_f32; -pub(crate) use filter_row_complex_u16_f32::filter_avx_row_complex_u16_f32; -pub(crate) use filter_row_complex_u16_i32::filter_avx_row_complex_u16_i32; pub(crate) use filter_row_complex_u8_f32::filter_avx_row_complex_u8_f32; pub(crate) use filter_row_complex_u8_i32::filter_avx_row_complex_u8_i32; +pub(crate) use filter_row_complex_u16_f32::filter_avx_row_complex_u16_f32; +pub(crate) use filter_row_complex_u16_i32::filter_avx_row_complex_u16_i32; pub(crate) use filter_row_f32::filter_row_avx_f32_f32; pub(crate) use filter_row_f32_f64_symm::filter_row_avx_f32_f64_symm; pub(crate) use filter_row_f32_symm::filter_row_avx_f32_f32_symm; @@ -99,6 +100,6 @@ pub(crate) use filter_row_symm_approx::filter_row_avx_symm_u8_i32_app; pub(crate) use filter_row_symm_u16::filter_row_avx_symm_u16_f32; pub(crate) use filter_row_symm_uq15_u16::filter_row_avx_symm_uq15_u16; pub(crate) use row_symm_approx_binter::filter_row_avx_symm_u8_i32_app_binter; +pub(crate) use row_symm_approx_binter_uq0_7::filter_row_avx_symm_u8_uq0_7_any; pub(crate) use row_symm_approx_binter3_uq0_7::filter_row_avx_symm_u8_uq0_7_k3; pub(crate) use row_symm_approx_binter5_uq0_7::filter_row_avx_symm_u8_uq0_7_k5; -pub(crate) use row_symm_approx_binter_uq0_7::filter_row_avx_symm_u8_uq0_7_any; diff --git a/src/filter1d/avx/row_symm_approx_binter.rs b/src/filter1d/avx/row_symm_approx_binter.rs index 9c53cfa9..1aac81a4 100644 --- a/src/filter1d/avx/row_symm_approx_binter.rs +++ b/src/filter1d/avx/row_symm_approx_binter.rs @@ -26,6 +26,7 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::BorderHandle; use crate::edge_mode::border_interpolate; use crate::filter1d::avx::sse_utils::{ _mm_mul_add_symm_epi8_by_epi16_x2, _mm_mul_add_symm_epi8_by_epi16_x4, _mm_mul_epi8_by_epi16_x2, @@ -34,11 +35,9 @@ use crate::filter1d::avx::sse_utils::{ use crate::filter1d::avx::utils::{ _mm256_mul_add_symm_epi8_by_epi16_x4, _mm256_mul_epi8_by_epi16_x4, _mm256_pack_epi32_x4_epi8, }; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; -use crate::BorderHandle; use std::arch::x86_64::*; pub(crate) fn filter_row_avx_symm_u8_i32_app_binter( @@ -46,7 +45,7 @@ pub(crate) fn filter_row_avx_symm_u8_i32_app_binter( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { let unit = ExecutionUnit::::default(); @@ -59,234 +58,245 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "avx2")] - unsafe fn pass( + fn pass( &self, edge_mode: BorderHandle, m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { - let width = image_size.width; + unsafe { + let width = image_size.width; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let min_left = half_len.min(width); + let s_kernel = half_len as i64; + + for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { + let dst = &mut **dst; + let mut f_cx = 0usize; + + while f_cx < min_left { + for c in 0..N { + let coeff = *scanned_kernel.get_unchecked(half_len); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); + + let mx = f_cx as i64 - s_kernel; + let mut k0: i32 = *src.get_unchecked(f_cx * N + c) as i32 * wb as i32; + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); + let rollback = length - i - 1; + + let src0 = border_interpolate!( + src, + edge_mode, + i as i64 + mx, + 0, + width as i64, + N, + c + ); + let src1 = border_interpolate!( + src, + edge_mode, + rollback as i64 + mx, + 0, + width as i64, + N, + c + ); + + k0 += (src0 as i32 + src1 as i32) * wb as i32; + } + + *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); + } + f_cx += 1; + } - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.weight.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) - }) - .collect::>(); + let mut m_cx = f_cx * N; - let length = scanned_kernel.len(); - let half_len = length / 2; + let s_half = half_len * N; + let m_right = width.saturating_sub(half_len); + let max_width = m_right * N; - let min_left = half_len.min(width); - let s_kernel = half_len as i64; + let coeff = _mm256_set1_epi32(*scanned_kernel.get_unchecked(half_len)); - for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { - let dst = &mut **dst; - let mut f_cx = 0usize; + while m_cx + 32 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - while f_cx < min_left { - for c in 0..N { - let coeff = *scanned_kernel.get_unchecked(half_len); - let mx = f_cx as i64 - s_kernel; - let mut k0: i32 = *src.get_unchecked(f_cx * N + c) as i32 * coeff.weight; + let source = _mm256_loadu_si256( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, + ); + let mut k0 = _mm256_mul_epi8_by_epi16_x4(source, coeff); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; - - let src0 = border_interpolate!( - src, - edge_mode, - i as i64 + mx, - 0, - width as i64, - N, - c + let coeff = _mm256_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm256_loadu_si256( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, ); - let src1 = border_interpolate!( - src, - edge_mode, - rollback as i64 + mx, - 0, - width as i64, - N, - c + let v_source1 = _mm256_loadu_si256( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, ); - - k0 += (src0 as i32 + src1 as i32) * coeff.weight; + k0 = _mm256_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); } - *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_epi32_x4_epi8(k0)); + m_cx += 32; } - f_cx += 1; - } - - let mut m_cx = f_cx * N; - - let s_half = half_len * N; - let m_right = width.saturating_sub(half_len); - let max_width = m_right * N; - - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(half_len)); - while m_cx + 32 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); + while m_cx + 16 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - let source = _mm256_loadu_si256( - shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ - ); - let mut k0 = _mm256_mul_epi8_by_epi16_x4(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = _mm256_loadu_si256( - shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, - ); - let v_source1 = _mm256_loadu_si256( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm_loadu_si128( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, ); - k0 = _mm256_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); - } + let mut k0 = _mm_mul_epi8_by_epi16_x4(source, _mm256_castsi256_si128(coeff)); - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_epi32_x4_epi8(k0)); - m_cx += 32; - } + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm_loadu_si128( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + } - while m_cx + 16 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source, _mm256_castsi256_si128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si128( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epi8(k0)); + m_cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epi8(k0)); - m_cx += 16; - } + while m_cx + 8 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - while m_cx + 8 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si64( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm_loadu_si64( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ ); - k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); - } + let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); - m_cx += 8; - } + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm_loadu_si64( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + } - while m_cx + 4 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si32(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si32(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si32( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); + m_cx += 8; } - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); - m_cx += 4; - } - - let coeff = *scanned_kernel.get_unchecked(half_len); + while m_cx + 4 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - for zx in m_cx..max_width { - let x = zx - s_half; - - let shifted_src = src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; + let source = _mm_loadu_si32( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ + ); + let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm_loadu_si32( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si32( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + } - k0 += (*shifted_src.get_unchecked(i * N) as i16 - + *shifted_src.get_unchecked(rollback * N) as i16) - as i32 - * coeff.weight; + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); + m_cx += 4; } - *dst.get_unchecked_mut(zx) = k0.to_approx_(); - } + let coeff = *scanned_kernel.get_unchecked(half_len); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); - f_cx = m_right; + for zx in m_cx..max_width { + let x = zx - s_half; - while f_cx < width { - for c in 0..N { - let mx = f_cx as i64 - s_kernel; - let mut k0 = *src.get_unchecked(f_cx * N + c) as i32 * coeff.weight; + let shifted_src = src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * wb as i32; for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); let rollback = length - i - 1; - let src0 = border_interpolate!( - src, - edge_mode, - i as i64 + mx, - 0, - width as i64, - N, - c - ); - let src1 = border_interpolate!( - src, - edge_mode, - rollback as i64 + mx, - 0, - width as i64, - N, - c - ); - - k0 += (src0 as i32 + src1 as i32) * coeff.weight; + k0 += (*shifted_src.get_unchecked(i * N) as i16 + + *shifted_src.get_unchecked(rollback * N) as i16) + as i32 + * wb as i32; } - *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); + *dst.get_unchecked_mut(zx) = k0.to_approx_(); + } + + f_cx = m_right; + + while f_cx < width { + for c in 0..N { + let mx = f_cx as i64 - s_kernel; + let mut k0 = *src.get_unchecked(f_cx * N + c) as i32 * wb as i32; + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); + let rollback = length - i - 1; + + let src0 = border_interpolate!( + src, + edge_mode, + i as i64 + mx, + 0, + width as i64, + N, + c + ); + let src1 = border_interpolate!( + src, + edge_mode, + rollback as i64 + mx, + 0, + width as i64, + N, + c + ); + + k0 += (src0 as i32 + src1 as i32) * wb as i32; + } + + *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); + } + f_cx += 1; } - f_cx += 1; } } } diff --git a/src/filter1d/avx/row_symm_approx_binter3_uq0_7.rs b/src/filter1d/avx/row_symm_approx_binter3_uq0_7.rs index ad0c7719..12a9c40e 100644 --- a/src/filter1d/avx/row_symm_approx_binter3_uq0_7.rs +++ b/src/filter1d/avx/row_symm_approx_binter3_uq0_7.rs @@ -26,11 +26,10 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::BorderHandle; use crate::edge_mode::clamp_edge; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::img_size::ImageSize; -use crate::BorderHandle; use std::arch::x86_64::*; pub(crate) fn filter_row_avx_symm_u8_uq0_7_k3( @@ -38,29 +37,11 @@ pub(crate) fn filter_row_avx_symm_u8_uq0_7_k3( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i8], ) { unsafe { - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8).min(i8::MAX as i32) as i8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } let unit = ExecutionUnit::::default(); - unit.pass(edge_mode, m_src, m_dst, image_size, &shifted); + unit.pass(edge_mode, m_src, m_dst, image_size, scanned_kernel); } } @@ -69,7 +50,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "avx2")] - unsafe fn pass( + fn pass( &self, edge_mode: BorderHandle, m_src: &RowsHolder, @@ -77,236 +58,252 @@ impl ExecutionUnit { image_size: ImageSize, scanned_kernel: &[i8], ) { - let width = image_size.width; + unsafe { + let width = image_size.width; - let length = scanned_kernel.len(); - let half_len = length / 2; + let length = scanned_kernel.len(); + let half_len = length / 2; - let min_left = half_len.min(width); - let s_kernel = half_len as i64; + let min_left = half_len.min(width); + let s_kernel = half_len as i64; - let c0 = scanned_kernel[0]; - let c1 = scanned_kernel[1]; + let c0 = scanned_kernel[0]; + let c1 = scanned_kernel[1]; - let vc0 = _mm256_set1_epi8(scanned_kernel[0]); - let vc1 = _mm256_set1_epi8(scanned_kernel[1]); + let vc0 = _mm256_set1_epi8(scanned_kernel[0]); + let vc1 = _mm256_set1_epi8(scanned_kernel[1]); - let rnd = _mm256_set1_epi16(1i16 << 6); + let rnd = _mm256_set1_epi16(1i16 << 6); - for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { - let dst = &mut **dst; - let mut f_cx = 0usize; + for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { + let dst = &mut **dst; + let mut f_cx = 0usize; - while f_cx < min_left { - let mx = f_cx as i64 - s_kernel; - let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); - let e1 = clamp_edge!(edge_mode.edge_mode, 2i64 + mx, 0, width as i64); + while f_cx < min_left { + let mx = f_cx as i64 - s_kernel; + let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); + let e1 = clamp_edge!(edge_mode.edge_mode, 2i64 + mx, 0, width as i64); - for c in 0..N { - let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c1 as u16; + for c in 0..N { + let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c1 as u16; - let src0 = *src.get_unchecked(e0 * N + c); - let src1 = *src.get_unchecked(e1 * N + c); + let src0 = *src.get_unchecked(e0 * N + c); + let src1 = *src.get_unchecked(e1 * N + c); - k0 += (src0 as u16 + src1 as u16) * c0 as u16; + k0 += (src0 as u16 + src1 as u16) * c0 as u16; - *dst.get_unchecked_mut(f_cx * N + c) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + *dst.get_unchecked_mut(f_cx * N + c) = + ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + f_cx += 1; } - f_cx += 1; - } - let mut m_cx = f_cx * N; - - let s_half = half_len * N; - let m_right = width.saturating_sub(half_len); - let max_width = m_right * N; - - while m_cx + 32 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let v_source0 = - _mm256_loadu_si256(shifted_src.get_unchecked(0..).as_ptr() as *const _); - let source = - _mm256_loadu_si256(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let v_source4 = - _mm256_loadu_si256(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); - - let mut k0 = _mm256_add_epi16( - rnd, - _mm256_maddubs_epi16(_mm256_unpacklo_epi8(source, _mm256_setzero_si256()), vc1), - ); - let mut k1 = _mm256_add_epi16( - rnd, - _mm256_maddubs_epi16(_mm256_unpackhi_epi8(source, _mm256_setzero_si256()), vc1), - ); - - k0 = _mm256_add_epi16( - k0, - _mm256_maddubs_epi16(_mm256_unpacklo_epi8(v_source0, v_source4), vc0), - ); - k1 = _mm256_add_epi16( - k1, - _mm256_maddubs_epi16(_mm256_unpackhi_epi8(v_source0, v_source4), vc0), - ); - - k0 = _mm256_srai_epi16::<7>(k0); - k1 = _mm256_srai_epi16::<7>(k1); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_packus_epi16(k0, k1)); - m_cx += 32; - } + let mut m_cx = f_cx * N; + + let s_half = half_len * N; + let m_right = width.saturating_sub(half_len); + let max_width = m_right * N; + + while m_cx + 32 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let v_source0 = + _mm256_loadu_si256(shifted_src.get_unchecked(0..).as_ptr() as *const _); + let source = + _mm256_loadu_si256(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let v_source4 = _mm256_loadu_si256( + shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _, + ); + + let mut k0 = _mm256_add_epi16( + rnd, + _mm256_maddubs_epi16( + _mm256_unpacklo_epi8(source, _mm256_setzero_si256()), + vc1, + ), + ); + let mut k1 = _mm256_add_epi16( + rnd, + _mm256_maddubs_epi16( + _mm256_unpackhi_epi8(source, _mm256_setzero_si256()), + vc1, + ), + ); + + k0 = _mm256_add_epi16( + k0, + _mm256_maddubs_epi16(_mm256_unpacklo_epi8(v_source0, v_source4), vc0), + ); + k1 = _mm256_add_epi16( + k1, + _mm256_maddubs_epi16(_mm256_unpackhi_epi8(v_source0, v_source4), vc0), + ); + + k0 = _mm256_srai_epi16::<7>(k0); + k1 = _mm256_srai_epi16::<7>(k1); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_packus_epi16(k0, k1)); + m_cx += 32; + } - while m_cx + 16 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked(0..).as_ptr() as *const _); - let source = _mm_loadu_si128(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let v_source4 = - _mm_loadu_si128(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); - - let mut k0 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(vc1), - ), - ); - let mut k1 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpackhi_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(vc1), - ), - ); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source0, v_source4), - _mm256_castsi256_si128(vc0), - ), - ); - k1 = _mm_add_epi16( - k1, - _mm_maddubs_epi16( - _mm_unpackhi_epi8(v_source0, v_source4), - _mm256_castsi256_si128(vc0), - ), - ); - - k0 = _mm_srai_epi16::<7>(k0); - k1 = _mm_srai_epi16::<7>(k1); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_packus_epi16(k0, k1)); - m_cx += 16; - } + while m_cx + 16 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked(0..).as_ptr() as *const _); + let source = + _mm_loadu_si128(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let v_source4 = + _mm_loadu_si128(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); + + let mut k0 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(vc1), + ), + ); + let mut k1 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpackhi_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(vc1), + ), + ); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source0, v_source4), + _mm256_castsi256_si128(vc0), + ), + ); + k1 = _mm_add_epi16( + k1, + _mm_maddubs_epi16( + _mm_unpackhi_epi8(v_source0, v_source4), + _mm256_castsi256_si128(vc0), + ), + ); + + k0 = _mm_srai_epi16::<7>(k0); + k1 = _mm_srai_epi16::<7>(k1); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_packus_epi16(k0, k1)); + m_cx += 16; + } - while m_cx + 8 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = _mm_loadu_si64(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(vc1), - ), - ); - - let v_source0 = _mm_loadu_si64(shifted_src.get_unchecked(..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si64(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source0, v_source1), - _mm256_castsi256_si128(vc0), - ), - ); - - k0 = _mm_srai_epi16::<7>(k0); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si64( - dst_ptr0 as *mut _, - _mm_packus_epi16(k0, _mm_setzero_si128()), - ); - m_cx += 8; - } + while m_cx + 8 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let source = + _mm_loadu_si64(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let mut k0 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(vc1), + ), + ); + + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked(..).as_ptr() as *const _); + let v_source1 = + _mm_loadu_si64(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source0, v_source1), + _mm256_castsi256_si128(vc0), + ), + ); + + k0 = _mm_srai_epi16::<7>(k0); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si64( + dst_ptr0 as *mut _, + _mm_packus_epi16(k0, _mm_setzero_si128()), + ); + m_cx += 8; + } - while m_cx + 4 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = _mm_loadu_si32(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(vc1), - ), - ); - - let v_source0 = _mm_loadu_si32(shifted_src.get_unchecked(..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si32(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source0, v_source1), - _mm256_castsi256_si128(vc0), - ), - ); - - k0 = _mm_srai_epi16::<7>(k0); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si32( - dst_ptr0 as *mut _, - _mm_packus_epi16(k0, _mm_setzero_si128()), - ); - m_cx += 4; - } + while m_cx + 4 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let source = + _mm_loadu_si32(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let mut k0 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(vc1), + ), + ); + + let v_source0 = + _mm_loadu_si32(shifted_src.get_unchecked(..).as_ptr() as *const _); + let v_source1 = + _mm_loadu_si32(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source0, v_source1), + _mm256_castsi256_si128(vc0), + ), + ); + + k0 = _mm_srai_epi16::<7>(k0); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si32( + dst_ptr0 as *mut _, + _mm_packus_epi16(k0, _mm_setzero_si128()), + ); + m_cx += 4; + } - for zx in m_cx..max_width { - let x = zx - s_half; + for zx in m_cx..max_width { + let x = zx - s_half; - let shifted_src = src.get_unchecked(x..); - let mut k0: u16 = *shifted_src.get_unchecked(N) as u16 * c1 as u16; + let shifted_src = src.get_unchecked(x..); + let mut k0: u16 = *shifted_src.get_unchecked(N) as u16 * c1 as u16; - k0 += (*shifted_src.get_unchecked(0) as u16 - + *shifted_src.get_unchecked(2 * N) as u16) - * c0 as u16; + k0 += (*shifted_src.get_unchecked(0) as u16 + + *shifted_src.get_unchecked(2 * N) as u16) + * c0 as u16; - *dst.get_unchecked_mut(zx) = ((k0 + (1 << 6)) >> 7).min(255) as u8; - } + *dst.get_unchecked_mut(zx) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + } - f_cx = m_right; + f_cx = m_right; - while f_cx < width { - let mx = f_cx as i64 - s_kernel; - let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); - let e1 = clamp_edge!(edge_mode.edge_mode, 2i64 + mx, 0, width as i64); + while f_cx < width { + let mx = f_cx as i64 - s_kernel; + let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); + let e1 = clamp_edge!(edge_mode.edge_mode, 2i64 + mx, 0, width as i64); - for c in 0..N { - let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c1 as u16; + for c in 0..N { + let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c1 as u16; - let src0 = *src.get_unchecked(e0 * N + c); - let src1 = *src.get_unchecked(e1 * N + c); + let src0 = *src.get_unchecked(e0 * N + c); + let src1 = *src.get_unchecked(e1 * N + c); - k0 += (src0 as u16 + src1 as u16) * c0 as u16; + k0 += (src0 as u16 + src1 as u16) * c0 as u16; - *dst.get_unchecked_mut(f_cx * N + c) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + *dst.get_unchecked_mut(f_cx * N + c) = + ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + f_cx += 1; } - f_cx += 1; } } } diff --git a/src/filter1d/avx/row_symm_approx_binter5_uq0_7.rs b/src/filter1d/avx/row_symm_approx_binter5_uq0_7.rs index cd29ee58..6a3d4bd7 100644 --- a/src/filter1d/avx/row_symm_approx_binter5_uq0_7.rs +++ b/src/filter1d/avx/row_symm_approx_binter5_uq0_7.rs @@ -26,11 +26,10 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::BorderHandle; use crate::edge_mode::clamp_edge; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::img_size::ImageSize; -use crate::BorderHandle; use std::arch::x86_64::*; pub(crate) fn filter_row_avx_symm_u8_uq0_7_k5( @@ -38,29 +37,11 @@ pub(crate) fn filter_row_avx_symm_u8_uq0_7_k5( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i8], ) { unsafe { - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8).min(i8::MAX as i32) as i8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } let unit = ExecutionUnit::::default(); - unit.pass(edge_mode, m_src, m_dst, image_size, &shifted); + unit.pass(edge_mode, m_src, m_dst, image_size, scanned_kernel); } } @@ -69,7 +50,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "avx2")] - unsafe fn pass( + fn pass( &self, edge_mode: BorderHandle, m_src: &RowsHolder, @@ -94,295 +75,314 @@ impl ExecutionUnit { let rnd = _mm256_set1_epi16(1i16 << 6); - for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { - let dst = &mut **dst; - let mut f_cx = 0usize; + unsafe { + for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { + let dst = &mut **dst; + let mut f_cx = 0usize; - while f_cx < min_left { - let mx = f_cx as i64 - s_kernel; - let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); - let e1 = clamp_edge!(edge_mode.edge_mode, 4i64 + mx, 0, width as i64); - let e2 = clamp_edge!(edge_mode.edge_mode, mx + 1, 0, width as i64); - let e3 = clamp_edge!(edge_mode.edge_mode, 3i64 + mx, 0, width as i64); + while f_cx < min_left { + let mx = f_cx as i64 - s_kernel; + let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); + let e1 = clamp_edge!(edge_mode.edge_mode, 4i64 + mx, 0, width as i64); + let e2 = clamp_edge!(edge_mode.edge_mode, mx + 1, 0, width as i64); + let e3 = clamp_edge!(edge_mode.edge_mode, 3i64 + mx, 0, width as i64); - for c in 0..N { - let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c2 as u16; + for c in 0..N { + let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c2 as u16; - let src0 = *src.get_unchecked(e0 * N + c); - let src1 = *src.get_unchecked(e1 * N + c); + let src0 = *src.get_unchecked(e0 * N + c); + let src1 = *src.get_unchecked(e1 * N + c); - let src2 = *src.get_unchecked(e2 * N + c); - let src3 = *src.get_unchecked(e3 * N + c); + let src2 = *src.get_unchecked(e2 * N + c); + let src3 = *src.get_unchecked(e3 * N + c); - k0 += (src0 as u16 + src1 as u16) * c0 as u16; - k0 += (src2 as u16 + src3 as u16) * c1 as u16; + k0 += (src0 as u16 + src1 as u16) * c0 as u16; + k0 += (src2 as u16 + src3 as u16) * c1 as u16; - *dst.get_unchecked_mut(f_cx * N + c) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + *dst.get_unchecked_mut(f_cx * N + c) = + ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + f_cx += 1; } - f_cx += 1; - } - let mut m_cx = f_cx * N; - - let s_half = half_len * N; - let m_right = width.saturating_sub(half_len); - let max_width = m_right * N; - - while m_cx + 32 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let v_source0 = - _mm256_loadu_si256(shifted_src.get_unchecked(0..).as_ptr() as *const _); - let source = _mm256_loadu_si256( - shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ - ); - let v_source4 = - _mm256_loadu_si256(shifted_src.get_unchecked((4 * N)..).as_ptr() as *const _); - let v_source2 = - _mm256_loadu_si256(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let v_source3 = - _mm256_loadu_si256(shifted_src.get_unchecked((3 * N)..).as_ptr() as *const _); - - let mut k0 = _mm256_add_epi16( - rnd, - _mm256_maddubs_epi16(_mm256_unpacklo_epi8(source, _mm256_setzero_si256()), vc2), - ); - let mut k1 = _mm256_add_epi16( - rnd, - _mm256_maddubs_epi16(_mm256_unpackhi_epi8(source, _mm256_setzero_si256()), vc2), - ); - - k0 = _mm256_add_epi16( - k0, - _mm256_maddubs_epi16(_mm256_unpacklo_epi8(v_source0, v_source4), vc0), - ); - k1 = _mm256_add_epi16( - k1, - _mm256_maddubs_epi16(_mm256_unpackhi_epi8(v_source0, v_source4), vc0), - ); - - k0 = _mm256_add_epi16( - k0, - _mm256_maddubs_epi16(_mm256_unpacklo_epi8(v_source2, v_source3), vc1), - ); - k1 = _mm256_add_epi16( - k1, - _mm256_maddubs_epi16(_mm256_unpackhi_epi8(v_source2, v_source3), vc1), - ); - - k0 = _mm256_srai_epi16::<7>(k0); - k1 = _mm256_srai_epi16::<7>(k1); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_packus_epi16(k0, k1)); - m_cx += 32; - } + let mut m_cx = f_cx * N; + + let s_half = half_len * N; + let m_right = width.saturating_sub(half_len); + let max_width = m_right * N; + + while m_cx + 32 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let v_source0 = + _mm256_loadu_si256(shifted_src.get_unchecked(0..).as_ptr() as *const _); + let source = _mm256_loadu_si256( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, + ); + let v_source4 = _mm256_loadu_si256( + shifted_src.get_unchecked((4 * N)..).as_ptr() as *const _, + ); + let v_source2 = + _mm256_loadu_si256(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let v_source3 = _mm256_loadu_si256( + shifted_src.get_unchecked((3 * N)..).as_ptr() as *const _, + ); + + let mut k0 = _mm256_add_epi16( + rnd, + _mm256_maddubs_epi16( + _mm256_unpacklo_epi8(source, _mm256_setzero_si256()), + vc2, + ), + ); + let mut k1 = _mm256_add_epi16( + rnd, + _mm256_maddubs_epi16( + _mm256_unpackhi_epi8(source, _mm256_setzero_si256()), + vc2, + ), + ); + + k0 = _mm256_add_epi16( + k0, + _mm256_maddubs_epi16(_mm256_unpacklo_epi8(v_source0, v_source4), vc0), + ); + k1 = _mm256_add_epi16( + k1, + _mm256_maddubs_epi16(_mm256_unpackhi_epi8(v_source0, v_source4), vc0), + ); + + k0 = _mm256_add_epi16( + k0, + _mm256_maddubs_epi16(_mm256_unpacklo_epi8(v_source2, v_source3), vc1), + ); + k1 = _mm256_add_epi16( + k1, + _mm256_maddubs_epi16(_mm256_unpackhi_epi8(v_source2, v_source3), vc1), + ); + + k0 = _mm256_srai_epi16::<7>(k0); + k1 = _mm256_srai_epi16::<7>(k1); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_packus_epi16(k0, k1)); + m_cx += 32; + } - while m_cx + 16 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked(0..).as_ptr() as *const _); - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let v_source4 = - _mm_loadu_si128(shifted_src.get_unchecked((4 * N)..).as_ptr() as *const _); - let v_source2 = - _mm_loadu_si128(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let v_source3 = - _mm_loadu_si128(shifted_src.get_unchecked((3 * N)..).as_ptr() as *const _); - - let mut k0 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(vc2), - ), - ); - let mut k1 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpackhi_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(vc2), - ), - ); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source0, v_source4), - _mm256_castsi256_si128(vc0), - ), - ); - k1 = _mm_add_epi16( - k1, - _mm_maddubs_epi16( - _mm_unpackhi_epi8(v_source0, v_source4), - _mm256_castsi256_si128(vc0), - ), - ); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source2, v_source3), - _mm256_castsi256_si128(vc1), - ), - ); - k1 = _mm_add_epi16( - k1, - _mm_maddubs_epi16( - _mm_unpackhi_epi8(v_source2, v_source3), - _mm256_castsi256_si128(vc1), - ), - ); - - k0 = _mm_srai_epi16::<7>(k0); - k1 = _mm_srai_epi16::<7>(k1); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_packus_epi16(k0, k1)); - m_cx += 16; - } + while m_cx + 16 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked(0..).as_ptr() as *const _); + let source = _mm_loadu_si128( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, + ); + let v_source4 = + _mm_loadu_si128(shifted_src.get_unchecked((4 * N)..).as_ptr() as *const _); + let v_source2 = + _mm_loadu_si128(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let v_source3 = + _mm_loadu_si128(shifted_src.get_unchecked((3 * N)..).as_ptr() as *const _); + + let mut k0 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(vc2), + ), + ); + let mut k1 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpackhi_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(vc2), + ), + ); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source0, v_source4), + _mm256_castsi256_si128(vc0), + ), + ); + k1 = _mm_add_epi16( + k1, + _mm_maddubs_epi16( + _mm_unpackhi_epi8(v_source0, v_source4), + _mm256_castsi256_si128(vc0), + ), + ); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source2, v_source3), + _mm256_castsi256_si128(vc1), + ), + ); + k1 = _mm_add_epi16( + k1, + _mm_maddubs_epi16( + _mm_unpackhi_epi8(v_source2, v_source3), + _mm256_castsi256_si128(vc1), + ), + ); + + k0 = _mm_srai_epi16::<7>(k0); + k1 = _mm_srai_epi16::<7>(k1); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_packus_epi16(k0, k1)); + m_cx += 16; + } - while m_cx + 8 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(vc2), - ), - ); - - let v_source0 = _mm_loadu_si64(shifted_src.get_unchecked(..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si64(shifted_src.get_unchecked((4 * N)..).as_ptr() as *const _); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source0, v_source1), - _mm256_castsi256_si128(vc0), - ), - ); - - let v_source2 = _mm_loadu_si64(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let v_source3 = - _mm_loadu_si64(shifted_src.get_unchecked((3 * N)..).as_ptr() as *const _); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source2, v_source3), - _mm256_castsi256_si128(vc1), - ), - ); - - k0 = _mm_srai_epi16::<7>(k0); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si64( - dst_ptr0 as *mut _, - _mm_packus_epi16(k0, _mm_setzero_si128()), - ); - m_cx += 8; - } + while m_cx + 8 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let source = _mm_loadu_si64( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ + ); + let mut k0 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(vc2), + ), + ); + + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked(..).as_ptr() as *const _); + let v_source1 = + _mm_loadu_si64(shifted_src.get_unchecked((4 * N)..).as_ptr() as *const _); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source0, v_source1), + _mm256_castsi256_si128(vc0), + ), + ); + + let v_source2 = + _mm_loadu_si64(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let v_source3 = + _mm_loadu_si64(shifted_src.get_unchecked((3 * N)..).as_ptr() as *const _); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source2, v_source3), + _mm256_castsi256_si128(vc1), + ), + ); + + k0 = _mm_srai_epi16::<7>(k0); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si64( + dst_ptr0 as *mut _, + _mm_packus_epi16(k0, _mm_setzero_si128()), + ); + m_cx += 8; + } - while m_cx + 4 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si32(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(vc2), - ), - ); - - let v_source0 = _mm_loadu_si32(shifted_src.get_unchecked(..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si32(shifted_src.get_unchecked((4 * N)..).as_ptr() as *const _); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source0, v_source1), - _mm256_castsi256_si128(vc0), - ), - ); - - let v_source2 = _mm_loadu_si32(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let v_source3 = - _mm_loadu_si32(shifted_src.get_unchecked((3 * N)..).as_ptr() as *const _); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source2, v_source3), - _mm256_castsi256_si128(vc1), - ), - ); - - k0 = _mm_srai_epi16::<7>(k0); - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si32( - dst_ptr0 as *mut _, - _mm_packus_epi16(k0, _mm_setzero_si128()), - ); - m_cx += 4; - } + while m_cx + 4 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let source = _mm_loadu_si32( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ + ); + let mut k0 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(vc2), + ), + ); + + let v_source0 = + _mm_loadu_si32(shifted_src.get_unchecked(..).as_ptr() as *const _); + let v_source1 = + _mm_loadu_si32(shifted_src.get_unchecked((4 * N)..).as_ptr() as *const _); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source0, v_source1), + _mm256_castsi256_si128(vc0), + ), + ); + + let v_source2 = + _mm_loadu_si32(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let v_source3 = + _mm_loadu_si32(shifted_src.get_unchecked((3 * N)..).as_ptr() as *const _); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source2, v_source3), + _mm256_castsi256_si128(vc1), + ), + ); + + k0 = _mm_srai_epi16::<7>(k0); + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si32( + dst_ptr0 as *mut _, + _mm_packus_epi16(k0, _mm_setzero_si128()), + ); + m_cx += 4; + } - for zx in m_cx..max_width { - let x = zx - s_half; + for zx in m_cx..max_width { + let x = zx - s_half; - let shifted_src = src.get_unchecked(x..); - let mut k0: u16 = *shifted_src.get_unchecked(half_len * N) as u16 * c2 as u16; + let shifted_src = src.get_unchecked(x..); + let mut k0: u16 = *shifted_src.get_unchecked(half_len * N) as u16 * c2 as u16; - k0 += (*shifted_src.get_unchecked(0) as u16 - + *shifted_src.get_unchecked(4 * N) as u16) - * c0 as u16; + k0 += (*shifted_src.get_unchecked(0) as u16 + + *shifted_src.get_unchecked(4 * N) as u16) + * c0 as u16; - k0 += (*shifted_src.get_unchecked(N) as u16 - + *shifted_src.get_unchecked(3 * N) as u16) - * c1 as u16; + k0 += (*shifted_src.get_unchecked(N) as u16 + + *shifted_src.get_unchecked(3 * N) as u16) + * c1 as u16; - *dst.get_unchecked_mut(zx) = ((k0 + (1 << 6)) >> 7).min(255) as u8; - } + *dst.get_unchecked_mut(zx) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + } - f_cx = m_right; + f_cx = m_right; - while f_cx < width { - let mx = f_cx as i64 - s_kernel; - let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); - let e1 = clamp_edge!(edge_mode.edge_mode, 4i64 + mx, 0, width as i64); - let e2 = clamp_edge!(edge_mode.edge_mode, mx + 1, 0, width as i64); - let e3 = clamp_edge!(edge_mode.edge_mode, 3i64 + mx, 0, width as i64); + while f_cx < width { + let mx = f_cx as i64 - s_kernel; + let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); + let e1 = clamp_edge!(edge_mode.edge_mode, 4i64 + mx, 0, width as i64); + let e2 = clamp_edge!(edge_mode.edge_mode, mx + 1, 0, width as i64); + let e3 = clamp_edge!(edge_mode.edge_mode, 3i64 + mx, 0, width as i64); - for c in 0..N { - let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c2 as u16; + for c in 0..N { + let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c2 as u16; - let src0 = *src.get_unchecked(e0 * N + c); - let src1 = *src.get_unchecked(e1 * N + c); + let src0 = *src.get_unchecked(e0 * N + c); + let src1 = *src.get_unchecked(e1 * N + c); - let src2 = *src.get_unchecked(e2 * N + c); - let src3 = *src.get_unchecked(e3 * N + c); + let src2 = *src.get_unchecked(e2 * N + c); + let src3 = *src.get_unchecked(e3 * N + c); - k0 += (src0 as u16 + src1 as u16) * c0 as u16; - k0 += (src2 as u16 + src3 as u16) * c1 as u16; + k0 += (src0 as u16 + src1 as u16) * c0 as u16; + k0 += (src2 as u16 + src3 as u16) * c1 as u16; - *dst.get_unchecked_mut(f_cx * N + c) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + *dst.get_unchecked_mut(f_cx * N + c) = + ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + f_cx += 1; } - f_cx += 1; } } } diff --git a/src/filter1d/avx/row_symm_approx_binter_uq0_7.rs b/src/filter1d/avx/row_symm_approx_binter_uq0_7.rs index 5946ba3d..98f92363 100644 --- a/src/filter1d/avx/row_symm_approx_binter_uq0_7.rs +++ b/src/filter1d/avx/row_symm_approx_binter_uq0_7.rs @@ -26,11 +26,10 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::BorderHandle; use crate::edge_mode::border_interpolate; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::img_size::ImageSize; -use crate::BorderHandle; use std::arch::x86_64::*; pub(crate) fn filter_row_avx_symm_u8_uq0_7_any( @@ -38,29 +37,11 @@ pub(crate) fn filter_row_avx_symm_u8_uq0_7_any( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i8], ) { unsafe { - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8).min(i8::MAX as i32) as i8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } let unit = ExecutionUnit::::default(); - unit.pass(edge_mode, m_src, m_dst, image_size, &shifted); + unit.pass(edge_mode, m_src, m_dst, image_size, scanned_kernel); } } @@ -69,7 +50,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "avx2")] - unsafe fn pass( + fn pass( &self, edge_mode: BorderHandle, m_src: &RowsHolder, @@ -77,301 +58,311 @@ impl ExecutionUnit { image_size: ImageSize, scanned_kernel: &[i8], ) { - let width = image_size.width; - - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[0], z[0], z[0]]) - }) - .collect::>(); + unsafe { + let width = image_size.width; + + let v_prepared = scanned_kernel + .iter() + .map(|&x| { + let z = x.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[0], z[0], z[0]]) + }) + .collect::>(); + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let min_left = half_len.min(width); + let s_kernel = half_len as i64; + + for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { + let dst = &mut **dst; + let mut f_cx = 0usize; + + let coeff = *scanned_kernel.get_unchecked(half_len); + + while f_cx < min_left { + for c in 0..N { + let mx = f_cx as i64 - s_kernel; + let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * coeff as u16; + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + + let src0 = border_interpolate!( + src, + edge_mode, + i as i64 + mx, + 0, + width as i64, + N, + c + ); + let src1 = border_interpolate!( + src, + edge_mode, + rollback as i64 + mx, + 0, + width as i64, + N, + c + ); + + k0 += (src0 as u16 + src1 as u16) * coeff as u16; + } + + *dst.get_unchecked_mut(f_cx * N + c) = + ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + f_cx += 1; + } - let length = scanned_kernel.len(); - let half_len = length / 2; + let mut m_cx = f_cx * N; - let min_left = half_len.min(width); - let s_kernel = half_len as i64; + let s_half = half_len * N; + let m_right = width.saturating_sub(half_len); + let max_width = m_right * N; - for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { - let dst = &mut **dst; - let mut f_cx = 0usize; + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(half_len)); + let rnd = _mm256_set1_epi16(1 << 6); - let coeff = *scanned_kernel.get_unchecked(half_len); + while m_cx + 32 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - while f_cx < min_left { - for c in 0..N { - let mx = f_cx as i64 - s_kernel; - let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * coeff as u16; + let source = _mm256_loadu_si256( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, + ); + let mut k0 = _mm256_add_epi16( + rnd, + _mm256_maddubs_epi16( + _mm256_unpacklo_epi8(source, _mm256_setzero_si256()), + coeff, + ), + ); + let mut k1 = _mm256_add_epi16( + rnd, + _mm256_maddubs_epi16( + _mm256_unpackhi_epi8(source, _mm256_setzero_si256()), + coeff, + ), + ); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; - - let src0 = border_interpolate!( - src, - edge_mode, - i as i64 + mx, - 0, - width as i64, - N, - c + let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source0 = _mm256_loadu_si256( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, ); - let src1 = border_interpolate!( - src, - edge_mode, - rollback as i64 + mx, - 0, - width as i64, - N, - c + let v_source1 = _mm256_loadu_si256( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm256_add_epi16( + k0, + _mm256_maddubs_epi16(_mm256_unpacklo_epi8(v_source0, v_source1), coeff), + ); + k1 = _mm256_add_epi16( + k1, + _mm256_maddubs_epi16(_mm256_unpackhi_epi8(v_source0, v_source1), coeff), ); - - k0 += (src0 as u16 + src1 as u16) * coeff as u16; } - *dst.get_unchecked_mut(f_cx * N + c) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + k0 = _mm256_srai_epi16::<7>(k0); + k1 = _mm256_srai_epi16::<7>(k1); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_packus_epi16(k0, k1)); + m_cx += 32; } - f_cx += 1; - } - let mut m_cx = f_cx * N; - - let s_half = half_len * N; - let m_right = width.saturating_sub(half_len); - let max_width = m_right * N; - - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(half_len)); - let rnd = _mm256_set1_epi16(1 << 6); - - while m_cx + 32 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = _mm256_loadu_si256( - shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ - ); - let mut k0 = _mm256_add_epi16( - rnd, - _mm256_maddubs_epi16( - _mm256_unpacklo_epi8(source, _mm256_setzero_si256()), - coeff, - ), - ); - let mut k1 = _mm256_add_epi16( - rnd, - _mm256_maddubs_epi16( - _mm256_unpackhi_epi8(source, _mm256_setzero_si256()), - coeff, - ), - ); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = _mm256_loadu_si256( - shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, - ); - let v_source1 = _mm256_loadu_si256( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + while m_cx + 16 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let source = _mm_loadu_si128( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, ); - k0 = _mm256_add_epi16( - k0, - _mm256_maddubs_epi16(_mm256_unpacklo_epi8(v_source0, v_source1), coeff), + let mut k0 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(coeff), + ), ); - k1 = _mm256_add_epi16( - k1, - _mm256_maddubs_epi16(_mm256_unpackhi_epi8(v_source0, v_source1), coeff), + let mut k1 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpackhi_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(coeff), + ), ); - } - k0 = _mm256_srai_epi16::<7>(k0); - k1 = _mm256_srai_epi16::<7>(k1); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source0 = _mm_loadu_si128( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), + ); + k1 = _mm_add_epi16( + k1, + _mm_maddubs_epi16(_mm_unpackhi_epi8(v_source0, v_source1), coeff), + ); + } - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut __m256i, _mm256_packus_epi16(k0, k1)); - m_cx += 32; - } + k0 = _mm_srai_epi16::<7>(k0); + k1 = _mm_srai_epi16::<7>(k1); - while m_cx + 16 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(coeff), - ), - ); - let mut k1 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpackhi_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(coeff), - ), - ); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si128( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), - ); - k1 = _mm_add_epi16( - k1, - _mm_maddubs_epi16(_mm_unpackhi_epi8(v_source0, v_source1), coeff), - ); + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_packus_epi16(k0, k1)); + m_cx += 16; } - k0 = _mm_srai_epi16::<7>(k0); - k1 = _mm_srai_epi16::<7>(k1); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_packus_epi16(k0, k1)); - m_cx += 16; - } + while m_cx + 8 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - while m_cx + 8 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(coeff), - ), - ); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si64( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm_loadu_si64( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ ); - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), + let mut k0 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(coeff), + ), ); - } - k0 = _mm_srai_epi16::<7>(k0); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source0 = _mm_loadu_si64( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), + ); + } - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si64( - dst_ptr0 as *mut _, - _mm_packus_epi16(k0, _mm_setzero_si128()), - ); - m_cx += 8; - } + k0 = _mm_srai_epi16::<7>(k0); - while m_cx + 4 <= max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si32(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - _mm256_castsi256_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm256_castsi256_si128(coeff), - ), - ); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si32(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si32( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si64( + dst_ptr0 as *mut _, + _mm_packus_epi16(k0, _mm_setzero_si128()), ); + m_cx += 8; } - k0 = _mm_srai_epi16::<7>(k0); + while m_cx + 4 <= max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si32( - dst_ptr0 as *mut _, - _mm_packus_epi16(k0, _mm_setzero_si128()), - ); - m_cx += 4; - } - - let coeff = *scanned_kernel.get_unchecked(half_len); - - for zx in m_cx..max_width { - let x = zx - s_half; + let source = _mm_loadu_si32( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ + ); + let mut k0 = _mm_add_epi16( + _mm256_castsi256_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm256_castsi256_si128(coeff), + ), + ); - let shifted_src = src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as u16 * coeff as u16; + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let v_source0 = _mm_loadu_si32( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si32( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), + ); + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + k0 = _mm_srai_epi16::<7>(k0); - k0 += (*shifted_src.get_unchecked(i * N) as u16 - + *shifted_src.get_unchecked(rollback * N) as u16) - * coeff as u16; + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si32( + dst_ptr0 as *mut _, + _mm_packus_epi16(k0, _mm_setzero_si128()), + ); + m_cx += 4; } - *dst.get_unchecked_mut(zx) = ((k0 + (1 << 6)) >> 7).min(255) as u8; - } + let coeff = *scanned_kernel.get_unchecked(half_len); - f_cx = m_right; + for zx in m_cx..max_width { + let x = zx - s_half; - while f_cx < width { - for c in 0..N { - let mx = f_cx as i64 - s_kernel; - let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * coeff as u16; + let shifted_src = src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as u16 * coeff as u16; for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; - let src0 = border_interpolate!( - src, - edge_mode, - i as i64 + mx, - 0, - width as i64, - N, - c - ); - let src1 = border_interpolate!( - src, - edge_mode, - rollback as i64 + mx, - 0, - width as i64, - N, - c - ); - - k0 += (src0 as u16 + src1 as u16) * coeff as u16; + k0 += (*shifted_src.get_unchecked(i * N) as u16 + + *shifted_src.get_unchecked(rollback * N) as u16) + * coeff as u16; } - *dst.get_unchecked_mut(f_cx * N + c) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + *dst.get_unchecked_mut(zx) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + + f_cx = m_right; + + while f_cx < width { + for c in 0..N { + let mx = f_cx as i64 - s_kernel; + let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * coeff as u16; + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + + let src0 = border_interpolate!( + src, + edge_mode, + i as i64 + mx, + 0, + width as i64, + N, + c + ); + let src1 = border_interpolate!( + src, + edge_mode, + rollback as i64 + mx, + 0, + width as i64, + N, + c + ); + + k0 += (src0 as u16 + src1 as u16) * coeff as u16; + } + + *dst.get_unchecked_mut(f_cx * N + c) = + ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + f_cx += 1; } - f_cx += 1; } } } diff --git a/src/filter1d/avx/sse_utils.rs b/src/filter1d/avx/sse_utils.rs index 3106adcf..0d3ef45f 100644 --- a/src/filter1d/avx/sse_utils.rs +++ b/src/filter1d/avx/sse_utils.rs @@ -33,25 +33,26 @@ use std::arch::x86::*; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_epi16_x4( - input: __m128i, - weight: __m128i, -) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); - - ( - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), - ) +pub(crate) fn _mm_mul_epi8_by_epi16_x4(input: __m128i, weight: __m128i) -> (__m128i, __m128i) { + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); + + ( + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), + ) + } } #[inline(always)] pub(crate) unsafe fn _mm_mul_epu16_widen(input: __m128i, weight: __m128i) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi16(input, _mm_setzero_si128()); - let k1 = _mm_unpackhi_epi16(input, _mm_setzero_si128()); + unsafe { + let k0 = _mm_unpacklo_epi16(input, _mm_setzero_si128()); + let k1 = _mm_unpackhi_epi16(input, _mm_setzero_si128()); - (_mm_mullo_epi32(k0, weight), _mm_mullo_epi32(k1, weight)) + (_mm_mullo_epi32(k0, weight), _mm_mullo_epi32(k1, weight)) + } } #[inline(always)] @@ -59,41 +60,45 @@ pub(crate) unsafe fn _mm_mul_epi8_by_ps_x4( input: __m128i, weight: __m128, ) -> (__m128, __m128, __m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); - let hi_16 = _mm_unpackhi_epi8(input, zeros); - - let a0 = _mm_unpacklo_epi16(lo_16, zeros); - let a1 = _mm_unpackhi_epi16(lo_16, zeros); - let a2 = _mm_unpacklo_epi16(hi_16, zeros); - let a3 = _mm_unpackhi_epi16(hi_16, zeros); - - let v0 = _mm_cvtepi32_ps(a0); - let v1 = _mm_cvtepi32_ps(a1); - let v2 = _mm_cvtepi32_ps(a2); - let v3 = _mm_cvtepi32_ps(a3); - - ( - _mm_mul_ps(v0, weight), - _mm_mul_ps(v1, weight), - _mm_mul_ps(v2, weight), - _mm_mul_ps(v3, weight), - ) + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); + let hi_16 = _mm_unpackhi_epi8(input, zeros); + + let a0 = _mm_unpacklo_epi16(lo_16, zeros); + let a1 = _mm_unpackhi_epi16(lo_16, zeros); + let a2 = _mm_unpacklo_epi16(hi_16, zeros); + let a3 = _mm_unpackhi_epi16(hi_16, zeros); + + let v0 = _mm_cvtepi32_ps(a0); + let v1 = _mm_cvtepi32_ps(a1); + let v2 = _mm_cvtepi32_ps(a2); + let v3 = _mm_cvtepi32_ps(a3); + + ( + _mm_mul_ps(v0, weight), + _mm_mul_ps(v1, weight), + _mm_mul_ps(v2, weight), + _mm_mul_ps(v3, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi16_by_ps_x2( +pub(crate) fn _mm_mul_epi16_by_ps_x2( input: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let a0 = _mm_unpacklo_epi16(input, zeros); - let a1 = _mm_unpackhi_epi16(input, zeros); + unsafe { + let zeros = _mm_setzero_si128(); + let a0 = _mm_unpacklo_epi16(input, zeros); + let a1 = _mm_unpackhi_epi16(input, zeros); - let v0 = _mm_cvtepi32_ps(a0); - let v1 = _mm_cvtepi32_ps(a1); + let v0 = _mm_cvtepi32_ps(a0); + let v1 = _mm_cvtepi32_ps(a1); - (_mm_mul_ps(v0, weight), _mm_mul_ps(v1, weight)) + (_mm_mul_ps(v0, weight), _mm_mul_ps(v1, weight)) + } } #[inline(always)] @@ -101,226 +106,246 @@ pub(crate) unsafe fn _mm_mull_epi8_by_epi16_x4( input: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); - - ( - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), - ) + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); + + ( + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); +pub(crate) fn _mm_mull_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_ps_x2(input: __m128i, weight: __m128) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); +pub(crate) fn _mm_mul_epi8_by_ps_x2(input: __m128i, weight: __m128) -> (__m128, __m128) { + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); - (_mm_mul_ps(a0, weight), _mm_mul_ps(a1, weight)) + (_mm_mul_ps(a0, weight), _mm_mul_ps(a1, weight)) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_ps(input: __m128i, weight: __m128) -> __m128 { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); +pub(crate) fn _mm_mul_epi8_by_ps(input: __m128i, weight: __m128) -> __m128 { + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); + let a0 = _mm_cvtepi32_ps(k0); - _mm_mul_ps(a0, weight) + _mm_mul_ps(a0, weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi16_by_ps( - input: __m128i, - weight: __m128, -) -> __m128 { - let zeros = _mm_setzero_si128(); - let k0 = _mm_unpacklo_epi16(input, zeros); - let a0 = _mm_cvtepi32_ps(k0); +pub(crate) fn _mm_mul_epi16_by_ps(input: __m128i, weight: __m128) -> __m128 { + unsafe { + let zeros = _mm_setzero_si128(); + let k0 = _mm_unpacklo_epi16(input, zeros); + let a0 = _mm_cvtepi32_ps(k0); - _mm_mul_ps(a0, weight) + _mm_mul_ps(a0, weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) +pub(crate) fn _mm_mul_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_opt_fmlaf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { - if FMA { - _mm_fmadd_ps(b, c, a) - } else { - _mm_add_ps(_mm_mul_ps(b, c), a) +pub(crate) fn _mm_opt_fmlaf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { + unsafe { + if FMA { + _mm_fmadd_ps(b, c, a) + } else { + _mm_add_ps(_mm_mul_ps(b, c), a) + } } } #[inline(always)] -pub(crate) unsafe fn _mm_opt_fmla_pd( - a: __m128d, - b: __m128d, - c: __m128d, -) -> __m128d { - if FMA { - _mm_fmadd_pd(b, c, a) - } else { - _mm_add_pd(_mm_mul_pd(b, c), a) +pub(crate) fn _mm_opt_fmla_pd(a: __m128d, b: __m128d, c: __m128d) -> __m128d { + unsafe { + if FMA { + _mm_fmadd_pd(b, c, a) + } else { + _mm_add_pd(_mm_mul_pd(b, c), a) + } } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_ps_x4( +pub(crate) fn _mm_mul_add_epi8_by_ps_x4( accumulator: (__m128, __m128, __m128, __m128), input: __m128i, weight: __m128, ) -> (__m128, __m128, __m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); - let hi_16 = _mm_unpackhi_epi8(input, zeros); - - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let k2 = _mm_unpacklo_epi16(hi_16, zeros); - let k3 = _mm_unpackhi_epi16(hi_16, zeros); - - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); - let a2 = _mm_cvtepi32_ps(k2); - let a3 = _mm_cvtepi32_ps(k3); - - ( - _mm_opt_fmlaf_ps::(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps::(accumulator.1, a1, weight), - _mm_opt_fmlaf_ps::(accumulator.2, a2, weight), - _mm_opt_fmlaf_ps::(accumulator.3, a3, weight), - ) + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); + let hi_16 = _mm_unpackhi_epi8(input, zeros); + + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k2 = _mm_unpacklo_epi16(hi_16, zeros); + let k3 = _mm_unpackhi_epi16(hi_16, zeros); + + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); + let a2 = _mm_cvtepi32_ps(k2); + let a3 = _mm_cvtepi32_ps(k3); + + ( + _mm_opt_fmlaf_ps::(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps::(accumulator.1, a1, weight), + _mm_opt_fmlaf_ps::(accumulator.2, a2, weight), + _mm_opt_fmlaf_ps::(accumulator.3, a3, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_epi8_by_epi16_x4( +pub(crate) fn _mm_mull_add_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); - - let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); - let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); - ( - _mm_add_epi16(accumulator.0, j0), - _mm_add_epi16(accumulator.1, j1), - ) + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); + + let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); + let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); + ( + _mm_add_epi16(accumulator.0, j0), + _mm_add_epi16(accumulator.1, j1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_epi8_by_epi16_x2( +pub(crate) fn _mm_mull_add_epi8_by_epi16_x2( accumulator: __m128i, input: __m128i, weight: __m128i, ) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_add_epi16( - accumulator, - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), - ) + _mm_add_epi16( + accumulator, + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_symm_epi8_by_epi16_x4( +pub(crate) fn _mm_mull_add_symm_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); - let hi_16 = _mm_add_epi16( - _mm_unpackhi_epi8(input0, zeros), - _mm_unpackhi_epi8(input1, zeros), - ); - let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight); - let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(hi_16), weight); - - ( - _mm_add_epi16(accumulator.0, j0), - _mm_add_epi16(accumulator.1, j1), - ) -} - -#[inline(always)] -pub(crate) unsafe fn _mm_madd_symm_epi8_by_epi16_x4( + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); + let hi_16 = _mm_add_epi16( + _mm_unpackhi_epi8(input0, zeros), + _mm_unpackhi_epi8(input1, zeros), + ); + let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight); + let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(hi_16), weight); + + ( + _mm_add_epi16(accumulator.0, j0), + _mm_add_epi16(accumulator.1, j1), + ) + } +} + +#[inline(always)] +pub(crate) fn _mm_madd_symm_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); - let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); - let v2 = _mm_unpacklo_epi8(input1, _mm_setzero_si128()); - let v3 = _mm_unpackhi_epi8(input1, _mm_setzero_si128()); - - let m0 = _mm_add_epi16(v0, v2); - let m1 = _mm_add_epi16(v1, v3); - - let a0 = _mm_mullo_epi16(m0, weight); - let a1 = _mm_mullo_epi16(m1, weight); - - ( - _mm_add_epi16(accumulator.0, a0), - _mm_add_epi16(accumulator.1, a1), - ) + unsafe { + let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); + let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); + let v2 = _mm_unpacklo_epi8(input1, _mm_setzero_si128()); + let v3 = _mm_unpackhi_epi8(input1, _mm_setzero_si128()); + + let m0 = _mm_add_epi16(v0, v2); + let m1 = _mm_add_epi16(v1, v3); + + let a0 = _mm_mullo_epi16(m0, weight); + let a1 = _mm_mullo_epi16(m1, weight); + + ( + _mm_add_epi16(accumulator.0, a0), + _mm_add_epi16(accumulator.1, a1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_madd_s_epi8_by_epi16_x4( +pub(crate) fn _mm_madd_s_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); - let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); + unsafe { + let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); + let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); - let a0 = _mm_mullo_epi16(v0, weight); - let a1 = _mm_mullo_epi16(v1, weight); + let a0 = _mm_mullo_epi16(v0, weight); + let a1 = _mm_mullo_epi16(v1, weight); - ( - _mm_add_epi16(accumulator.0, a0), - _mm_add_epi16(accumulator.1, a1), - ) + ( + _mm_add_epi16(accumulator.0, a0), + _mm_add_epi16(accumulator.1, a1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_madd_epi8_by_epi16_x4( - input0: __m128i, - weight: __m128i, -) -> (__m128i, __m128i) { - let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); - let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); +pub(crate) fn _mm_madd_epi8_by_epi16_x4(input0: __m128i, weight: __m128i) -> (__m128i, __m128i) { + unsafe { + let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); + let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); - let a0 = _mm_mullo_epi16(v0, weight); - let a1 = _mm_mullo_epi16(v1, weight); + let a0 = _mm_mullo_epi16(v0, weight); + let a1 = _mm_mullo_epi16(v1, weight); - (a0, a1) + (a0, a1) + } } #[inline(always)] @@ -330,330 +355,365 @@ pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_ps_x4( input1: __m128i, weight: __m128, ) -> (__m128, __m128, __m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); - let hi_16 = _mm_add_epi16( - _mm_unpackhi_epi8(input0, zeros), - _mm_unpackhi_epi8(input1, zeros), - ); - - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let k2 = _mm_unpacklo_epi16(hi_16, zeros); - let k3 = _mm_unpackhi_epi16(hi_16, zeros); - - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); - let a2 = _mm_cvtepi32_ps(k2); - let a3 = _mm_cvtepi32_ps(k3); - - ( - _mm_opt_fmlaf_ps::(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps::(accumulator.1, a1, weight), - _mm_opt_fmlaf_ps::(accumulator.2, a2, weight), - _mm_opt_fmlaf_ps::(accumulator.3, a3, weight), - ) -} - -#[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi16_by_ps_x2( + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); + let hi_16 = _mm_add_epi16( + _mm_unpackhi_epi8(input0, zeros), + _mm_unpackhi_epi8(input1, zeros), + ); + + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k2 = _mm_unpacklo_epi16(hi_16, zeros); + let k3 = _mm_unpackhi_epi16(hi_16, zeros); + + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); + let a2 = _mm_cvtepi32_ps(k2); + let a3 = _mm_cvtepi32_ps(k3); + + ( + _mm_opt_fmlaf_ps::(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps::(accumulator.1, a1, weight), + _mm_opt_fmlaf_ps::(accumulator.2, a2, weight), + _mm_opt_fmlaf_ps::(accumulator.3, a3, weight), + ) + } +} + +#[inline(always)] +pub(crate) fn _mm_mul_add_symm_epi16_by_ps_x2( accumulator: (__m128, __m128), input0: __m128i, input1: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let l32 = _mm_add_epi32( - _mm_unpacklo_epi16(input0, zeros), - _mm_unpacklo_epi16(input1, zeros), - ); - let h32 = _mm_add_epi32( - _mm_unpackhi_epi16(input0, zeros), - _mm_unpackhi_epi16(input1, zeros), - ); - - let a0 = _mm_cvtepi32_ps(l32); - let a1 = _mm_cvtepi32_ps(h32); - - ( - _mm_opt_fmlaf_ps::(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps::(accumulator.1, a1, weight), - ) + unsafe { + let zeros = _mm_setzero_si128(); + let l32 = _mm_add_epi32( + _mm_unpacklo_epi16(input0, zeros), + _mm_unpacklo_epi16(input1, zeros), + ); + let h32 = _mm_add_epi32( + _mm_unpackhi_epi16(input0, zeros), + _mm_unpackhi_epi16(input1, zeros), + ); + + let a0 = _mm_cvtepi32_ps(l32); + let a1 = _mm_cvtepi32_ps(h32); + + ( + _mm_opt_fmlaf_ps::(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps::(accumulator.1, a1, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_ps_x2( +pub(crate) fn _mm_mul_add_epi8_by_ps_x2( accumulator: (__m128, __m128), input: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); - ( - _mm_opt_fmlaf_ps::(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps::(accumulator.1, a1, weight), - ) + ( + _mm_opt_fmlaf_ps::(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps::(accumulator.1, a1, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_ps_x2( +pub(crate) fn _mm_mul_add_symm_epi8_by_ps_x2( accumulator: (__m128, __m128), input0: __m128i, input1: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); - - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); - - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); - - ( - _mm_opt_fmlaf_ps::(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps::(accumulator.1, a1, weight), - ) + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); + + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); + + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); + + ( + _mm_opt_fmlaf_ps::(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps::(accumulator.1, a1, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_ps( +pub(crate) fn _mm_mul_add_symm_epi8_by_ps( accumulator: __m128, input0: __m128i, input1: __m128i, weight: __m128, ) -> __m128 { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); + let a0 = _mm_cvtepi32_ps(k0); - _mm_opt_fmlaf_ps::(accumulator, a0, weight) + _mm_opt_fmlaf_ps::(accumulator, a0, weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi16_by_ps( +pub(crate) fn _mm_mul_add_symm_epi16_by_ps( accumulator: __m128, input0: __m128i, input1: __m128i, weight: __m128, ) -> __m128 { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi32( - _mm_unpacklo_epi16(input0, zeros), - _mm_unpacklo_epi16(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi32( + _mm_unpacklo_epi16(input0, zeros), + _mm_unpacklo_epi16(input1, zeros), + ); - let a0 = _mm_cvtepi32_ps(lo_16); + let a0 = _mm_cvtepi32_ps(lo_16); - _mm_opt_fmlaf_ps::(accumulator, a0, weight) + _mm_opt_fmlaf_ps::(accumulator, a0, weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_symm_epi8_by_epi16_x2( +pub(crate) fn _mm_mull_add_symm_epi8_by_epi16_x2( accumulator: __m128i, input0: __m128i, input1: __m128i, weight: __m128i, ) -> __m128i { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); - - _mm_add_epi16(accumulator, _mm_mulhrs_epi16(lo_16, weight)) + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); + + _mm_add_epi16(accumulator, _mm_mulhrs_epi16(lo_16, weight)) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_epi16_x4( +pub(crate) fn _mm_mul_add_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); - let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); - let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); + let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); + let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); - ( - _mm_add_epi16(j0, accumulator.0), - _mm_add_epi16(j1, accumulator.1), - ) + ( + _mm_add_epi16(j0, accumulator.0), + _mm_add_epi16(j1, accumulator.1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_epi16_x4( +pub(crate) fn _mm_mul_add_symm_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let zeros = _mm_setzero_si128(); - let k0 = _mm_unpacklo_epi8(input0, zeros); - let k1 = _mm_unpacklo_epi8(input1, zeros); - let k2 = _mm_unpackhi_epi8(input0, zeros); - let k3 = _mm_unpackhi_epi8(input1, zeros); - let v0 = _mm_add_epi16(k0, k1); - let v1 = _mm_add_epi16(k2, k3); - - let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v0), weight); - let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v1), weight); - - ( - _mm_add_epi16(j0, accumulator.0), - _mm_add_epi16(j1, accumulator.1), - ) + unsafe { + let zeros = _mm_setzero_si128(); + let k0 = _mm_unpacklo_epi8(input0, zeros); + let k1 = _mm_unpacklo_epi8(input1, zeros); + let k2 = _mm_unpackhi_epi8(input0, zeros); + let k3 = _mm_unpackhi_epi8(input1, zeros); + let v0 = _mm_add_epi16(k0, k1); + let v1 = _mm_add_epi16(k2, k3); + + let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v0), weight); + let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v1), weight); + + ( + _mm_add_epi16(j0, accumulator.0), + _mm_add_epi16(j1, accumulator.1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epu16_by_epu16_x4( +pub(crate) fn _mm_mul_add_symm_epu16_by_epu16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let zeros = _mm_setzero_si128(); - let k0 = _mm_unpacklo_epi16(input0, zeros); - let k1 = _mm_unpacklo_epi16(input1, zeros); - let k2 = _mm_unpackhi_epi16(input0, zeros); - let k3 = _mm_unpackhi_epi16(input1, zeros); - let v0 = _mm_add_epi32(k0, k1); - let v1 = _mm_add_epi32(k2, k3); - - let j0 = _mm_mullo_epi32(v0, weight); - let j1 = _mm_mullo_epi32(v1, weight); - - ( - _mm_add_epi32(j0, accumulator.0), - _mm_add_epi32(j1, accumulator.1), - ) + unsafe { + let zeros = _mm_setzero_si128(); + let k0 = _mm_unpacklo_epi16(input0, zeros); + let k1 = _mm_unpacklo_epi16(input1, zeros); + let k2 = _mm_unpackhi_epi16(input0, zeros); + let k3 = _mm_unpackhi_epi16(input1, zeros); + let v0 = _mm_add_epi32(k0, k1); + let v1 = _mm_add_epi32(k2, k3); + + let j0 = _mm_mullo_epi32(v0, weight); + let j1 = _mm_mullo_epi32(v1, weight); + + ( + _mm_add_epi32(j0, accumulator.0), + _mm_add_epi32(j1, accumulator.1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_epi16_x2( +pub(crate) fn _mm_mul_add_epi8_by_epi16_x2( accumulator: __m128i, input: __m128i, weight: __m128i, ) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_add_epi16( - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), - accumulator, - ) + _mm_add_epi16( + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), + accumulator, + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_epi16_x2( +pub(crate) fn _mm_mul_add_symm_epi8_by_epi16_x2( accumulator: __m128i, input0: __m128i, input1: __m128i, weight: __m128i, ) -> __m128i { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); - - _mm_add_epi16( - _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight), - accumulator, - ) + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); + + _mm_add_epi16( + _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight), + accumulator, + ) + } } #[inline(always)] pub(crate) unsafe fn _mm_pack_epi32_x2_epi8(store: (__m128i, __m128i)) -> __m128i { - let rnd = _mm_set1_epi16((1 << 5) - 1); - _mm_packus_epi16( - _mm_srai_epi16::<6>(_mm_add_epi16(store.0, rnd)), - _mm_srai_epi16::<6>(_mm_add_epi16(store.1, rnd)), - ) + unsafe { + let rnd = _mm_set1_epi16((1 << 5) - 1); + _mm_packus_epi16( + _mm_srai_epi16::<6>(_mm_add_epi16(store.0, rnd)), + _mm_srai_epi16::<6>(_mm_add_epi16(store.1, rnd)), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_epi32_x2_epu16(store: (__m128i, __m128i)) -> __m128i { - let rnd = _mm_set1_epi32((1 << 14) - 1); - _mm_packus_epi32( - _mm_srli_epi32::<15>(_mm_add_epi32(store.0, rnd)), - _mm_srli_epi32::<15>(_mm_add_epi32(store.1, rnd)), - ) +pub(crate) fn _mm_pack_epi32_x2_epu16(store: (__m128i, __m128i)) -> __m128i { + unsafe { + let rnd = _mm_set1_epi32((1 << 14) - 1); + _mm_packus_epi32( + _mm_srli_epi32::<15>(_mm_add_epi32(store.0, rnd)), + _mm_srli_epi32::<15>(_mm_add_epi32(store.1, rnd)), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_epi32_epi8(store: __m128i) -> __m128i { - let rnd = _mm_set1_epi16((1 << 5) - 1); - _mm_packus_epi16( - _mm_srai_epi16::<6>(_mm_add_epi16(store, rnd)), - _mm_setzero_si128(), - ) +pub(crate) fn _mm_pack_epi32_epi8(store: __m128i) -> __m128i { + unsafe { + let rnd = _mm_set1_epi16((1 << 5) - 1); + _mm_packus_epi16( + _mm_srai_epi16::<6>(_mm_add_epi16(store, rnd)), + _mm_setzero_si128(), + ) + } } #[inline(always)] pub(crate) unsafe fn _mm_pack_ps_x4_epi8(store: (__m128, __m128, __m128, __m128)) -> __m128i { - let o0 = _mm_cvtps_epi32(store.0); - let o1 = _mm_cvtps_epi32(store.1); - let o2 = _mm_cvtps_epi32(store.2); - let o3 = _mm_cvtps_epi32(store.3); - let hi_s = _mm_packs_epi32(o2, o3); - let lo_s = _mm_packs_epi32(o0, o1); - _mm_packus_epi16(lo_s, hi_s) + unsafe { + let o0 = _mm_cvtps_epi32(store.0); + let o1 = _mm_cvtps_epi32(store.1); + let o2 = _mm_cvtps_epi32(store.2); + let o3 = _mm_cvtps_epi32(store.3); + let hi_s = _mm_packs_epi32(o2, o3); + let lo_s = _mm_packs_epi32(o0, o1); + _mm_packus_epi16(lo_s, hi_s) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_x2_epi16(store: (__m128, __m128)) -> __m128i { - let o0 = _mm_cvtps_epi32(store.0); - let o1 = _mm_cvtps_epi32(store.1); - _mm_packus_epi32(o0, o1) +pub(crate) fn _mm_pack_ps_x2_epi16(store: (__m128, __m128)) -> __m128i { + unsafe { + let o0 = _mm_cvtps_epi32(store.0); + let o1 = _mm_cvtps_epi32(store.1); + _mm_packus_epi32(o0, o1) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_x2_epi8(store: (__m128, __m128)) -> __m128i { - let lo_s = _mm_packs_epi32(_mm_cvtps_epi32(store.0), _mm_cvtps_epi32(store.1)); - _mm_packus_epi16(lo_s, _mm_setzero_si128()) +pub(crate) fn _mm_pack_ps_x2_epi8(store: (__m128, __m128)) -> __m128i { + unsafe { + let lo_s = _mm_packs_epi32(_mm_cvtps_epi32(store.0), _mm_cvtps_epi32(store.1)); + _mm_packus_epi16(lo_s, _mm_setzero_si128()) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_epi8(store: __m128) -> __m128i { - let lo_s = _mm_packs_epi32(_mm_cvtps_epi32(store), _mm_setzero_si128()); - _mm_packus_epi16(lo_s, _mm_setzero_si128()) +pub(crate) fn _mm_pack_ps_epi8(store: __m128) -> __m128i { + unsafe { + let lo_s = _mm_packs_epi32(_mm_cvtps_epi32(store), _mm_setzero_si128()); + _mm_packus_epi16(lo_s, _mm_setzero_si128()) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_epi16(store: __m128) -> __m128i { - _mm_packus_epi32(_mm_cvtps_epi32(store), _mm_setzero_si128()) +pub(crate) fn _mm_pack_ps_epi16(store: __m128) -> __m128i { + unsafe { _mm_packus_epi32(_mm_cvtps_epi32(store), _mm_setzero_si128()) } } #[inline(always)] -pub(crate) unsafe fn _mm256_packus_four_epi32( - a: __m256i, - b: __m256i, - c: __m256i, - d: __m256i, -) -> __m256i { - let ab = _mm256_packs_epi32(a, b); - let cd = _mm256_packs_epi32(c, d); +pub(crate) fn _mm256_packus_four_epi32(a: __m256i, b: __m256i, c: __m256i, d: __m256i) -> __m256i { + unsafe { + let ab = _mm256_packs_epi32(a, b); + let cd = _mm256_packs_epi32(c, d); - _mm256_packus_epi16(ab, cd) + _mm256_packus_epi16(ab, cd) + } } diff --git a/src/filter1d/avx/utils.rs b/src/filter1d/avx/utils.rs index 15d7078c..5b3ef92a 100644 --- a/src/filter1d/avx/utils.rs +++ b/src/filter1d/avx/utils.rs @@ -30,247 +30,260 @@ use crate::filter1d::avx::sse_utils::_mm256_packus_four_epi32; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm256_mul_epi8_by_ps_x4( +pub(crate) fn _mm256_mul_epi8_by_ps_x4( input: __m256i, weight: __m256, ) -> (__m256, __m256, __m256, __m256) { - let lo_16 = _mm256_unpacklo_epi8(input, _mm256_setzero_si256()); - let hi_16 = _mm256_unpackhi_epi8(input, _mm256_setzero_si256()); - - let j0 = _mm256_unpacklo_epi16(lo_16, _mm256_setzero_si256()); - let j1 = _mm256_unpackhi_epi16(lo_16, _mm256_setzero_si256()); - let j2 = _mm256_unpacklo_epi16(hi_16, _mm256_setzero_si256()); - let j3 = _mm256_unpackhi_epi16(hi_16, _mm256_setzero_si256()); - - let o0 = _mm256_cvtepi32_ps(j0); - let o1 = _mm256_cvtepi32_ps(j1); - let o2 = _mm256_cvtepi32_ps(j2); - let o3 = _mm256_cvtepi32_ps(j3); - - ( - _mm256_mul_ps(o0, weight), - _mm256_mul_ps(o1, weight), - _mm256_mul_ps(o2, weight), - _mm256_mul_ps(o3, weight), - ) + unsafe { + let lo_16 = _mm256_unpacklo_epi8(input, _mm256_setzero_si256()); + let hi_16 = _mm256_unpackhi_epi8(input, _mm256_setzero_si256()); + + let j0 = _mm256_unpacklo_epi16(lo_16, _mm256_setzero_si256()); + let j1 = _mm256_unpackhi_epi16(lo_16, _mm256_setzero_si256()); + let j2 = _mm256_unpacklo_epi16(hi_16, _mm256_setzero_si256()); + let j3 = _mm256_unpackhi_epi16(hi_16, _mm256_setzero_si256()); + + let o0 = _mm256_cvtepi32_ps(j0); + let o1 = _mm256_cvtepi32_ps(j1); + let o2 = _mm256_cvtepi32_ps(j2); + let o3 = _mm256_cvtepi32_ps(j3); + + ( + _mm256_mul_ps(o0, weight), + _mm256_mul_ps(o1, weight), + _mm256_mul_ps(o2, weight), + _mm256_mul_ps(o3, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_epi16_by_ps_x2( +pub(crate) fn _mm256_mul_epi16_by_ps_x2( input: __m256i, weight: __m256, ) -> (__m256, __m256) { - let j0 = _mm256_unpacklo_epi16(input, _mm256_setzero_si256()); - let j1 = _mm256_unpackhi_epi16(input, _mm256_setzero_si256()); + unsafe { + let j0 = _mm256_unpacklo_epi16(input, _mm256_setzero_si256()); + let j1 = _mm256_unpackhi_epi16(input, _mm256_setzero_si256()); - let o0 = _mm256_cvtepi32_ps(j0); - let o1 = _mm256_cvtepi32_ps(j1); + let o0 = _mm256_cvtepi32_ps(j0); + let o1 = _mm256_cvtepi32_ps(j1); - (_mm256_mul_ps(o0, weight), _mm256_mul_ps(o1, weight)) + (_mm256_mul_ps(o0, weight), _mm256_mul_ps(o1, weight)) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_epi8_by_epi16_x4( - input: __m256i, - weight: __m256i, -) -> (__m256i, __m256i) { - let j0 = _mm256_unpacklo_epi8(input, input); - let j1 = _mm256_unpackhi_epi8(input, input); - - ( - _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j0), weight), - _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j1), weight), - ) +pub(crate) fn _mm256_mul_epi8_by_epi16_x4(input: __m256i, weight: __m256i) -> (__m256i, __m256i) { + unsafe { + let j0 = _mm256_unpacklo_epi8(input, input); + let j1 = _mm256_unpackhi_epi8(input, input); + + ( + _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j0), weight), + _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j1), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_epu16_widen(input: __m256i, weight: __m256i) -> (__m256i, __m256i) { - let v_lo = _mm256_unpacklo_epi16(input, _mm256_setzero_si256()); - let v_hi = _mm256_unpackhi_epi16(input, _mm256_setzero_si256()); - ( - _mm256_mullo_epi32(v_lo, weight), - _mm256_mullo_epi32(v_hi, weight), - ) +pub(crate) fn _mm256_mul_epu16_widen(input: __m256i, weight: __m256i) -> (__m256i, __m256i) { + unsafe { + let v_lo = _mm256_unpacklo_epi16(input, _mm256_setzero_si256()); + let v_hi = _mm256_unpackhi_epi16(input, _mm256_setzero_si256()); + ( + _mm256_mullo_epi32(v_lo, weight), + _mm256_mullo_epi32(v_hi, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_add_epi8_by_epi16_x4( +pub(crate) fn _mm256_mul_add_epi8_by_epi16_x4( accumulator: (__m256i, __m256i), input: __m256i, weight: __m256i, ) -> (__m256i, __m256i) { - let j0 = _mm256_unpacklo_epi8(input, input); - let j1 = _mm256_unpackhi_epi8(input, input); - - let vj0 = _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j0), weight); - let vj1 = _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j1), weight); - ( - _mm256_add_epi16(accumulator.0, vj0), - _mm256_add_epi16(accumulator.1, vj1), - ) + unsafe { + let j0 = _mm256_unpacklo_epi8(input, input); + let j1 = _mm256_unpackhi_epi8(input, input); + + let vj0 = _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j0), weight); + let vj1 = _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j1), weight); + ( + _mm256_add_epi16(accumulator.0, vj0), + _mm256_add_epi16(accumulator.1, vj1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_add_symm_epi8_by_epi16_x4( +pub(crate) fn _mm256_mul_add_symm_epi8_by_epi16_x4( accumulator: (__m256i, __m256i), input0: __m256i, input1: __m256i, weight: __m256i, ) -> (__m256i, __m256i) { - let a0 = _mm256_unpacklo_epi8(input0, _mm256_setzero_si256()); - let a1 = _mm256_unpacklo_epi8(input1, _mm256_setzero_si256()); - let a2 = _mm256_unpackhi_epi8(input0, _mm256_setzero_si256()); - let a3 = _mm256_unpackhi_epi8(input1, _mm256_setzero_si256()); - - let lo_16 = _mm256_add_epi16(a0, a1); - let hi_16 = _mm256_add_epi16(a2, a3); - - let vj0 = _mm256_mulhrs_epi16(_mm256_slli_epi16::<6>(lo_16), weight); - let vj1 = _mm256_mulhrs_epi16(_mm256_slli_epi16::<6>(hi_16), weight); - - ( - _mm256_add_epi16(accumulator.0, vj0), - _mm256_add_epi16(accumulator.1, vj1), - ) + unsafe { + let a0 = _mm256_unpacklo_epi8(input0, _mm256_setzero_si256()); + let a1 = _mm256_unpacklo_epi8(input1, _mm256_setzero_si256()); + let a2 = _mm256_unpackhi_epi8(input0, _mm256_setzero_si256()); + let a3 = _mm256_unpackhi_epi8(input1, _mm256_setzero_si256()); + + let lo_16 = _mm256_add_epi16(a0, a1); + let hi_16 = _mm256_add_epi16(a2, a3); + + let vj0 = _mm256_mulhrs_epi16(_mm256_slli_epi16::<6>(lo_16), weight); + let vj1 = _mm256_mulhrs_epi16(_mm256_slli_epi16::<6>(hi_16), weight); + + ( + _mm256_add_epi16(accumulator.0, vj0), + _mm256_add_epi16(accumulator.1, vj1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_add_symm_epu16_by_epu16_x4( +pub(crate) fn _mm256_mul_add_symm_epu16_by_epu16_x4( accumulator: (__m256i, __m256i), input0: __m256i, input1: __m256i, weight: __m256i, ) -> (__m256i, __m256i) { - let a0 = _mm256_unpacklo_epi16(input0, _mm256_setzero_si256()); - let a1 = _mm256_unpacklo_epi16(input1, _mm256_setzero_si256()); - let a2 = _mm256_unpackhi_epi16(input0, _mm256_setzero_si256()); - let a3 = _mm256_unpackhi_epi16(input1, _mm256_setzero_si256()); - - let lo_16 = _mm256_add_epi32(a0, a1); - let hi_16 = _mm256_add_epi32(a2, a3); - - let vj0 = _mm256_mullo_epi32(lo_16, weight); - let vj1 = _mm256_mullo_epi32(hi_16, weight); - - ( - _mm256_add_epi32(accumulator.0, vj0), - _mm256_add_epi32(accumulator.1, vj1), - ) + unsafe { + let a0 = _mm256_unpacklo_epi16(input0, _mm256_setzero_si256()); + let a1 = _mm256_unpacklo_epi16(input1, _mm256_setzero_si256()); + let a2 = _mm256_unpackhi_epi16(input0, _mm256_setzero_si256()); + let a3 = _mm256_unpackhi_epi16(input1, _mm256_setzero_si256()); + + let lo_16 = _mm256_add_epi32(a0, a1); + let hi_16 = _mm256_add_epi32(a2, a3); + + let vj0 = _mm256_mullo_epi32(lo_16, weight); + let vj1 = _mm256_mullo_epi32(hi_16, weight); + + ( + _mm256_add_epi32(accumulator.0, vj0), + _mm256_add_epi32(accumulator.1, vj1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_opt_fmlaf_ps( - a: __m256, - b: __m256, - c: __m256, -) -> __m256 { - if FMA { - _mm256_fmadd_ps(b, c, a) - } else { - _mm256_add_ps(_mm256_mul_ps(b, c), a) +pub(crate) fn _mm256_opt_fmlaf_ps(a: __m256, b: __m256, c: __m256) -> __m256 { + unsafe { + if FMA { + _mm256_fmadd_ps(b, c, a) + } else { + _mm256_add_ps(_mm256_mul_ps(b, c), a) + } } } #[inline(always)] -pub(crate) unsafe fn _mm256_opt_fmla_pd( - a: __m256d, - b: __m256d, - c: __m256d, -) -> __m256d { - if FMA { - _mm256_fmadd_pd(b, c, a) - } else { - _mm256_add_pd(_mm256_mul_pd(b, c), a) +pub(crate) fn _mm256_opt_fmla_pd(a: __m256d, b: __m256d, c: __m256d) -> __m256d { + unsafe { + if FMA { + _mm256_fmadd_pd(b, c, a) + } else { + _mm256_add_pd(_mm256_mul_pd(b, c), a) + } } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_add_epi8_by_ps_x4( +pub(crate) fn _mm256_mul_add_epi8_by_ps_x4( accumulator: (__m256, __m256, __m256, __m256), input: __m256i, weight: __m256, ) -> (__m256, __m256, __m256, __m256) { - let lo_16 = _mm256_unpacklo_epi8(input, _mm256_setzero_si256()); - let hi_16 = _mm256_unpackhi_epi8(input, _mm256_setzero_si256()); - - let j0 = _mm256_unpacklo_epi16(lo_16, _mm256_setzero_si256()); - let j1 = _mm256_unpackhi_epi16(lo_16, _mm256_setzero_si256()); - let j2 = _mm256_unpacklo_epi16(hi_16, _mm256_setzero_si256()); - let j3 = _mm256_unpackhi_epi16(hi_16, _mm256_setzero_si256()); - - let a0 = _mm256_cvtepi32_ps(j0); - let a1 = _mm256_cvtepi32_ps(j1); - let a2 = _mm256_cvtepi32_ps(j2); - let a3 = _mm256_cvtepi32_ps(j3); - - ( - _mm256_opt_fmlaf_ps::(accumulator.0, a0, weight), - _mm256_opt_fmlaf_ps::(accumulator.1, a1, weight), - _mm256_opt_fmlaf_ps::(accumulator.2, a2, weight), - _mm256_opt_fmlaf_ps::(accumulator.3, a3, weight), - ) + unsafe { + let lo_16 = _mm256_unpacklo_epi8(input, _mm256_setzero_si256()); + let hi_16 = _mm256_unpackhi_epi8(input, _mm256_setzero_si256()); + + let j0 = _mm256_unpacklo_epi16(lo_16, _mm256_setzero_si256()); + let j1 = _mm256_unpackhi_epi16(lo_16, _mm256_setzero_si256()); + let j2 = _mm256_unpacklo_epi16(hi_16, _mm256_setzero_si256()); + let j3 = _mm256_unpackhi_epi16(hi_16, _mm256_setzero_si256()); + + let a0 = _mm256_cvtepi32_ps(j0); + let a1 = _mm256_cvtepi32_ps(j1); + let a2 = _mm256_cvtepi32_ps(j2); + let a3 = _mm256_cvtepi32_ps(j3); + + ( + _mm256_opt_fmlaf_ps::(accumulator.0, a0, weight), + _mm256_opt_fmlaf_ps::(accumulator.1, a1, weight), + _mm256_opt_fmlaf_ps::(accumulator.2, a2, weight), + _mm256_opt_fmlaf_ps::(accumulator.3, a3, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_add_symm_epi8_by_ps_x4( +pub(crate) fn _mm256_mul_add_symm_epi8_by_ps_x4( accumulator: (__m256, __m256, __m256, __m256), input0: __m256i, input1: __m256i, weight: __m256, ) -> (__m256, __m256, __m256, __m256) { - let j0 = _mm256_unpacklo_epi8(input0, _mm256_setzero_si256()); - let j1 = _mm256_unpacklo_epi8(input1, _mm256_setzero_si256()); - let j2 = _mm256_unpackhi_epi8(input0, _mm256_setzero_si256()); - let j3 = _mm256_unpackhi_epi8(input1, _mm256_setzero_si256()); - - let lo_16 = _mm256_add_epi16(j0, j1); - let hi_16 = _mm256_add_epi16(j2, j3); - - let a0 = _mm256_unpacklo_epi16(lo_16, _mm256_setzero_si256()); - let a1 = _mm256_unpackhi_epi16(lo_16, _mm256_setzero_si256()); - let a2 = _mm256_unpacklo_epi16(hi_16, _mm256_setzero_si256()); - let a3 = _mm256_unpackhi_epi16(hi_16, _mm256_setzero_si256()); - - let v0 = _mm256_cvtepi32_ps(a0); - let v1 = _mm256_cvtepi32_ps(a1); - let v2 = _mm256_cvtepi32_ps(a2); - let v3 = _mm256_cvtepi32_ps(a3); - - ( - _mm256_opt_fmlaf_ps::(accumulator.0, v0, weight), - _mm256_opt_fmlaf_ps::(accumulator.1, v1, weight), - _mm256_opt_fmlaf_ps::(accumulator.2, v2, weight), - _mm256_opt_fmlaf_ps::(accumulator.3, v3, weight), - ) + unsafe { + let j0 = _mm256_unpacklo_epi8(input0, _mm256_setzero_si256()); + let j1 = _mm256_unpacklo_epi8(input1, _mm256_setzero_si256()); + let j2 = _mm256_unpackhi_epi8(input0, _mm256_setzero_si256()); + let j3 = _mm256_unpackhi_epi8(input1, _mm256_setzero_si256()); + + let lo_16 = _mm256_add_epi16(j0, j1); + let hi_16 = _mm256_add_epi16(j2, j3); + + let a0 = _mm256_unpacklo_epi16(lo_16, _mm256_setzero_si256()); + let a1 = _mm256_unpackhi_epi16(lo_16, _mm256_setzero_si256()); + let a2 = _mm256_unpacklo_epi16(hi_16, _mm256_setzero_si256()); + let a3 = _mm256_unpackhi_epi16(hi_16, _mm256_setzero_si256()); + + let v0 = _mm256_cvtepi32_ps(a0); + let v1 = _mm256_cvtepi32_ps(a1); + let v2 = _mm256_cvtepi32_ps(a2); + let v3 = _mm256_cvtepi32_ps(a3); + + ( + _mm256_opt_fmlaf_ps::(accumulator.0, v0, weight), + _mm256_opt_fmlaf_ps::(accumulator.1, v1, weight), + _mm256_opt_fmlaf_ps::(accumulator.2, v2, weight), + _mm256_opt_fmlaf_ps::(accumulator.3, v3, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_add_symm_epi16_by_ps_x2( +pub(crate) fn _mm256_mul_add_symm_epi16_by_ps_x2( accumulator: (__m256, __m256), input0: __m256i, input1: __m256i, weight: __m256, ) -> (__m256, __m256) { - let j0 = _mm256_unpacklo_epi16(input0, _mm256_setzero_si256()); - let j1 = _mm256_unpacklo_epi16(input1, _mm256_setzero_si256()); - let j2 = _mm256_unpackhi_epi16(input0, _mm256_setzero_si256()); - let j3 = _mm256_unpackhi_epi16(input1, _mm256_setzero_si256()); - - let a0 = _mm256_add_epi32(j0, j1); - let a1 = _mm256_add_epi32(j2, j3); - - let v0 = _mm256_cvtepi32_ps(a0); - let v1 = _mm256_cvtepi32_ps(a1); - - ( - _mm256_opt_fmlaf_ps::(accumulator.0, v0, weight), - _mm256_opt_fmlaf_ps::(accumulator.1, v1, weight), - ) + unsafe { + let j0 = _mm256_unpacklo_epi16(input0, _mm256_setzero_si256()); + let j1 = _mm256_unpacklo_epi16(input1, _mm256_setzero_si256()); + let j2 = _mm256_unpackhi_epi16(input0, _mm256_setzero_si256()); + let j3 = _mm256_unpackhi_epi16(input1, _mm256_setzero_si256()); + + let a0 = _mm256_add_epi32(j0, j1); + let a1 = _mm256_add_epi32(j2, j3); + + let v0 = _mm256_cvtepi32_ps(a0); + let v1 = _mm256_cvtepi32_ps(a1); + + ( + _mm256_opt_fmlaf_ps::(accumulator.0, v0, weight), + _mm256_opt_fmlaf_ps::(accumulator.1, v1, weight), + ) + } } #[inline] #[target_feature(enable = "avx2")] -pub(crate) unsafe fn _mm256_pack_ps_x4_epi8(store: (__m256, __m256, __m256, __m256)) -> __m256i { +pub(crate) fn _mm256_pack_ps_x4_epi8(store: (__m256, __m256, __m256, __m256)) -> __m256i { let v0 = _mm256_cvtps_epi32(store.0); let v1 = _mm256_cvtps_epi32(store.1); let v2 = _mm256_cvtps_epi32(store.2); @@ -281,7 +294,7 @@ pub(crate) unsafe fn _mm256_pack_ps_x4_epi8(store: (__m256, __m256, __m256, __m2 #[inline] #[target_feature(enable = "avx2")] -pub(crate) unsafe fn _mm256_pack_ps_x2_epi16(store: (__m256, __m256)) -> __m256i { +pub(crate) fn _mm256_pack_ps_x2_epi16(store: (__m256, __m256)) -> __m256i { let v0 = _mm256_cvtps_epi32(store.0); let v1 = _mm256_cvtps_epi32(store.1); _mm256_packus_epi32(v0, v1) @@ -289,7 +302,7 @@ pub(crate) unsafe fn _mm256_pack_ps_x2_epi16(store: (__m256, __m256)) -> __m256i #[inline] #[target_feature(enable = "avx2")] -pub(crate) unsafe fn _mm256_pack_epi32_x4_epi8(store: (__m256i, __m256i)) -> __m256i { +pub(crate) fn _mm256_pack_epi32_x4_epi8(store: (__m256i, __m256i)) -> __m256i { let rnd = _mm256_set1_epi16((1 << 5) - 1); _mm256_packus_epi16( _mm256_srai_epi16::<6>(_mm256_add_epi16(store.0, rnd)), @@ -299,7 +312,7 @@ pub(crate) unsafe fn _mm256_pack_epi32_x4_epi8(store: (__m256i, __m256i)) -> __m #[inline] #[target_feature(enable = "avx2")] -pub(crate) unsafe fn _mm256_pack_epi32_x4_epu16(store: (__m256i, __m256i)) -> __m256i { +pub(crate) fn _mm256_pack_epi32_x4_epu16(store: (__m256i, __m256i)) -> __m256i { let rnd = _mm256_set1_epi32((1 << 14) - 1); _mm256_packus_epi32( _mm256_srli_epi32::<15>(_mm256_add_epi32(store.0, rnd)), diff --git a/src/filter1d/avx512/filter_column_symm_approx.rs b/src/filter1d/avx512/filter_column_symm_approx.rs index 25ca0fa4..95b72281 100644 --- a/src/filter1d/avx512/filter_column_symm_approx.rs +++ b/src/filter1d/avx512/filter_column_symm_approx.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::avx512::filter_column_symm_approx_uq0_7::filter_column_avx512_symm_u8_uq0_7; use crate::filter1d::avx512::sse_utils::*; use crate::filter1d::avx512::utils::{ _mm256_mul_add_symm_epi8_by_epi16_x4, _mm256_mul_epi8_by_epi16_x4, _mm256_pack_epi32_x4_epi8, @@ -35,7 +34,6 @@ use crate::filter1d::avx512::utils::{ }; use crate::filter1d::avx512::v_load::_mm512_load_pack_x2; use crate::filter1d::avx512::v_store::_mm512_store_pack_x2; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; @@ -48,29 +46,16 @@ pub(crate) fn filter_column_avx512_symm_u8_i32_app( dst: &mut [u8], image_size: ImageSize, filter_region: FilterRegion, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { - if scanned_kernel.len() <= 7 { - let is_all_positive = scanned_kernel.iter().all(|&x| x.weight > 0); - if is_all_positive { - return filter_column_avx512_symm_u8_uq0_7( - arena, - arena_src, - dst, - image_size, - filter_region, - scanned_kernel, - ); - } - } filter_column_avx512_symm_u8_i32_impl( arena, arena_src, dst, image_size, filter_region, - scanned_kernel, + kernel, ); } } @@ -82,29 +67,21 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; - let length = scanned_kernel.len(); + let length = kernel.len(); let half_len = length / 2; let ref0 = arena_src.get_unchecked(half_len); - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.weight.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) - }) - .collect::>(); - let mut cx = 0usize; - let coeff = _mm512_set1_epi32(*v_prepared.get_unchecked(half_len)); + let coeff = _mm512_set1_epi32(*kernel.get_unchecked(half_len)); - while cx + 128 < image_width { + while cx + 128 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm512_load_pack_x2(v_src.as_ptr()); @@ -113,7 +90,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm512_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm512_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm512_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = _mm512_load_pack_x2( @@ -134,7 +111,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( cx += 128; } - while cx + 64 < image_width { + while cx + 64 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm512_loadu_si512(v_src.as_ptr() as *const _); @@ -142,7 +119,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm512_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm512_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm512_loadu_si512( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -160,7 +137,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( cx += 64; } - while cx + 32 < image_width { + while cx + 32 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); @@ -168,7 +145,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm256_loadu_si256( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, ); @@ -186,7 +163,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( cx += 32; } - while cx + 16 < image_width { + while cx + 16 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); @@ -194,7 +171,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si128( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -212,7 +189,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( cx += 16; } - while cx + 8 < image_width { + while cx + 8 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm_loadu_si64(v_src.as_ptr() as *const _); @@ -220,7 +197,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si64( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -238,7 +215,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( cx += 8; } - while cx + 4 < image_width { + while cx + 4 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm_loadu_si32(v_src.as_ptr() as *const _); @@ -246,7 +223,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si32( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -264,19 +241,23 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( cx += 4; } - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); for x in cx..image_width { let v_src = ref0.get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(wb as i32); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(wb as i32) .add(k0); } diff --git a/src/filter1d/avx512/filter_column_symm_approx_uq0_7.rs b/src/filter1d/avx512/filter_column_symm_approx_uq0_7.rs index e5ce4297..1638c7d2 100644 --- a/src/filter1d/avx512/filter_column_symm_approx_uq0_7.rs +++ b/src/filter1d/avx512/filter_column_symm_approx_uq0_7.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::img_size::ImageSize; use std::arch::x86_64::*; @@ -39,34 +38,16 @@ pub(crate) fn filter_column_avx512_symm_u8_uq0_7( dst: &mut [u8], image_size: ImageSize, filter_region: FilterRegion, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8).min(i8::MAX as i32) as i8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } filter_column_avx512_symm_u8_i32_impl( arena, arena_src, dst, image_size, filter_region, - &shifted, + kernel, ); } } @@ -78,31 +59,23 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[i8], + kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; - let length = scanned_kernel.len(); + let length = kernel.len(); let half_len = length / 2; let ref0 = arena_src.get_unchecked(half_len); - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[0], z[0], z[0]]) - }) - .collect::>(); - let mut cx = 0usize; - let coeff = _mm512_set1_epi32(*v_prepared.get_unchecked(half_len)); + let coeff = _mm512_set1_epi32(*kernel.get_unchecked(half_len)); let rnd = _mm512_set1_epi16(1 << 6); - while cx + 64 < image_width { + while cx + 64 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm512_loadu_si512(v_src.as_ptr() as *const _); @@ -117,7 +90,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm512_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm512_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm512_loadu_si512( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -148,7 +121,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( let rnd = _mm512_castsi512_si256(rnd); let coeff = _mm512_castsi512_si256(coeff); - while cx + 32 < image_width { + while cx + 32 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm256_loadu_si256(v_src.as_ptr() as *const __m256i); @@ -163,7 +136,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm256_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm256_loadu_si256( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m256i, ); @@ -191,7 +164,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( cx += 32; } - while cx + 16 < image_width { + while cx + 16 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); @@ -212,7 +185,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si128( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -240,7 +213,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( cx += 16; } - while cx + 8 < image_width { + while cx + 8 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = _mm_loadu_si64(v_src.as_ptr() as *const _); @@ -254,7 +227,7 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si64( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -280,19 +253,23 @@ unsafe fn filter_column_avx512_symm_u8_i32_impl( cx += 8; } - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); + let cb = coeff.to_ne_bytes(); + let wb = i8::from_ne_bytes([cb[0]]); for x in cx..image_width { let v_src = ref0.get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as u16).mul(coeff as u16); + let mut k0 = ((*v_src.get_unchecked(0)) as u16).mul(wb as u16); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; + let cb = coeff.to_ne_bytes(); + let wb = i8::from_ne_bytes([cb[0]]); k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as u16) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as u16) - .mul(coeff as u16) + .mul(wb as u16) .add(k0); } diff --git a/src/filter1d/avx512/filter_row_symm_uq15_u16.rs b/src/filter1d/avx512/filter_row_symm_uq15_u16.rs index 287d9e82..4fbe49c1 100644 --- a/src/filter1d/avx512/filter_row_symm_uq15_u16.rs +++ b/src/filter1d/avx512/filter_row_symm_uq15_u16.rs @@ -60,7 +60,7 @@ pub(crate) fn filter_row_avx512_symm_uq15_u16( } #[target_feature(enable = "avx512bw")] -unsafe fn filter_row_avx512_symm_uq15_u16_impl( +fn filter_row_avx512_symm_uq15_u16_impl( arena: Arena, arena_src: &[u16], dst: &mut [u16], @@ -68,221 +68,226 @@ unsafe fn filter_row_avx512_symm_uq15_u16_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let width = image_size.width; + unsafe { + let width = image_size.width; - let src = arena_src; + let src = arena_src; - let length = scanned_kernel.len(); - let half_len = length / 2; + let length = scanned_kernel.len(); + let half_len = length / 2; - let local_src = src; + let local_src = src; - let mut cx = 0usize; + let mut cx = 0usize; - let max_width = width * arena.components; + let max_width = width * arena.components; - let coeff = _mm512_set1_epi32(scanned_kernel.get_unchecked(half_len).weight as i32); - let rnd = _mm512_set1_epi32((1 << 14) - 1); + let coeff = _mm512_set1_epi32(scanned_kernel.get_unchecked(half_len).weight as i32); + let rnd = _mm512_set1_epi32((1 << 14) - 1); - while cx + 64 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + while cx + 64 < max_width { + let shifted_src = local_src.get_unchecked(cx..); - let source = - _mm512_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm512_mul_epu16_widen(source.0, coeff); - let mut k1 = _mm512_mul_epu16_widen(source.1, coeff); + let source = + _mm512_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm512_mul_epu16_widen(source.0, coeff); + let mut k1 = _mm512_mul_epu16_widen(source.1, coeff); - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm512_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = - _mm512_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm512_load_pack_x2( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm512_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = + _mm512_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm512_load_pack_x2( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm512_mul_add_symm_epu16_by_epu16_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm512_mul_add_symm_epu16_by_epu16_x4(k1, v_source0.1, v_source1.1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm512_store_pack_x2( + dst_ptr0 as *mut _, + ( + _mm512_pack_epi32_x4_epu16(k0), + _mm512_pack_epi32_x4_epu16(k1), + ), ); - k0 = _mm512_mul_add_symm_epu16_by_epu16_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm512_mul_add_symm_epu16_by_epu16_x4(k1, v_source0.1, v_source1.1, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm512_store_pack_x2( - dst_ptr0 as *mut _, - ( - _mm512_pack_epi32_x4_epu16(k0), - _mm512_pack_epi32_x4_epu16(k1), - ), - ); - cx += 64; - } + while cx + 32 < max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = + _mm512_loadu_si512(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm512_mul_epu16_widen(source, coeff); - let source = - _mm512_loadu_si512(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm512_mul_epu16_widen(source, coeff); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm512_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = + _mm512_loadu_si512(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm512_loadu_si512( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm512_mul_add_symm_epu16_by_epu16_x4(k0, v_source0, v_source1, coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm512_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = - _mm512_loadu_si512(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm512_loadu_si512( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm512_mul_add_symm_epu16_by_epu16_x4(k0, v_source0, v_source1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm512_storeu_si512(dst_ptr0 as *mut _, _mm512_pack_epi32_x4_epu16(k0)); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm512_storeu_si512(dst_ptr0 as *mut _, _mm512_pack_epi32_x4_epu16(k0)); - cx += 32; - } - - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + while cx + 16 < max_width { + let shifted_src = local_src.get_unchecked(cx..); - let sh_mask = _mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7); + let sh_mask = _mm512_setr_epi64(0, 2, 4, 6, 1, 3, 5, 7); - let source = - _mm256_loadu_si256(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm512_mullo_epi32( - _mm512_permutexvar_epi64(sh_mask, _mm512_castsi256_si512(source)), - coeff, - ); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm512_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = _mm512_permutexvar_epi64( - sh_mask, - _mm512_castsi256_si512(_mm256_loadu_si256( - shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, - )), + let source = + _mm256_loadu_si256(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm512_mullo_epi32( + _mm512_permutexvar_epi64(sh_mask, _mm512_castsi256_si512(source)), + coeff, ); - let v_source1 = _mm512_permutexvar_epi64( - sh_mask, - _mm512_castsi256_si512(_mm256_loadu_si256( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - )), - ); - - let uexp0 = _mm512_unpacklo_epi8(v_source0, _mm512_setzero_si512()); - let uexp1 = _mm512_unpacklo_epi8(v_source1, _mm512_setzero_si512()); - k0 = _mm512_add_epi32( - k0, - _mm512_mullo_epi32(_mm512_add_epi32(uexp0, uexp1), coeff), + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm512_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = _mm512_permutexvar_epi64( + sh_mask, + _mm512_castsi256_si512(_mm256_loadu_si256( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + )), + ); + let v_source1 = _mm512_permutexvar_epi64( + sh_mask, + _mm512_castsi256_si512(_mm256_loadu_si256( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + )), + ); + + let uexp0 = _mm512_unpacklo_epi8(v_source0, _mm512_setzero_si512()); + let uexp1 = _mm512_unpacklo_epi8(v_source1, _mm512_setzero_si512()); + + k0 = _mm512_add_epi32( + k0, + _mm512_mullo_epi32(_mm512_add_epi32(uexp0, uexp1), coeff), + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256( + dst_ptr0 as *mut _, + _mm512_castsi512_si256(_mm512_packus_epi32( + _mm512_srli_epi32::<15>(_mm512_add_epi32(k0, rnd)), + _mm512_setzero_si512(), + )), ); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256( - dst_ptr0 as *mut _, - _mm512_castsi512_si256(_mm512_packus_epi32( - _mm512_srli_epi32::<15>(_mm512_add_epi32(k0, rnd)), - _mm512_setzero_si512(), - )), - ); - cx += 16; - } - - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + while cx + 8 < max_width { + let shifted_src = local_src.get_unchecked(cx..); - const M: i32 = v512_shuffle(3, 1, 2, 0); + const M: i32 = v512_shuffle(3, 1, 2, 0); - let source = _mm256_castsi128_si256(_mm_loadu_si128( - shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, - )); - let mut k0 = _mm256_mullo_epi32( - _mm256_unpacklo_epi8( - _mm256_permute4x64_epi64::(source), - _mm256_setzero_si256(), - ), - _mm512_castsi512_si256(coeff), - ); + let source = _mm256_castsi128_si256(_mm_loadu_si128( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, + )); + let mut k0 = _mm256_mullo_epi32( + _mm256_unpacklo_epi8( + _mm256_permute4x64_epi64::(source), + _mm256_setzero_si256(), + ), + _mm512_castsi512_si256(coeff), + ); - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = _mm256_permute4x64_epi64::(_mm256_castsi128_si256(_mm_loadu_si128( - shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, - ))); - let v_source1 = _mm256_permute4x64_epi64::(_mm256_castsi128_si256(_mm_loadu_si128( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ))); - k0 = _mm256_add_epi32( - k0, - _mm256_mullo_epi32(_mm256_add_epi32(v_source0, v_source1), coeff), + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = _mm256_permute4x64_epi64::(_mm256_castsi128_si256( + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _), + )); + let v_source1 = + _mm256_permute4x64_epi64::(_mm256_castsi128_si256(_mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ))); + k0 = _mm256_add_epi32( + k0, + _mm256_mullo_epi32(_mm256_add_epi32(v_source0, v_source1), coeff), + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128( + dst_ptr0 as *mut _, + _mm256_castsi256_si128(_mm256_packus_epi32( + _mm256_srli_epi32::<15>(_mm256_add_epi32(k0, _mm512_castsi512_si256(rnd))), + _mm256_setzero_si256(), + )), ); + cx += 8; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128( - dst_ptr0 as *mut _, - _mm256_castsi256_si128(_mm256_packus_epi32( - _mm256_srli_epi32::<15>(_mm256_add_epi32(k0, _mm512_castsi512_si256(rnd))), - _mm256_setzero_si256(), - )), - ); - cx += 8; - } - - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + while cx + 4 < max_width { + let shifted_src = local_src.get_unchecked(cx..); - let source = _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mullo_epi32( - _mm_unpacklo_epi16(source, _mm_setzero_si128()), - _mm512_castsi512_si128(coeff), - ); + let source = + _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mullo_epi32( + _mm_unpacklo_epi16(source, _mm_setzero_si128()), + _mm512_castsi512_si128(coeff), + ); - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si64(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_add_epi32( - k0, - _mm_mullo_epi32( - _mm_add_epi32( - _mm_unpacklo_epi16(v_source0, _mm_setzero_si128()), - _mm_unpacklo_epi16(v_source1, _mm_setzero_si128()), + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_add_epi32( + k0, + _mm_mullo_epi32( + _mm_add_epi32( + _mm_unpacklo_epi16(v_source0, _mm_setzero_si128()), + _mm_unpacklo_epi16(v_source1, _mm_setzero_si128()), + ), + coeff, ), - coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + + _mm_storeu_si64( + dst_ptr0 as *mut _, + _mm_packus_epi32( + _mm_srli_epi32::<15>(_mm_add_epi32(k0, _mm512_castsi512_si128(rnd))), + _mm_setzero_si128(), ), ); + cx += 4; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + let coeff = *scanned_kernel.get_unchecked(half_len); - _mm_storeu_si64( - dst_ptr0 as *mut _, - _mm_packus_epi32( - _mm_srli_epi32::<15>(_mm_add_epi32(k0, _mm512_castsi512_si128(rnd))), - _mm_setzero_si128(), - ), - ); - cx += 4; - } + for x in cx..max_width { + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as u32 * coeff.weight; - let coeff = *scanned_kernel.get_unchecked(half_len); + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; - for x in cx..max_width { - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as u32 * coeff.weight; + k0 += (*shifted_src.get_unchecked(i * N) as u32 + + *shifted_src.get_unchecked(rollback * N) as u32) + * coeff.weight; + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - - k0 += (*shifted_src.get_unchecked(i * N) as u32 - + *shifted_src.get_unchecked(rollback * N) as u32) - * coeff.weight; + *dst.get_unchecked_mut(x) = k0.to_approx_(); } - - *dst.get_unchecked_mut(x) = k0.to_approx_(); } } diff --git a/src/filter1d/avx512/mod.rs b/src/filter1d/avx512/mod.rs index 23bc20f8..6527bb99 100644 --- a/src/filter1d/avx512/mod.rs +++ b/src/filter1d/avx512/mod.rs @@ -39,6 +39,7 @@ mod v_load; mod v_store; pub(crate) use filter_column_symm_approx::filter_column_avx512_symm_u8_i32_app; +pub(crate) use filter_column_symm_approx_uq0_7::filter_column_avx512_symm_u8_uq0_7; pub(crate) use filter_row_symm_uq15_u16::filter_row_avx512_symm_uq15_u16; pub(crate) use row_symm_approx_binter::filter_row_avx512_symm_u8_i32_app_binter; pub(crate) use row_symm_approx_binter3_uq0_7::filter_row_avx512_symm_u8_uq0_7_k3; diff --git a/src/filter1d/avx512/pack.rs b/src/filter1d/avx512/pack.rs index 0934aa91..a9f84b76 100644 --- a/src/filter1d/avx512/pack.rs +++ b/src/filter1d/avx512/pack.rs @@ -31,13 +31,14 @@ use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgba_epi8( +pub(crate) fn _mm256_deinterleave_rgba_epi8( rgba0: __m256i, rgba1: __m256i, rgba2: __m256i, rgba3: __m256i, ) -> (__m256i, __m256i, __m256i, __m256i) { - #[rustfmt::skip] + unsafe { + #[rustfmt::skip] let sh = _mm256_setr_epi8( 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, @@ -47,103 +48,109 @@ pub(crate) unsafe fn _mm256_deinterleave_rgba_epi8( 11, 15, ); - let p0 = _mm256_shuffle_epi8(rgba0, sh); - let p1 = _mm256_shuffle_epi8(rgba1, sh); - let p2 = _mm256_shuffle_epi8(rgba2, sh); - let p3 = _mm256_shuffle_epi8(rgba3, sh); + let p0 = _mm256_shuffle_epi8(rgba0, sh); + let p1 = _mm256_shuffle_epi8(rgba1, sh); + let p2 = _mm256_shuffle_epi8(rgba2, sh); + let p3 = _mm256_shuffle_epi8(rgba3, sh); - let p01l = _mm256_unpacklo_epi32(p0, p1); - let p01h = _mm256_unpackhi_epi32(p0, p1); - let p23l = _mm256_unpacklo_epi32(p2, p3); - let p23h = _mm256_unpackhi_epi32(p2, p3); + let p01l = _mm256_unpacklo_epi32(p0, p1); + let p01h = _mm256_unpackhi_epi32(p0, p1); + let p23l = _mm256_unpacklo_epi32(p2, p3); + let p23h = _mm256_unpackhi_epi32(p2, p3); - let pll = _mm256_permute2x128_si256::<32>(p01l, p23l); - let plh = _mm256_permute2x128_si256::<49>(p01l, p23l); - let phl = _mm256_permute2x128_si256::<32>(p01h, p23h); - let phh = _mm256_permute2x128_si256::<49>(p01h, p23h); + let pll = _mm256_permute2x128_si256::<32>(p01l, p23l); + let plh = _mm256_permute2x128_si256::<49>(p01l, p23l); + let phl = _mm256_permute2x128_si256::<32>(p01h, p23h); + let phh = _mm256_permute2x128_si256::<49>(p01h, p23h); - let b0 = _mm256_unpacklo_epi32(pll, plh); - let g0 = _mm256_unpackhi_epi32(pll, plh); - let r0 = _mm256_unpacklo_epi32(phl, phh); - let a0 = _mm256_unpackhi_epi32(phl, phh); + let b0 = _mm256_unpacklo_epi32(pll, plh); + let g0 = _mm256_unpackhi_epi32(pll, plh); + let r0 = _mm256_unpacklo_epi32(phl, phh); + let a0 = _mm256_unpackhi_epi32(phl, phh); - (b0, g0, r0, a0) + (b0, g0, r0, a0) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgba_epi8( +pub(crate) fn _mm256_interleave_rgba_epi8( vals: (__m256i, __m256i, __m256i, __m256i), ) -> (__m256i, __m256i, __m256i, __m256i) { - let bg0 = _mm256_unpacklo_epi8(vals.0, vals.1); - let bg1 = _mm256_unpackhi_epi8(vals.0, vals.1); - let ra0 = _mm256_unpacklo_epi8(vals.2, vals.3); - let ra1 = _mm256_unpackhi_epi8(vals.2, vals.3); - - let rgba0_ = _mm256_unpacklo_epi16(bg0, ra0); - let rgba1_ = _mm256_unpackhi_epi16(bg0, ra0); - let rgba2_ = _mm256_unpacklo_epi16(bg1, ra1); - let rgba3_ = _mm256_unpackhi_epi16(bg1, ra1); - - let rgba0 = _mm256_permute2x128_si256::<32>(rgba0_, rgba1_); - let rgba2 = _mm256_permute2x128_si256::<49>(rgba0_, rgba1_); - let rgba1 = _mm256_permute2x128_si256::<32>(rgba2_, rgba3_); - let rgba3 = _mm256_permute2x128_si256::<49>(rgba2_, rgba3_); - (rgba0, rgba1, rgba2, rgba3) + unsafe { + let bg0 = _mm256_unpacklo_epi8(vals.0, vals.1); + let bg1 = _mm256_unpackhi_epi8(vals.0, vals.1); + let ra0 = _mm256_unpacklo_epi8(vals.2, vals.3); + let ra1 = _mm256_unpackhi_epi8(vals.2, vals.3); + + let rgba0_ = _mm256_unpacklo_epi16(bg0, ra0); + let rgba1_ = _mm256_unpackhi_epi16(bg0, ra0); + let rgba2_ = _mm256_unpacklo_epi16(bg1, ra1); + let rgba3_ = _mm256_unpackhi_epi16(bg1, ra1); + + let rgba0 = _mm256_permute2x128_si256::<32>(rgba0_, rgba1_); + let rgba2 = _mm256_permute2x128_si256::<49>(rgba0_, rgba1_); + let rgba1 = _mm256_permute2x128_si256::<32>(rgba2_, rgba3_); + let rgba3 = _mm256_permute2x128_si256::<49>(rgba2_, rgba3_); + (rgba0, rgba1, rgba2, rgba3) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgb( +pub(crate) fn _mm256_interleave_rgb( r: __m256i, g: __m256i, b: __m256i, ) -> (__m256i, __m256i, __m256i) { - let sh_b = _mm256_setr_epi8( - 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, - 9, 4, 15, 10, 5, - ); - let sh_g = _mm256_setr_epi8( - 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, - 14, 9, 4, 15, 10, - ); - let sh_r = _mm256_setr_epi8( - 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, - 3, 14, 9, 4, 15, - ); - - let b0 = _mm256_shuffle_epi8(r, sh_b); - let g0 = _mm256_shuffle_epi8(g, sh_g); - let r0 = _mm256_shuffle_epi8(b, sh_r); - - let m0 = _mm256_setr_epi8( - 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, - 0, 0, -1, 0, 0, - ); - let m1 = _mm256_setr_epi8( - 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, - -1, 0, 0, -1, 0, - ); - - let p0 = _mm256_blendv_epi8(_mm256_blendv_epi8(b0, g0, m0), r0, m1); - let p1 = _mm256_blendv_epi8(_mm256_blendv_epi8(g0, r0, m0), b0, m1); - let p2 = _mm256_blendv_epi8(_mm256_blendv_epi8(r0, b0, m0), g0, m1); - - let bgr0 = _mm256_permute2x128_si256::<32>(p0, p1); - let bgr1 = _mm256_permute2x128_si256::<48>(p2, p0); - let bgr2 = _mm256_permute2x128_si256::<49>(p1, p2); - - (bgr0, bgr1, bgr2) + unsafe { + let sh_b = _mm256_setr_epi8( + 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, + 14, 9, 4, 15, 10, 5, + ); + let sh_g = _mm256_setr_epi8( + 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, + 3, 14, 9, 4, 15, 10, + ); + let sh_r = _mm256_setr_epi8( + 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5, 0, 11, 6, 1, 12, 7, 2, 13, + 8, 3, 14, 9, 4, 15, + ); + + let b0 = _mm256_shuffle_epi8(r, sh_b); + let g0 = _mm256_shuffle_epi8(g, sh_g); + let r0 = _mm256_shuffle_epi8(b, sh_r); + + let m0 = _mm256_setr_epi8( + 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, + -1, 0, 0, -1, 0, 0, + ); + let m1 = _mm256_setr_epi8( + 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, + 0, -1, 0, 0, -1, 0, + ); + + let p0 = _mm256_blendv_epi8(_mm256_blendv_epi8(b0, g0, m0), r0, m1); + let p1 = _mm256_blendv_epi8(_mm256_blendv_epi8(g0, r0, m0), b0, m1); + let p2 = _mm256_blendv_epi8(_mm256_blendv_epi8(r0, b0, m0), g0, m1); + + let bgr0 = _mm256_permute2x128_si256::<32>(p0, p1); + let bgr1 = _mm256_permute2x128_si256::<48>(p2, p0); + let bgr2 = _mm256_permute2x128_si256::<49>(p1, p2); + + (bgr0, bgr1, bgr2) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgb( +pub(crate) fn _mm256_deinterleave_rgb( rgb0: __m256i, rgb1: __m256i, rgb2: __m256i, ) -> (__m256i, __m256i, __m256i) { - let s02_low = _mm256_permute2x128_si256::<32>(rgb0, rgb2); - let s02_high = _mm256_permute2x128_si256::<49>(rgb0, rgb2); + unsafe { + let s02_low = _mm256_permute2x128_si256::<32>(rgb0, rgb2); + let s02_high = _mm256_permute2x128_si256::<49>(rgb0, rgb2); - #[rustfmt::skip] + #[rustfmt::skip] let m0 = _mm256_setr_epi8( 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, @@ -154,7 +161,7 @@ pub(crate) unsafe fn _mm256_deinterleave_rgb( 0, 0, ); - #[rustfmt::skip] + #[rustfmt::skip] let m1 = _mm256_setr_epi8( 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, @@ -165,11 +172,11 @@ pub(crate) unsafe fn _mm256_deinterleave_rgb( 0, -1, ); - let b0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_low, s02_high, m0), rgb1, m1); - let g0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_high, s02_low, m1), rgb1, m0); - let r0 = _mm256_blendv_epi8(_mm256_blendv_epi8(rgb1, s02_low, m0), s02_high, m1); + let b0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_low, s02_high, m0), rgb1, m1); + let g0 = _mm256_blendv_epi8(_mm256_blendv_epi8(s02_high, s02_low, m1), rgb1, m0); + let r0 = _mm256_blendv_epi8(_mm256_blendv_epi8(rgb1, s02_low, m0), s02_high, m1); - #[rustfmt::skip] + #[rustfmt::skip] let sh_b = _mm256_setr_epi8( 0, 3, 6, 9, 12, 15, 2, 5, 8, 11, @@ -180,7 +187,7 @@ pub(crate) unsafe fn _mm256_deinterleave_rgb( 10, 13, ); - #[rustfmt::skip] + #[rustfmt::skip] let sh_g = _mm256_setr_epi8( 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, @@ -191,7 +198,7 @@ pub(crate) unsafe fn _mm256_deinterleave_rgb( 11, 14, ); - #[rustfmt::skip] + #[rustfmt::skip] let sh_r = _mm256_setr_epi8( 2, 5, 8, 11, 14, 1, 4, 7, 10, 13, @@ -201,154 +208,169 @@ pub(crate) unsafe fn _mm256_deinterleave_rgb( 13, 0, 3, 6, 9, 12, 15, ); - let b0 = _mm256_shuffle_epi8(b0, sh_b); - let g0 = _mm256_shuffle_epi8(g0, sh_g); - let r0 = _mm256_shuffle_epi8(r0, sh_r); - (b0, g0, r0) + let b0 = _mm256_shuffle_epi8(b0, sh_b); + let g0 = _mm256_shuffle_epi8(g0, sh_g); + let r0 = _mm256_shuffle_epi8(r0, sh_r); + (b0, g0, r0) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgba_epi32( +pub(crate) fn _mm256_deinterleave_rgba_epi32( vals: (__m256i, __m256i, __m256i, __m256i), ) -> (__m256i, __m256i, __m256i, __m256i) { - let p01l = _mm256_unpacklo_epi32(vals.0, vals.1); - let p01h = _mm256_unpackhi_epi32(vals.0, vals.1); - let p23l = _mm256_unpacklo_epi32(vals.2, vals.3); - let p23h = _mm256_unpackhi_epi32(vals.2, vals.3); - - let pll = _mm256_permute2x128_si256::<32>(p01l, p23l); - let plh = _mm256_permute2x128_si256::<49>(p01l, p23l); - let phl = _mm256_permute2x128_si256::<32>(p01h, p23h); - let phh = _mm256_permute2x128_si256::<49>(p01h, p23h); - - let v0 = _mm256_unpacklo_epi32(pll, plh); - let v1 = _mm256_unpackhi_epi32(pll, plh); - let v2 = _mm256_unpacklo_epi32(phl, phh); - let v3 = _mm256_unpackhi_epi32(phl, phh); - (v0, v1, v2, v3) + unsafe { + let p01l = _mm256_unpacklo_epi32(vals.0, vals.1); + let p01h = _mm256_unpackhi_epi32(vals.0, vals.1); + let p23l = _mm256_unpacklo_epi32(vals.2, vals.3); + let p23h = _mm256_unpackhi_epi32(vals.2, vals.3); + + let pll = _mm256_permute2x128_si256::<32>(p01l, p23l); + let plh = _mm256_permute2x128_si256::<49>(p01l, p23l); + let phl = _mm256_permute2x128_si256::<32>(p01h, p23h); + let phh = _mm256_permute2x128_si256::<49>(p01h, p23h); + + let v0 = _mm256_unpacklo_epi32(pll, plh); + let v1 = _mm256_unpackhi_epi32(pll, plh); + let v2 = _mm256_unpacklo_epi32(phl, phh); + let v3 = _mm256_unpackhi_epi32(phl, phh); + (v0, v1, v2, v3) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgb_epi32( +pub(crate) fn _mm256_deinterleave_rgb_epi32( vals: (__m256i, __m256i, __m256i), ) -> (__m256i, __m256i, __m256i) { - let s02_low = _mm256_permute2x128_si256::<32>(vals.0, vals.2); - let s02_high = _mm256_permute2x128_si256::<49>(vals.0, vals.2); - - let b0 = _mm256_blend_epi32::<0x92>(_mm256_blend_epi32::<0x24>(s02_low, s02_high), vals.1); - let g0 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(s02_high, s02_low), vals.1); - let r0 = _mm256_blend_epi32::<0x92>(_mm256_blend_epi32::<0x24>(vals.1, s02_low), s02_high); - - let v0 = _mm256_shuffle_epi32::<0x6c>(b0); - let v1 = _mm256_shuffle_epi32::<0xb1>(g0); - let v2 = _mm256_shuffle_epi32::<0xc6>(r0); - (v0, v1, v2) + unsafe { + let s02_low = _mm256_permute2x128_si256::<32>(vals.0, vals.2); + let s02_high = _mm256_permute2x128_si256::<49>(vals.0, vals.2); + + let b0 = _mm256_blend_epi32::<0x92>(_mm256_blend_epi32::<0x24>(s02_low, s02_high), vals.1); + let g0 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(s02_high, s02_low), vals.1); + let r0 = _mm256_blend_epi32::<0x92>(_mm256_blend_epi32::<0x24>(vals.1, s02_low), s02_high); + + let v0 = _mm256_shuffle_epi32::<0x6c>(b0); + let v1 = _mm256_shuffle_epi32::<0xb1>(g0); + let v2 = _mm256_shuffle_epi32::<0xc6>(r0); + (v0, v1, v2) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgba_ps( +pub(crate) fn _mm256_deinterleave_rgba_ps( vals: (__m256, __m256, __m256, __m256), ) -> (__m256, __m256, __m256, __m256) { - let (v0, v1, v2, v3) = _mm256_deinterleave_rgba_epi32(( - _mm256_castps_si256(vals.0), - _mm256_castps_si256(vals.1), - _mm256_castps_si256(vals.2), - _mm256_castps_si256(vals.3), - )); - ( - _mm256_castsi256_ps(v0), - _mm256_castsi256_ps(v1), - _mm256_castsi256_ps(v2), - _mm256_castsi256_ps(v3), - ) + unsafe { + let (v0, v1, v2, v3) = _mm256_deinterleave_rgba_epi32(( + _mm256_castps_si256(vals.0), + _mm256_castps_si256(vals.1), + _mm256_castps_si256(vals.2), + _mm256_castps_si256(vals.3), + )); + ( + _mm256_castsi256_ps(v0), + _mm256_castsi256_ps(v1), + _mm256_castsi256_ps(v2), + _mm256_castsi256_ps(v3), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_deinterleave_rgb_ps( +pub(crate) fn _mm256_deinterleave_rgb_ps( vals: (__m256, __m256, __m256), ) -> (__m256, __m256, __m256) { - let (v0, v1, v2) = _mm256_deinterleave_rgb_epi32(( - _mm256_castps_si256(vals.0), - _mm256_castps_si256(vals.1), - _mm256_castps_si256(vals.2), - )); - ( - _mm256_castsi256_ps(v0), - _mm256_castsi256_ps(v1), - _mm256_castsi256_ps(v2), - ) + unsafe { + let (v0, v1, v2) = _mm256_deinterleave_rgb_epi32(( + _mm256_castps_si256(vals.0), + _mm256_castps_si256(vals.1), + _mm256_castps_si256(vals.2), + )); + ( + _mm256_castsi256_ps(v0), + _mm256_castsi256_ps(v1), + _mm256_castsi256_ps(v2), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgb_epi32( +pub(crate) fn _mm256_interleave_rgb_epi32( vals: (__m256i, __m256i, __m256i), ) -> (__m256i, __m256i, __m256i) { - let b0 = _mm256_shuffle_epi32::<0x6c>(vals.0); - let g0 = _mm256_shuffle_epi32::<0xb1>(vals.1); - let r0 = _mm256_shuffle_epi32::<0xc6>(vals.2); - - let p0 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(b0, g0), r0); - let p1 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(g0, r0), b0); - let p2 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(r0, b0), g0); - - let v0 = _mm256_permute2x128_si256::<32>(p0, p1); - let v1 = p2; - let v2 = _mm256_permute2x128_si256::<49>(p0, p1); - (v0, v1, v2) + unsafe { + let b0 = _mm256_shuffle_epi32::<0x6c>(vals.0); + let g0 = _mm256_shuffle_epi32::<0xb1>(vals.1); + let r0 = _mm256_shuffle_epi32::<0xc6>(vals.2); + + let p0 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(b0, g0), r0); + let p1 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(g0, r0), b0); + let p2 = _mm256_blend_epi32::<0x24>(_mm256_blend_epi32::<0x92>(r0, b0), g0); + + let v0 = _mm256_permute2x128_si256::<32>(p0, p1); + let v1 = p2; + let v2 = _mm256_permute2x128_si256::<49>(p0, p1); + (v0, v1, v2) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgb_ps( - vals: (__m256, __m256, __m256), -) -> (__m256, __m256, __m256) { - let (v0, v1, v2) = _mm256_interleave_rgb_epi32(( - _mm256_castps_si256(vals.0), - _mm256_castps_si256(vals.1), - _mm256_castps_si256(vals.2), - )); - ( - _mm256_castsi256_ps(v0), - _mm256_castsi256_ps(v1), - _mm256_castsi256_ps(v2), - ) +pub(crate) fn _mm256_interleave_rgb_ps(vals: (__m256, __m256, __m256)) -> (__m256, __m256, __m256) { + unsafe { + let (v0, v1, v2) = _mm256_interleave_rgb_epi32(( + _mm256_castps_si256(vals.0), + _mm256_castps_si256(vals.1), + _mm256_castps_si256(vals.2), + )); + ( + _mm256_castsi256_ps(v0), + _mm256_castsi256_ps(v1), + _mm256_castsi256_ps(v2), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgba_epi32( +pub(crate) fn _mm256_interleave_rgba_epi32( vals: (__m256i, __m256i, __m256i, __m256i), ) -> (__m256i, __m256i, __m256i, __m256i) { - let bg0 = _mm256_unpacklo_epi32(vals.0, vals.1); - let bg1 = _mm256_unpackhi_epi32(vals.0, vals.1); - let ra0 = _mm256_unpacklo_epi32(vals.2, vals.3); - let ra1 = _mm256_unpackhi_epi32(vals.2, vals.3); - - let bgra0_ = _mm256_unpacklo_epi64(bg0, ra0); - let bgra1_ = _mm256_unpackhi_epi64(bg0, ra0); - let bgra2_ = _mm256_unpacklo_epi64(bg1, ra1); - let bgra3_ = _mm256_unpackhi_epi64(bg1, ra1); - - let v0 = _mm256_permute2x128_si256::<32>(bgra0_, bgra1_); - let v1 = _mm256_permute2x128_si256::<32>(bgra2_, bgra3_); - let v2 = _mm256_permute2x128_si256::<49>(bgra0_, bgra1_); - let v3 = _mm256_permute2x128_si256::<49>(bgra2_, bgra3_); - - (v0, v1, v2, v3) + unsafe { + let bg0 = _mm256_unpacklo_epi32(vals.0, vals.1); + let bg1 = _mm256_unpackhi_epi32(vals.0, vals.1); + let ra0 = _mm256_unpacklo_epi32(vals.2, vals.3); + let ra1 = _mm256_unpackhi_epi32(vals.2, vals.3); + + let bgra0_ = _mm256_unpacklo_epi64(bg0, ra0); + let bgra1_ = _mm256_unpackhi_epi64(bg0, ra0); + let bgra2_ = _mm256_unpacklo_epi64(bg1, ra1); + let bgra3_ = _mm256_unpackhi_epi64(bg1, ra1); + + let v0 = _mm256_permute2x128_si256::<32>(bgra0_, bgra1_); + let v1 = _mm256_permute2x128_si256::<32>(bgra2_, bgra3_); + let v2 = _mm256_permute2x128_si256::<49>(bgra0_, bgra1_); + let v3 = _mm256_permute2x128_si256::<49>(bgra2_, bgra3_); + + (v0, v1, v2, v3) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_interleave_rgba_ps( +pub(crate) fn _mm256_interleave_rgba_ps( vals: (__m256, __m256, __m256, __m256), ) -> (__m256, __m256, __m256, __m256) { - let (v0, v1, v2, v3) = _mm256_interleave_rgba_epi32(( - _mm256_castps_si256(vals.0), - _mm256_castps_si256(vals.1), - _mm256_castps_si256(vals.2), - _mm256_castps_si256(vals.3), - )); - ( - _mm256_castsi256_ps(v0), - _mm256_castsi256_ps(v1), - _mm256_castsi256_ps(v2), - _mm256_castsi256_ps(v3), - ) + unsafe { + let (v0, v1, v2, v3) = _mm256_interleave_rgba_epi32(( + _mm256_castps_si256(vals.0), + _mm256_castps_si256(vals.1), + _mm256_castps_si256(vals.2), + _mm256_castps_si256(vals.3), + )); + ( + _mm256_castsi256_ps(v0), + _mm256_castsi256_ps(v1), + _mm256_castsi256_ps(v2), + _mm256_castsi256_ps(v3), + ) + } } diff --git a/src/filter1d/avx512/row_symm_approx_binter.rs b/src/filter1d/avx512/row_symm_approx_binter.rs index e45e99ea..525b96da 100644 --- a/src/filter1d/avx512/row_symm_approx_binter.rs +++ b/src/filter1d/avx512/row_symm_approx_binter.rs @@ -26,17 +26,16 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::BorderHandle; use crate::edge_mode::border_interpolate; use crate::filter1d::avx512::sse_utils::*; use crate::filter1d::avx512::utils::{ _mm256_mul_add_symm_epi8_by_epi16_x4, _mm256_mul_epi8_by_epi16_x4, _mm256_pack_epi32_x4_epi8, _mm512_mul_add_symm_epi8_by_epi16_x4, _mm512_mul_epi8_by_epi16_x4, _mm512_pack_epi32_x4_epi8, }; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; -use crate::BorderHandle; use std::arch::x86_64::*; pub(crate) fn filter_row_avx512_symm_u8_i32_app_binter( @@ -44,7 +43,7 @@ pub(crate) fn filter_row_avx512_symm_u8_i32_app_binter( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { let unit = ExecutionUnit::::default(); @@ -63,256 +62,266 @@ impl ExecutionUnit { m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { - let width = image_size.width; + unsafe { + let width = image_size.width; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let min_left = half_len.min(width); + let s_kernel = half_len as i64; + + for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { + let dst = &mut **dst; + let mut f_cx = 0usize; + + while f_cx < min_left { + for c in 0..N { + let coeff = *scanned_kernel.get_unchecked(half_len); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); + let mx = f_cx as i64 - s_kernel; + let mut k0: i32 = *src.get_unchecked(f_cx * N + c) as i32 * wb as i32; + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); + let rollback = length - i - 1; + + let src0 = border_interpolate!( + src, + edge_mode, + i as i64 + mx, + 0, + width as i64, + N, + c + ); + let src1 = border_interpolate!( + src, + edge_mode, + rollback as i64 + mx, + 0, + width as i64, + N, + c + ); + + k0 += (src0 as i32 + src1 as i32) * wb as i32; + } + + *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); + } + f_cx += 1; + } - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.weight.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) - }) - .collect::>(); + let mut m_cx = f_cx * N; - let length = scanned_kernel.len(); - let half_len = length / 2; + let s_half = half_len * N; + let m_right = width.saturating_sub(half_len); + let max_width = m_right * N; - let min_left = half_len.min(width); - let s_kernel = half_len as i64; + let coeff = _mm512_set1_epi32(*scanned_kernel.get_unchecked(half_len)); - for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { - let dst = &mut **dst; - let mut f_cx = 0usize; + while m_cx + 64 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - while f_cx < min_left { - for c in 0..N { - let coeff = *scanned_kernel.get_unchecked(half_len); - let mx = f_cx as i64 - s_kernel; - let mut k0: i32 = *src.get_unchecked(f_cx * N + c) as i32 * coeff.weight; + let source = _mm512_loadu_si512( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, + ); + let mut k0 = _mm512_mul_epi8_by_epi16_x4(source, coeff); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; - - let src0 = border_interpolate!( - src, - edge_mode, - i as i64 + mx, - 0, - width as i64, - N, - c + let coeff = _mm512_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm512_loadu_si512( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, ); - let src1 = border_interpolate!( - src, - edge_mode, - rollback as i64 + mx, - 0, - width as i64, - N, - c + let v_source1 = _mm512_loadu_si512( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, ); - - k0 += (src0 as i32 + src1 as i32) * coeff.weight; + k0 = _mm512_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); } - *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm512_storeu_si512(dst_ptr0 as *mut _, _mm512_pack_epi32_x4_epi8(k0)); + m_cx += 64; } - f_cx += 1; - } - - let mut m_cx = f_cx * N; - - let s_half = half_len * N; - let m_right = width.saturating_sub(half_len); - let max_width = m_right * N; - let coeff = _mm512_set1_epi32(*v_prepared.get_unchecked(half_len)); + let coeff = _mm512_castsi512_si256(coeff); - while m_cx + 64 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); + while m_cx + 32 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - let source = _mm512_loadu_si512( - shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ - ); - let mut k0 = _mm512_mul_epi8_by_epi16_x4(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm512_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = _mm512_loadu_si512( - shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, - ); - let v_source1 = _mm512_loadu_si512( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm256_loadu_si256( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, ); - k0 = _mm512_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); - } + let mut k0 = _mm256_mul_epi8_by_epi16_x4(source, coeff); - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm512_storeu_si512(dst_ptr0 as *mut _, _mm512_pack_epi32_x4_epi8(k0)); - m_cx += 64; - } - - let coeff = _mm512_castsi512_si256(coeff); - - while m_cx + 32 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = _mm256_loadu_si256( - shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ - ); - let mut k0 = _mm256_mul_epi8_by_epi16_x4(source, coeff); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm256_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm256_loadu_si256( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm256_loadu_si256( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm256_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = _mm256_loadu_si256( - shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, - ); - let v_source1 = _mm256_loadu_si256( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm256_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_epi32_x4_epi8(k0)); + m_cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_pack_epi32_x4_epi8(k0)); - m_cx += 32; - } + while m_cx + 16 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - while m_cx + 16 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source, _mm256_castsi256_si128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si128( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm_loadu_si128( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, ); - k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); - } + let mut k0 = _mm_mul_epi8_by_epi16_x4(source, _mm256_castsi256_si128(coeff)); - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epi8(k0)); - m_cx += 16; - } + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm_loadu_si128( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + } - while m_cx + 8 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si64( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epi8(k0)); + m_cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); - m_cx += 8; - } + while m_cx + 8 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - while m_cx + 4 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si32(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si32(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si32( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm_loadu_si64( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ ); - k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); - } + let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); - m_cx += 4; - } + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm_loadu_si64( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + } - let coeff = *scanned_kernel.get_unchecked(half_len); + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); + m_cx += 8; + } - for zx in m_cx..max_width { - let x = zx - s_half; + while m_cx + 4 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - let shifted_src = src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; + let source = _mm_loadu_si32( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ + ); + let mut k0 = _mm_mul_epi8_by_epi16_x2(source, _mm256_castsi256_si128(coeff)); - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm_loadu_si32( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si32( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + } - k0 += (*shifted_src.get_unchecked(i * N) as i16 - + *shifted_src.get_unchecked(rollback * N) as i16) - as i32 - * coeff.weight; + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si32(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); + m_cx += 4; } - *dst.get_unchecked_mut(zx) = k0.to_approx_(); - } + let coeff = *scanned_kernel.get_unchecked(half_len); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); - f_cx = m_right; + for zx in m_cx..max_width { + let x = zx - s_half; - while f_cx < width { - for c in 0..N { - let mx = f_cx as i64 - s_kernel; - let mut k0 = *src.get_unchecked(f_cx * N + c) as i32 * coeff.weight; + let shifted_src = src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * wb as i32; for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); let rollback = length - i - 1; - let src0 = border_interpolate!( - src, - edge_mode, - i as i64 + mx, - 0, - width as i64, - N, - c - ); - let src1 = border_interpolate!( - src, - edge_mode, - rollback as i64 + mx, - 0, - width as i64, - N, - c - ); - - k0 += (src0 as i32 + src1 as i32) * coeff.weight; + k0 += (*shifted_src.get_unchecked(i * N) as i16 + + *shifted_src.get_unchecked(rollback * N) as i16) + as i32 + * wb as i32; } - *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); + *dst.get_unchecked_mut(zx) = k0.to_approx_(); + } + + f_cx = m_right; + + while f_cx < width { + for c in 0..N { + let mx = f_cx as i64 - s_kernel; + let mut k0 = *src.get_unchecked(f_cx * N + c) as i32 * wb as i32; + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); + let rollback = length - i - 1; + + let src0 = border_interpolate!( + src, + edge_mode, + i as i64 + mx, + 0, + width as i64, + N, + c + ); + let src1 = border_interpolate!( + src, + edge_mode, + rollback as i64 + mx, + 0, + width as i64, + N, + c + ); + + k0 += (src0 as i32 + src1 as i32) * wb as i32; + } + + *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); + } + f_cx += 1; } - f_cx += 1; } } } diff --git a/src/filter1d/avx512/row_symm_approx_binter3_uq0_7.rs b/src/filter1d/avx512/row_symm_approx_binter3_uq0_7.rs index 23a5dbd8..760169b3 100644 --- a/src/filter1d/avx512/row_symm_approx_binter3_uq0_7.rs +++ b/src/filter1d/avx512/row_symm_approx_binter3_uq0_7.rs @@ -26,11 +26,10 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::BorderHandle; use crate::edge_mode::clamp_edge; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::img_size::ImageSize; -use crate::BorderHandle; use std::arch::x86_64::*; pub(crate) fn filter_row_avx512_symm_u8_uq0_7_k3( @@ -38,29 +37,11 @@ pub(crate) fn filter_row_avx512_symm_u8_uq0_7_k3( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i8], ) { unsafe { - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8).min(i8::MAX as i32) as i8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } let unit = ExecutionUnit::::default(); - unit.pass(edge_mode, m_src, m_dst, image_size, &shifted); + unit.pass(edge_mode, m_src, m_dst, image_size, scanned_kernel); } } @@ -69,7 +50,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "avx512bw")] - unsafe fn pass( + fn pass( &self, edge_mode: BorderHandle, m_src: &RowsHolder, @@ -93,269 +74,286 @@ impl ExecutionUnit { let rnd = _mm512_set1_epi16(1i16 << 6); - for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { - let dst = &mut **dst; - let mut f_cx = 0usize; + unsafe { + for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { + let dst = &mut **dst; + let mut f_cx = 0usize; - while f_cx < min_left { - let mx = f_cx as i64 - s_kernel; - let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); - let e1 = clamp_edge!(edge_mode.edge_mode, 2i64 + mx, 0, width as i64); + while f_cx < min_left { + let mx = f_cx as i64 - s_kernel; + let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); + let e1 = clamp_edge!(edge_mode.edge_mode, 2i64 + mx, 0, width as i64); - for c in 0..N { - let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c1 as u16; + for c in 0..N { + let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c1 as u16; - let src0 = *src.get_unchecked(e0 * N + c); - let src1 = *src.get_unchecked(e1 * N + c); + let src0 = *src.get_unchecked(e0 * N + c); + let src1 = *src.get_unchecked(e1 * N + c); - k0 += (src0 as u16 + src1 as u16) * c0 as u16; + k0 += (src0 as u16 + src1 as u16) * c0 as u16; - *dst.get_unchecked_mut(f_cx * N + c) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + *dst.get_unchecked_mut(f_cx * N + c) = + ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + f_cx += 1; } - f_cx += 1; - } - let mut m_cx = f_cx * N; - - let s_half = half_len * N; - let m_right = width.saturating_sub(half_len); - let max_width = m_right * N; - - while m_cx + 64 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let v_source0 = - _mm512_loadu_si512(shifted_src.get_unchecked(0..).as_ptr() as *const _); - let source = - _mm512_loadu_si512(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let v_source4 = - _mm512_loadu_si512(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); - - let mut k0 = _mm512_add_epi16( - rnd, - _mm512_maddubs_epi16(_mm512_unpacklo_epi8(source, _mm512_setzero_si512()), vc1), - ); - let mut k1 = _mm512_add_epi16( - rnd, - _mm512_maddubs_epi16(_mm512_unpackhi_epi8(source, _mm512_setzero_si512()), vc1), - ); - - k0 = _mm512_add_epi16( - k0, - _mm512_maddubs_epi16(_mm512_unpacklo_epi8(v_source0, v_source4), vc0), - ); - k1 = _mm512_add_epi16( - k1, - _mm512_maddubs_epi16(_mm512_unpackhi_epi8(v_source0, v_source4), vc0), - ); - - k0 = _mm512_srai_epi16::<7>(k0); - k1 = _mm512_srai_epi16::<7>(k1); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm512_storeu_si512(dst_ptr0 as *mut _, _mm512_packus_epi16(k0, k1)); - m_cx += 64; - } + let mut m_cx = f_cx * N; + + let s_half = half_len * N; + let m_right = width.saturating_sub(half_len); + let max_width = m_right * N; + + while m_cx + 64 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let v_source0 = + _mm512_loadu_si512(shifted_src.get_unchecked(0..).as_ptr() as *const _); + let source = + _mm512_loadu_si512(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let v_source4 = _mm512_loadu_si512( + shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _, + ); + + let mut k0 = _mm512_add_epi16( + rnd, + _mm512_maddubs_epi16( + _mm512_unpacklo_epi8(source, _mm512_setzero_si512()), + vc1, + ), + ); + let mut k1 = _mm512_add_epi16( + rnd, + _mm512_maddubs_epi16( + _mm512_unpackhi_epi8(source, _mm512_setzero_si512()), + vc1, + ), + ); + + k0 = _mm512_add_epi16( + k0, + _mm512_maddubs_epi16(_mm512_unpacklo_epi8(v_source0, v_source4), vc0), + ); + k1 = _mm512_add_epi16( + k1, + _mm512_maddubs_epi16(_mm512_unpackhi_epi8(v_source0, v_source4), vc0), + ); + + k0 = _mm512_srai_epi16::<7>(k0); + k1 = _mm512_srai_epi16::<7>(k1); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm512_storeu_si512(dst_ptr0 as *mut _, _mm512_packus_epi16(k0, k1)); + m_cx += 64; + } - while m_cx + 32 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let v_source0 = - _mm256_loadu_si256(shifted_src.get_unchecked(0..).as_ptr() as *const _); - let source = - _mm256_loadu_si256(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let v_source4 = - _mm256_loadu_si256(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); - - let mut k0 = _mm256_add_epi16( - _mm512_castsi512_si256(rnd), - _mm256_maddubs_epi16( - _mm256_unpacklo_epi8(source, _mm256_setzero_si256()), - _mm512_castsi512_si256(vc1), - ), - ); - let mut k1 = _mm256_add_epi16( - _mm512_castsi512_si256(rnd), - _mm256_maddubs_epi16( - _mm256_unpackhi_epi8(source, _mm256_setzero_si256()), - _mm512_castsi512_si256(vc1), - ), - ); - - k0 = _mm256_add_epi16( - k0, - _mm256_maddubs_epi16( - _mm256_unpacklo_epi8(v_source0, v_source4), - _mm512_castsi512_si256(vc0), - ), - ); - k1 = _mm256_add_epi16( - k1, - _mm256_maddubs_epi16( - _mm256_unpackhi_epi8(v_source0, v_source4), - _mm512_castsi512_si256(vc0), - ), - ); - - k0 = _mm256_srai_epi16::<7>(k0); - k1 = _mm256_srai_epi16::<7>(k1); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_packus_epi16(k0, k1)); - m_cx += 32; - } + while m_cx + 32 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let v_source0 = + _mm256_loadu_si256(shifted_src.get_unchecked(0..).as_ptr() as *const _); + let source = + _mm256_loadu_si256(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let v_source4 = _mm256_loadu_si256( + shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _, + ); + + let mut k0 = _mm256_add_epi16( + _mm512_castsi512_si256(rnd), + _mm256_maddubs_epi16( + _mm256_unpacklo_epi8(source, _mm256_setzero_si256()), + _mm512_castsi512_si256(vc1), + ), + ); + let mut k1 = _mm256_add_epi16( + _mm512_castsi512_si256(rnd), + _mm256_maddubs_epi16( + _mm256_unpackhi_epi8(source, _mm256_setzero_si256()), + _mm512_castsi512_si256(vc1), + ), + ); + + k0 = _mm256_add_epi16( + k0, + _mm256_maddubs_epi16( + _mm256_unpacklo_epi8(v_source0, v_source4), + _mm512_castsi512_si256(vc0), + ), + ); + k1 = _mm256_add_epi16( + k1, + _mm256_maddubs_epi16( + _mm256_unpackhi_epi8(v_source0, v_source4), + _mm512_castsi512_si256(vc0), + ), + ); + + k0 = _mm256_srai_epi16::<7>(k0); + k1 = _mm256_srai_epi16::<7>(k1); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, _mm256_packus_epi16(k0, k1)); + m_cx += 32; + } - while m_cx + 16 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked(0..).as_ptr() as *const _); - let source = _mm_loadu_si128(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let v_source4 = - _mm_loadu_si128(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); - - let mut k0 = _mm_add_epi16( - _mm512_castsi512_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm512_castsi512_si128(vc1), - ), - ); - let mut k1 = _mm_add_epi16( - _mm512_castsi512_si128(rnd), - _mm_maddubs_epi16( - _mm_unpackhi_epi8(source, _mm_setzero_si128()), - _mm512_castsi512_si128(vc1), - ), - ); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source0, v_source4), - _mm512_castsi512_si128(vc0), - ), - ); - k1 = _mm_add_epi16( - k1, - _mm_maddubs_epi16( - _mm_unpackhi_epi8(v_source0, v_source4), - _mm512_castsi512_si128(vc0), - ), - ); - - k0 = _mm_srai_epi16::<7>(k0); - k1 = _mm_srai_epi16::<7>(k1); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_packus_epi16(k0, k1)); - m_cx += 16; - } + while m_cx + 16 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked(0..).as_ptr() as *const _); + let source = + _mm_loadu_si128(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let v_source4 = + _mm_loadu_si128(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); + + let mut k0 = _mm_add_epi16( + _mm512_castsi512_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm512_castsi512_si128(vc1), + ), + ); + let mut k1 = _mm_add_epi16( + _mm512_castsi512_si128(rnd), + _mm_maddubs_epi16( + _mm_unpackhi_epi8(source, _mm_setzero_si128()), + _mm512_castsi512_si128(vc1), + ), + ); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source0, v_source4), + _mm512_castsi512_si128(vc0), + ), + ); + k1 = _mm_add_epi16( + k1, + _mm_maddubs_epi16( + _mm_unpackhi_epi8(v_source0, v_source4), + _mm512_castsi512_si128(vc0), + ), + ); + + k0 = _mm_srai_epi16::<7>(k0); + k1 = _mm_srai_epi16::<7>(k1); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_packus_epi16(k0, k1)); + m_cx += 16; + } - while m_cx + 8 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = _mm_loadu_si64(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - _mm512_castsi512_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm512_castsi512_si128(vc1), - ), - ); - - let v_source0 = _mm_loadu_si64(shifted_src.get_unchecked(..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si64(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source0, v_source1), - _mm512_castsi512_si128(vc0), - ), - ); - - k0 = _mm_srai_epi16::<7>(k0); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si64( - dst_ptr0 as *mut _, - _mm_packus_epi16(k0, _mm_setzero_si128()), - ); - m_cx += 8; - } + while m_cx + 8 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let source = + _mm_loadu_si64(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let mut k0 = _mm_add_epi16( + _mm512_castsi512_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm512_castsi512_si128(vc1), + ), + ); + + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked(..).as_ptr() as *const _); + let v_source1 = + _mm_loadu_si64(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source0, v_source1), + _mm512_castsi512_si128(vc0), + ), + ); + + k0 = _mm_srai_epi16::<7>(k0); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si64( + dst_ptr0 as *mut _, + _mm_packus_epi16(k0, _mm_setzero_si128()), + ); + m_cx += 8; + } - while m_cx + 4 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = _mm_loadu_si32(shifted_src.get_unchecked(N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - _mm512_castsi512_si128(rnd), - _mm_maddubs_epi16( - _mm_unpacklo_epi8(source, _mm_setzero_si128()), - _mm512_castsi512_si128(vc1), - ), - ); - - let v_source0 = _mm_loadu_si32(shifted_src.get_unchecked(..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si32(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); - - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16( - _mm_unpacklo_epi8(v_source0, v_source1), - _mm512_castsi512_si128(vc0), - ), - ); - - k0 = _mm_srai_epi16::<7>(k0); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si32( - dst_ptr0 as *mut _, - _mm_packus_epi16(k0, _mm_setzero_si128()), - ); - m_cx += 4; - } + while m_cx + 4 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); + + let source = + _mm_loadu_si32(shifted_src.get_unchecked(N..).as_ptr() as *const _); + let mut k0 = _mm_add_epi16( + _mm512_castsi512_si128(rnd), + _mm_maddubs_epi16( + _mm_unpacklo_epi8(source, _mm_setzero_si128()), + _mm512_castsi512_si128(vc1), + ), + ); + + let v_source0 = + _mm_loadu_si32(shifted_src.get_unchecked(..).as_ptr() as *const _); + let v_source1 = + _mm_loadu_si32(shifted_src.get_unchecked((2 * N)..).as_ptr() as *const _); + + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16( + _mm_unpacklo_epi8(v_source0, v_source1), + _mm512_castsi512_si128(vc0), + ), + ); + + k0 = _mm_srai_epi16::<7>(k0); + + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si32( + dst_ptr0 as *mut _, + _mm_packus_epi16(k0, _mm_setzero_si128()), + ); + m_cx += 4; + } - for zx in m_cx..max_width { - let x = zx - s_half; + for zx in m_cx..max_width { + let x = zx - s_half; - let shifted_src = src.get_unchecked(x..); - let mut k0: u16 = *shifted_src.get_unchecked(N) as u16 * c1 as u16; + let shifted_src = src.get_unchecked(x..); + let mut k0: u16 = *shifted_src.get_unchecked(N) as u16 * c1 as u16; - k0 += (*shifted_src.get_unchecked(0) as u16 - + *shifted_src.get_unchecked(2 * N) as u16) - * c0 as u16; + k0 += (*shifted_src.get_unchecked(0) as u16 + + *shifted_src.get_unchecked(2 * N) as u16) + * c0 as u16; - *dst.get_unchecked_mut(zx) = ((k0 + (1 << 6)) >> 7).min(255) as u8; - } + *dst.get_unchecked_mut(zx) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + } - f_cx = m_right; + f_cx = m_right; - while f_cx < width { - let mx = f_cx as i64 - s_kernel; - let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); - let e1 = clamp_edge!(edge_mode.edge_mode, 2i64 + mx, 0, width as i64); + while f_cx < width { + let mx = f_cx as i64 - s_kernel; + let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); + let e1 = clamp_edge!(edge_mode.edge_mode, 2i64 + mx, 0, width as i64); - for c in 0..N { - let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c1 as u16; + for c in 0..N { + let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * c1 as u16; - let src0 = *src.get_unchecked(e0 * N + c); - let src1 = *src.get_unchecked(e1 * N + c); + let src0 = *src.get_unchecked(e0 * N + c); + let src1 = *src.get_unchecked(e1 * N + c); - k0 += (src0 as u16 + src1 as u16) * c0 as u16; + k0 += (src0 as u16 + src1 as u16) * c0 as u16; - *dst.get_unchecked_mut(f_cx * N + c) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + *dst.get_unchecked_mut(f_cx * N + c) = + ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + f_cx += 1; } - f_cx += 1; } } } diff --git a/src/filter1d/avx512/sse_utils.rs b/src/filter1d/avx512/sse_utils.rs index e3a8de9a..971172e5 100644 --- a/src/filter1d/avx512/sse_utils.rs +++ b/src/filter1d/avx512/sse_utils.rs @@ -33,305 +33,327 @@ use std::arch::x86::*; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_epi16_x4( - input: __m128i, - weight: __m128i, -) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); +pub(crate) fn _mm_mul_epi8_by_epi16_x4(input: __m128i, weight: __m128i) -> (__m128i, __m128i) { + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); - ( - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), - ) + ( + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epu16_widen(input: __m128i, weight: __m128i) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi16(input, _mm_setzero_si128()); - let k1 = _mm_unpackhi_epi16(input, _mm_setzero_si128()); +pub(crate) fn _mm_mul_epu16_widen(input: __m128i, weight: __m128i) -> (__m128i, __m128i) { + unsafe { + let k0 = _mm_unpacklo_epi16(input, _mm_setzero_si128()); + let k1 = _mm_unpackhi_epi16(input, _mm_setzero_si128()); - (_mm_mullo_epi32(k0, weight), _mm_mullo_epi32(k1, weight)) + (_mm_mullo_epi32(k0, weight), _mm_mullo_epi32(k1, weight)) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_ps_x4( +pub(crate) fn _mm_mul_epi8_by_ps_x4( input: __m128i, weight: __m128, ) -> (__m128, __m128, __m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); - let hi_16 = _mm_unpackhi_epi8(input, zeros); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); + let hi_16 = _mm_unpackhi_epi8(input, zeros); - let a0 = _mm_unpacklo_epi16(lo_16, zeros); - let a1 = _mm_unpackhi_epi16(lo_16, zeros); - let a2 = _mm_unpacklo_epi16(hi_16, zeros); - let a3 = _mm_unpackhi_epi16(hi_16, zeros); + let a0 = _mm_unpacklo_epi16(lo_16, zeros); + let a1 = _mm_unpackhi_epi16(lo_16, zeros); + let a2 = _mm_unpacklo_epi16(hi_16, zeros); + let a3 = _mm_unpackhi_epi16(hi_16, zeros); - let v0 = _mm_cvtepi32_ps(a0); - let v1 = _mm_cvtepi32_ps(a1); - let v2 = _mm_cvtepi32_ps(a2); - let v3 = _mm_cvtepi32_ps(a3); + let v0 = _mm_cvtepi32_ps(a0); + let v1 = _mm_cvtepi32_ps(a1); + let v2 = _mm_cvtepi32_ps(a2); + let v3 = _mm_cvtepi32_ps(a3); - ( - _mm_mul_ps(v0, weight), - _mm_mul_ps(v1, weight), - _mm_mul_ps(v2, weight), - _mm_mul_ps(v3, weight), - ) + ( + _mm_mul_ps(v0, weight), + _mm_mul_ps(v1, weight), + _mm_mul_ps(v2, weight), + _mm_mul_ps(v3, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi16_by_ps_x2( +pub(crate) fn _mm_mul_epi16_by_ps_x2( input: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let a0 = _mm_unpacklo_epi16(input, zeros); - let a1 = _mm_unpackhi_epi16(input, zeros); + unsafe { + let zeros = _mm_setzero_si128(); + let a0 = _mm_unpacklo_epi16(input, zeros); + let a1 = _mm_unpackhi_epi16(input, zeros); - let v0 = _mm_cvtepi32_ps(a0); - let v1 = _mm_cvtepi32_ps(a1); + let v0 = _mm_cvtepi32_ps(a0); + let v1 = _mm_cvtepi32_ps(a1); - (_mm_mul_ps(v0, weight), _mm_mul_ps(v1, weight)) + (_mm_mul_ps(v0, weight), _mm_mul_ps(v1, weight)) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_epi8_by_epi16_x4( - input: __m128i, - weight: __m128i, -) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); +pub(crate) fn _mm_mull_epi8_by_epi16_x4(input: __m128i, weight: __m128i) -> (__m128i, __m128i) { + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); - ( - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), - ) + ( + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); +pub(crate) fn _mm_mull_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_ps_x2(input: __m128i, weight: __m128) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); +pub(crate) fn _mm_mul_epi8_by_ps_x2(input: __m128i, weight: __m128) -> (__m128, __m128) { + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); - (_mm_mul_ps(a0, weight), _mm_mul_ps(a1, weight)) + (_mm_mul_ps(a0, weight), _mm_mul_ps(a1, weight)) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi16_by_ps( - input: __m128i, - weight: __m128, -) -> __m128 { - let zeros = _mm_setzero_si128(); - let k0 = _mm_unpacklo_epi16(input, zeros); - let a0 = _mm_cvtepi32_ps(k0); +pub(crate) fn _mm_mul_epi16_by_ps(input: __m128i, weight: __m128) -> __m128 { + unsafe { + let zeros = _mm_setzero_si128(); + let k0 = _mm_unpacklo_epi16(input, zeros); + let a0 = _mm_cvtepi32_ps(k0); - _mm_mul_ps(a0, weight) + _mm_mul_ps(a0, weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) +pub(crate) fn _mm_mul_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) + } } #[inline(always)] pub(crate) unsafe fn _mm_opt_fmlaf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { - _mm_add_ps(_mm_mul_ps(b, c), a) + unsafe { _mm_add_ps(_mm_mul_ps(b, c), a) } } #[inline(always)] -pub(crate) unsafe fn _mm_fmla_pd(a: __m128d, b: __m128d, c: __m128d) -> __m128d { - _mm_add_pd(_mm_mul_pd(b, c), a) +pub(crate) fn _mm_fmla_pd(a: __m128d, b: __m128d, c: __m128d) -> __m128d { + unsafe { _mm_add_pd(_mm_mul_pd(b, c), a) } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_ps_x4( +pub(crate) fn _mm_mul_add_epi8_by_ps_x4( accumulator: (__m128, __m128, __m128, __m128), input: __m128i, weight: __m128, ) -> (__m128, __m128, __m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); - let hi_16 = _mm_unpackhi_epi8(input, zeros); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); + let hi_16 = _mm_unpackhi_epi8(input, zeros); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let k2 = _mm_unpacklo_epi16(hi_16, zeros); - let k3 = _mm_unpackhi_epi16(hi_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k2 = _mm_unpacklo_epi16(hi_16, zeros); + let k3 = _mm_unpackhi_epi16(hi_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); - let a2 = _mm_cvtepi32_ps(k2); - let a3 = _mm_cvtepi32_ps(k3); + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); + let a2 = _mm_cvtepi32_ps(k2); + let a3 = _mm_cvtepi32_ps(k3); - ( - _mm_opt_fmlaf_ps(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps(accumulator.1, a1, weight), - _mm_opt_fmlaf_ps(accumulator.2, a2, weight), - _mm_opt_fmlaf_ps(accumulator.3, a3, weight), - ) + ( + _mm_opt_fmlaf_ps(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps(accumulator.1, a1, weight), + _mm_opt_fmlaf_ps(accumulator.2, a2, weight), + _mm_opt_fmlaf_ps(accumulator.3, a3, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_epi8_by_epi16_x4( +pub(crate) fn _mm_mull_add_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); - let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); - let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); - ( - _mm_add_epi16(accumulator.0, j0), - _mm_add_epi16(accumulator.1, j1), - ) + let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); + let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); + ( + _mm_add_epi16(accumulator.0, j0), + _mm_add_epi16(accumulator.1, j1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_epi8_by_epi16_x2( +pub(crate) fn _mm_mull_add_epi8_by_epi16_x2( accumulator: __m128i, input: __m128i, weight: __m128i, ) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_add_epi16( - accumulator, - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), - ) + _mm_add_epi16( + accumulator, + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_symm_epi8_by_epi16_x4( +pub(crate) fn _mm_mull_add_symm_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); - let hi_16 = _mm_add_epi16( - _mm_unpackhi_epi8(input0, zeros), - _mm_unpackhi_epi8(input1, zeros), - ); - let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight); - let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(hi_16), weight); - - ( - _mm_add_epi16(accumulator.0, j0), - _mm_add_epi16(accumulator.1, j1), - ) -} - -#[inline(always)] -pub(crate) unsafe fn _mm_madd_symm_epi8_by_epi16_x4( + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); + let hi_16 = _mm_add_epi16( + _mm_unpackhi_epi8(input0, zeros), + _mm_unpackhi_epi8(input1, zeros), + ); + let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight); + let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(hi_16), weight); + + ( + _mm_add_epi16(accumulator.0, j0), + _mm_add_epi16(accumulator.1, j1), + ) + } +} + +#[inline(always)] +pub(crate) fn _mm_madd_symm_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); - let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); - let v2 = _mm_unpacklo_epi8(input1, _mm_setzero_si128()); - let v3 = _mm_unpackhi_epi8(input1, _mm_setzero_si128()); + unsafe { + let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); + let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); + let v2 = _mm_unpacklo_epi8(input1, _mm_setzero_si128()); + let v3 = _mm_unpackhi_epi8(input1, _mm_setzero_si128()); - let m0 = _mm_add_epi16(v0, v2); - let m1 = _mm_add_epi16(v1, v3); + let m0 = _mm_add_epi16(v0, v2); + let m1 = _mm_add_epi16(v1, v3); - let a0 = _mm_mullo_epi16(m0, weight); - let a1 = _mm_mullo_epi16(m1, weight); + let a0 = _mm_mullo_epi16(m0, weight); + let a1 = _mm_mullo_epi16(m1, weight); - ( - _mm_add_epi16(accumulator.0, a0), - _mm_add_epi16(accumulator.1, a1), - ) + ( + _mm_add_epi16(accumulator.0, a0), + _mm_add_epi16(accumulator.1, a1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_madd_s_epi8_by_epi16_x4( +pub(crate) fn _mm_madd_s_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); - let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); + unsafe { + let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); + let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); - let a0 = _mm_mullo_epi16(v0, weight); - let a1 = _mm_mullo_epi16(v1, weight); + let a0 = _mm_mullo_epi16(v0, weight); + let a1 = _mm_mullo_epi16(v1, weight); - ( - _mm_add_epi16(accumulator.0, a0), - _mm_add_epi16(accumulator.1, a1), - ) + ( + _mm_add_epi16(accumulator.0, a0), + _mm_add_epi16(accumulator.1, a1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_madd_epi8_by_epi16_x4( - input0: __m128i, - weight: __m128i, -) -> (__m128i, __m128i) { - let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); - let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); +pub(crate) fn _mm_madd_epi8_by_epi16_x4(input0: __m128i, weight: __m128i) -> (__m128i, __m128i) { + unsafe { + let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); + let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); - let a0 = _mm_mullo_epi16(v0, weight); - let a1 = _mm_mullo_epi16(v1, weight); + let a0 = _mm_mullo_epi16(v0, weight); + let a1 = _mm_mullo_epi16(v1, weight); - (a0, a1) + (a0, a1) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_ps_x4( +pub(crate) fn _mm_mul_add_symm_epi8_by_ps_x4( accumulator: (__m128, __m128, __m128, __m128), input0: __m128i, input1: __m128i, weight: __m128, ) -> (__m128, __m128, __m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); - let hi_16 = _mm_add_epi16( - _mm_unpackhi_epi8(input0, zeros), - _mm_unpackhi_epi8(input1, zeros), - ); - - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let k2 = _mm_unpacklo_epi16(hi_16, zeros); - let k3 = _mm_unpackhi_epi16(hi_16, zeros); - - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); - let a2 = _mm_cvtepi32_ps(k2); - let a3 = _mm_cvtepi32_ps(k3); - - ( - _mm_opt_fmlaf_ps(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps(accumulator.1, a1, weight), - _mm_opt_fmlaf_ps(accumulator.2, a2, weight), - _mm_opt_fmlaf_ps(accumulator.3, a3, weight), - ) + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); + let hi_16 = _mm_add_epi16( + _mm_unpackhi_epi8(input0, zeros), + _mm_unpackhi_epi8(input1, zeros), + ); + + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k2 = _mm_unpacklo_epi16(hi_16, zeros); + let k3 = _mm_unpackhi_epi16(hi_16, zeros); + + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); + let a2 = _mm_cvtepi32_ps(k2); + let a3 = _mm_cvtepi32_ps(k3); + + ( + _mm_opt_fmlaf_ps(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps(accumulator.1, a1, weight), + _mm_opt_fmlaf_ps(accumulator.2, a2, weight), + _mm_opt_fmlaf_ps(accumulator.3, a3, weight), + ) + } } #[inline(always)] @@ -341,256 +363,288 @@ pub(crate) unsafe fn _mm_mul_add_symm_epi16_by_ps_x2( input1: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let l32 = _mm_add_epi32( - _mm_unpacklo_epi16(input0, zeros), - _mm_unpacklo_epi16(input1, zeros), - ); - let h32 = _mm_add_epi32( - _mm_unpackhi_epi16(input0, zeros), - _mm_unpackhi_epi16(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let l32 = _mm_add_epi32( + _mm_unpacklo_epi16(input0, zeros), + _mm_unpacklo_epi16(input1, zeros), + ); + let h32 = _mm_add_epi32( + _mm_unpackhi_epi16(input0, zeros), + _mm_unpackhi_epi16(input1, zeros), + ); - let a0 = _mm_cvtepi32_ps(l32); - let a1 = _mm_cvtepi32_ps(h32); + let a0 = _mm_cvtepi32_ps(l32); + let a1 = _mm_cvtepi32_ps(h32); - ( - _mm_opt_fmlaf_ps(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps(accumulator.1, a1, weight), - ) + ( + _mm_opt_fmlaf_ps(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps(accumulator.1, a1, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_ps_x2( +pub(crate) fn _mm_mul_add_epi8_by_ps_x2( accumulator: (__m128, __m128), input: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); - ( - _mm_opt_fmlaf_ps(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps(accumulator.1, a1, weight), - ) + ( + _mm_opt_fmlaf_ps(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps(accumulator.1, a1, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_ps_x2( +pub(crate) fn _mm_mul_add_symm_epi8_by_ps_x2( accumulator: (__m128, __m128), input0: __m128i, input1: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); - ( - _mm_opt_fmlaf_ps(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps(accumulator.1, a1, weight), - ) + ( + _mm_opt_fmlaf_ps(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps(accumulator.1, a1, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi16_by_ps( +pub(crate) fn _mm_mul_add_symm_epi16_by_ps( accumulator: __m128, input0: __m128i, input1: __m128i, weight: __m128, ) -> __m128 { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi32( - _mm_unpacklo_epi16(input0, zeros), - _mm_unpacklo_epi16(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi32( + _mm_unpacklo_epi16(input0, zeros), + _mm_unpacklo_epi16(input1, zeros), + ); - let a0 = _mm_cvtepi32_ps(lo_16); + let a0 = _mm_cvtepi32_ps(lo_16); - _mm_opt_fmlaf_ps(accumulator, a0, weight) + _mm_opt_fmlaf_ps(accumulator, a0, weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_symm_epi8_by_epi16_x2( +pub(crate) fn _mm_mull_add_symm_epi8_by_epi16_x2( accumulator: __m128i, input0: __m128i, input1: __m128i, weight: __m128i, ) -> __m128i { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); - _mm_add_epi16(accumulator, _mm_mulhrs_epi16(lo_16, weight)) + _mm_add_epi16(accumulator, _mm_mulhrs_epi16(lo_16, weight)) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_epi16_x4( +pub(crate) fn _mm_mul_add_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); - let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); - let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); + let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); + let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); - ( - _mm_add_epi16(j0, accumulator.0), - _mm_add_epi16(j1, accumulator.1), - ) + ( + _mm_add_epi16(j0, accumulator.0), + _mm_add_epi16(j1, accumulator.1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_epi16_x4( +pub(crate) fn _mm_mul_add_symm_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let zeros = _mm_setzero_si128(); - let k0 = _mm_unpacklo_epi8(input0, zeros); - let k1 = _mm_unpacklo_epi8(input1, zeros); - let k2 = _mm_unpackhi_epi8(input0, zeros); - let k3 = _mm_unpackhi_epi8(input1, zeros); - let v0 = _mm_add_epi16(k0, k1); - let v1 = _mm_add_epi16(k2, k3); + unsafe { + let zeros = _mm_setzero_si128(); + let k0 = _mm_unpacklo_epi8(input0, zeros); + let k1 = _mm_unpacklo_epi8(input1, zeros); + let k2 = _mm_unpackhi_epi8(input0, zeros); + let k3 = _mm_unpackhi_epi8(input1, zeros); + let v0 = _mm_add_epi16(k0, k1); + let v1 = _mm_add_epi16(k2, k3); - let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v0), weight); - let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v1), weight); + let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v0), weight); + let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v1), weight); - ( - _mm_add_epi16(j0, accumulator.0), - _mm_add_epi16(j1, accumulator.1), - ) + ( + _mm_add_epi16(j0, accumulator.0), + _mm_add_epi16(j1, accumulator.1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epu16_by_epu16_x4( +pub(crate) fn _mm_mul_add_symm_epu16_by_epu16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let zeros = _mm_setzero_si128(); - let k0 = _mm_unpacklo_epi16(input0, zeros); - let k1 = _mm_unpacklo_epi16(input1, zeros); - let k2 = _mm_unpackhi_epi16(input0, zeros); - let k3 = _mm_unpackhi_epi16(input1, zeros); - let v0 = _mm_add_epi32(k0, k1); - let v1 = _mm_add_epi32(k2, k3); + unsafe { + let zeros = _mm_setzero_si128(); + let k0 = _mm_unpacklo_epi16(input0, zeros); + let k1 = _mm_unpacklo_epi16(input1, zeros); + let k2 = _mm_unpackhi_epi16(input0, zeros); + let k3 = _mm_unpackhi_epi16(input1, zeros); + let v0 = _mm_add_epi32(k0, k1); + let v1 = _mm_add_epi32(k2, k3); - let j0 = _mm_mullo_epi32(v0, weight); - let j1 = _mm_mullo_epi32(v1, weight); + let j0 = _mm_mullo_epi32(v0, weight); + let j1 = _mm_mullo_epi32(v1, weight); - ( - _mm_add_epi32(j0, accumulator.0), - _mm_add_epi32(j1, accumulator.1), - ) + ( + _mm_add_epi32(j0, accumulator.0), + _mm_add_epi32(j1, accumulator.1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_epi16_x2( +pub(crate) fn _mm_mul_add_epi8_by_epi16_x2( accumulator: __m128i, input: __m128i, weight: __m128i, ) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_add_epi16( - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), - accumulator, - ) + _mm_add_epi16( + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), + accumulator, + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_epi16_x2( +pub(crate) fn _mm_mul_add_symm_epi8_by_epi16_x2( accumulator: __m128i, input0: __m128i, input1: __m128i, weight: __m128i, ) -> __m128i { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); - _mm_add_epi16( - _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight), - accumulator, - ) + _mm_add_epi16( + _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight), + accumulator, + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_epi32_x2_epi8(store: (__m128i, __m128i)) -> __m128i { - let rnd = _mm_set1_epi16((1 << 5) - 1); - _mm_packus_epi16( - _mm_srai_epi16::<6>(_mm_add_epi16(store.0, rnd)), - _mm_srai_epi16::<6>(_mm_add_epi16(store.1, rnd)), - ) +pub(crate) fn _mm_pack_epi32_x2_epi8(store: (__m128i, __m128i)) -> __m128i { + unsafe { + let rnd = _mm_set1_epi16((1 << 5) - 1); + _mm_packus_epi16( + _mm_srai_epi16::<6>(_mm_add_epi16(store.0, rnd)), + _mm_srai_epi16::<6>(_mm_add_epi16(store.1, rnd)), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_epi32_x2_epu16(store: (__m128i, __m128i)) -> __m128i { - let rnd = _mm_set1_epi32((1 << 14) - 1); - _mm_packus_epi32( - _mm_srli_epi32::<15>(_mm_add_epi32(store.0, rnd)), - _mm_srli_epi32::<15>(_mm_add_epi32(store.1, rnd)), - ) +pub(crate) fn _mm_pack_epi32_x2_epu16(store: (__m128i, __m128i)) -> __m128i { + unsafe { + let rnd = _mm_set1_epi32((1 << 14) - 1); + _mm_packus_epi32( + _mm_srli_epi32::<15>(_mm_add_epi32(store.0, rnd)), + _mm_srli_epi32::<15>(_mm_add_epi32(store.1, rnd)), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_epi32_epi8(store: __m128i) -> __m128i { - let rnd = _mm_set1_epi16((1 << 5) - 1); - _mm_packus_epi16( - _mm_srai_epi16::<6>(_mm_add_epi16(store, rnd)), - _mm_setzero_si128(), - ) +pub(crate) fn _mm_pack_epi32_epi8(store: __m128i) -> __m128i { + unsafe { + let rnd = _mm_set1_epi16((1 << 5) - 1); + _mm_packus_epi16( + _mm_srai_epi16::<6>(_mm_add_epi16(store, rnd)), + _mm_setzero_si128(), + ) + } } #[inline(always)] pub(crate) unsafe fn _mm_pack_ps_x4_epi8(store: (__m128, __m128, __m128, __m128)) -> __m128i { - let o0 = _mm_cvtps_epi32(store.0); - let o1 = _mm_cvtps_epi32(store.1); - let o2 = _mm_cvtps_epi32(store.2); - let o3 = _mm_cvtps_epi32(store.3); - let hi_s = _mm_packs_epi32(o2, o3); - let lo_s = _mm_packs_epi32(o0, o1); - _mm_packus_epi16(lo_s, hi_s) + unsafe { + let o0 = _mm_cvtps_epi32(store.0); + let o1 = _mm_cvtps_epi32(store.1); + let o2 = _mm_cvtps_epi32(store.2); + let o3 = _mm_cvtps_epi32(store.3); + let hi_s = _mm_packs_epi32(o2, o3); + let lo_s = _mm_packs_epi32(o0, o1); + _mm_packus_epi16(lo_s, hi_s) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_x2_epi16(store: (__m128, __m128)) -> __m128i { - let o0 = _mm_cvtps_epi32(store.0); - let o1 = _mm_cvtps_epi32(store.1); - _mm_packus_epi32(o0, o1) +pub(crate) fn _mm_pack_ps_x2_epi16(store: (__m128, __m128)) -> __m128i { + unsafe { + let o0 = _mm_cvtps_epi32(store.0); + let o1 = _mm_cvtps_epi32(store.1); + _mm_packus_epi32(o0, o1) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_x2_epi8(store: (__m128, __m128)) -> __m128i { - let lo_s = _mm_packs_epi32(_mm_cvtps_epi32(store.0), _mm_cvtps_epi32(store.1)); - _mm_packus_epi16(lo_s, _mm_setzero_si128()) +pub(crate) fn _mm_pack_ps_x2_epi8(store: (__m128, __m128)) -> __m128i { + unsafe { + let lo_s = _mm_packs_epi32(_mm_cvtps_epi32(store.0), _mm_cvtps_epi32(store.1)); + _mm_packus_epi16(lo_s, _mm_setzero_si128()) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_epi16(store: __m128) -> __m128i { - _mm_packus_epi32(_mm_cvtps_epi32(store), _mm_setzero_si128()) +pub(crate) fn _mm_pack_ps_epi16(store: __m128) -> __m128i { + unsafe { _mm_packus_epi32(_mm_cvtps_epi32(store), _mm_setzero_si128()) } } diff --git a/src/filter1d/avx512/utils.rs b/src/filter1d/avx512/utils.rs index d80b345a..cc3b8a9c 100644 --- a/src/filter1d/avx512/utils.rs +++ b/src/filter1d/avx512/utils.rs @@ -29,66 +29,72 @@ use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm_fmlaf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { - _mm_fmadd_ps(b, c, a) +pub(crate) fn _mm_fmlaf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { + unsafe { _mm_fmadd_ps(b, c, a) } } #[inline(always)] -pub(crate) unsafe fn _mm256_fmlaf_ps(a: __m256, b: __m256, c: __m256) -> __m256 { - _mm256_fmadd_ps(b, c, a) +pub(crate) fn _mm256_fmlaf_ps(a: __m256, b: __m256, c: __m256) -> __m256 { + unsafe { _mm256_fmadd_ps(b, c, a) } } #[inline(always)] -pub(crate) unsafe fn _mm512_fmlaf_ps(a: __m512, b: __m512, c: __m512) -> __m512 { - _mm512_fmadd_ps(b, c, a) +pub(crate) fn _mm512_fmlaf_ps(a: __m512, b: __m512, c: __m512) -> __m512 { + unsafe { _mm512_fmadd_ps(b, c, a) } } #[inline(always)] -pub(crate) unsafe fn _mm512_fmla_pd(a: __m512d, b: __m512d, c: __m512d) -> __m512d { - _mm512_fmadd_pd(b, c, a) +pub(crate) fn _mm512_fmla_pd(a: __m512d, b: __m512d, c: __m512d) -> __m512d { + unsafe { _mm512_fmadd_pd(b, c, a) } } #[inline(always)] -pub(crate) unsafe fn _mm512_mul_epu16_widen(input: __m512i, weight: __m512i) -> (__m512i, __m512i) { - let v_lo = _mm512_unpacklo_epi16(input, _mm512_setzero_si512()); - let v_hi = _mm512_unpackhi_epi16(input, _mm512_setzero_si512()); - ( - _mm512_mullo_epi32(v_lo, weight), - _mm512_mullo_epi32(v_hi, weight), - ) +pub(crate) fn _mm512_mul_epu16_widen(input: __m512i, weight: __m512i) -> (__m512i, __m512i) { + unsafe { + let v_lo = _mm512_unpacklo_epi16(input, _mm512_setzero_si512()); + let v_hi = _mm512_unpackhi_epi16(input, _mm512_setzero_si512()); + ( + _mm512_mullo_epi32(v_lo, weight), + _mm512_mullo_epi32(v_hi, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm512_mul_add_symm_epu16_by_epu16_x4( +pub(crate) fn _mm512_mul_add_symm_epu16_by_epu16_x4( accumulator: (__m512i, __m512i), input0: __m512i, input1: __m512i, weight: __m512i, ) -> (__m512i, __m512i) { - let a0 = _mm512_unpacklo_epi16(input0, _mm512_setzero_si512()); - let a1 = _mm512_unpacklo_epi16(input1, _mm512_setzero_si512()); - let a2 = _mm512_unpackhi_epi16(input0, _mm512_setzero_si512()); - let a3 = _mm512_unpackhi_epi16(input1, _mm512_setzero_si512()); - - let lo_16 = _mm512_add_epi32(a0, a1); - let hi_16 = _mm512_add_epi32(a2, a3); - - let vj0 = _mm512_mullo_epi32(lo_16, weight); - let vj1 = _mm512_mullo_epi32(hi_16, weight); - - ( - _mm512_add_epi32(accumulator.0, vj0), - _mm512_add_epi32(accumulator.1, vj1), - ) + unsafe { + let a0 = _mm512_unpacklo_epi16(input0, _mm512_setzero_si512()); + let a1 = _mm512_unpacklo_epi16(input1, _mm512_setzero_si512()); + let a2 = _mm512_unpackhi_epi16(input0, _mm512_setzero_si512()); + let a3 = _mm512_unpackhi_epi16(input1, _mm512_setzero_si512()); + + let lo_16 = _mm512_add_epi32(a0, a1); + let hi_16 = _mm512_add_epi32(a2, a3); + + let vj0 = _mm512_mullo_epi32(lo_16, weight); + let vj1 = _mm512_mullo_epi32(hi_16, weight); + + ( + _mm512_add_epi32(accumulator.0, vj0), + _mm512_add_epi32(accumulator.1, vj1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm512_pack_epi32_x4_epu16(store: (__m512i, __m512i)) -> __m512i { - let rnd = _mm512_set1_epi32((1 << 14) - 1); - _mm512_packus_epi32( - _mm512_srli_epi32::<15>(_mm512_add_epi32(store.0, rnd)), - _mm512_srli_epi32::<15>(_mm512_add_epi32(store.1, rnd)), - ) +pub(crate) fn _mm512_pack_epi32_x4_epu16(store: (__m512i, __m512i)) -> __m512i { + unsafe { + let rnd = _mm512_set1_epi32((1 << 14) - 1); + _mm512_packus_epi32( + _mm512_srli_epi32::<15>(_mm512_add_epi32(store.0, rnd)), + _mm512_srli_epi32::<15>(_mm512_add_epi32(store.1, rnd)), + ) + } } #[inline(always)] @@ -98,84 +104,86 @@ pub(crate) const fn v512_shuffle(z: u32, y: u32, x: u32, w: u32) -> i32 { } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_epi8_by_epi16_x4( - input: __m256i, - weight: __m256i, -) -> (__m256i, __m256i) { - let j0 = _mm256_unpacklo_epi8(input, input); - let j1 = _mm256_unpackhi_epi8(input, input); - - ( - _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j0), weight), - _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j1), weight), - ) +pub(crate) fn _mm256_mul_epi8_by_epi16_x4(input: __m256i, weight: __m256i) -> (__m256i, __m256i) { + unsafe { + let j0 = _mm256_unpacklo_epi8(input, input); + let j1 = _mm256_unpackhi_epi8(input, input); + + ( + _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j0), weight), + _mm256_mulhrs_epi16(_mm256_srli_epi16::<2>(j1), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm512_mul_epi8_by_epi16_x4( - input: __m512i, - weight: __m512i, -) -> (__m512i, __m512i) { - let j0 = _mm512_unpacklo_epi8(input, input); - let j1 = _mm512_unpackhi_epi8(input, input); - - ( - _mm512_mulhrs_epi16(_mm512_srli_epi16::<2>(j0), weight), - _mm512_mulhrs_epi16(_mm512_srli_epi16::<2>(j1), weight), - ) +pub(crate) fn _mm512_mul_epi8_by_epi16_x4(input: __m512i, weight: __m512i) -> (__m512i, __m512i) { + unsafe { + let j0 = _mm512_unpacklo_epi8(input, input); + let j1 = _mm512_unpackhi_epi8(input, input); + + ( + _mm512_mulhrs_epi16(_mm512_srli_epi16::<2>(j0), weight), + _mm512_mulhrs_epi16(_mm512_srli_epi16::<2>(j1), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm256_mul_add_symm_epi8_by_epi16_x4( +pub(crate) fn _mm256_mul_add_symm_epi8_by_epi16_x4( accumulator: (__m256i, __m256i), input0: __m256i, input1: __m256i, weight: __m256i, ) -> (__m256i, __m256i) { - let a0 = _mm256_unpacklo_epi8(input0, _mm256_setzero_si256()); - let a1 = _mm256_unpacklo_epi8(input1, _mm256_setzero_si256()); - let a2 = _mm256_unpackhi_epi8(input0, _mm256_setzero_si256()); - let a3 = _mm256_unpackhi_epi8(input1, _mm256_setzero_si256()); - - let lo_16 = _mm256_add_epi16(a0, a1); - let hi_16 = _mm256_add_epi16(a2, a3); - - let vj0 = _mm256_mulhrs_epi16(_mm256_slli_epi16::<6>(lo_16), weight); - let vj1 = _mm256_mulhrs_epi16(_mm256_slli_epi16::<6>(hi_16), weight); - - ( - _mm256_add_epi16(accumulator.0, vj0), - _mm256_add_epi16(accumulator.1, vj1), - ) + unsafe { + let a0 = _mm256_unpacklo_epi8(input0, _mm256_setzero_si256()); + let a1 = _mm256_unpacklo_epi8(input1, _mm256_setzero_si256()); + let a2 = _mm256_unpackhi_epi8(input0, _mm256_setzero_si256()); + let a3 = _mm256_unpackhi_epi8(input1, _mm256_setzero_si256()); + + let lo_16 = _mm256_add_epi16(a0, a1); + let hi_16 = _mm256_add_epi16(a2, a3); + + let vj0 = _mm256_mulhrs_epi16(_mm256_slli_epi16::<6>(lo_16), weight); + let vj1 = _mm256_mulhrs_epi16(_mm256_slli_epi16::<6>(hi_16), weight); + + ( + _mm256_add_epi16(accumulator.0, vj0), + _mm256_add_epi16(accumulator.1, vj1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm512_mul_add_symm_epi8_by_epi16_x4( +pub(crate) fn _mm512_mul_add_symm_epi8_by_epi16_x4( accumulator: (__m512i, __m512i), input0: __m512i, input1: __m512i, weight: __m512i, ) -> (__m512i, __m512i) { - let a0 = _mm512_unpacklo_epi8(input0, _mm512_setzero_si512()); - let a1 = _mm512_unpacklo_epi8(input1, _mm512_setzero_si512()); - let a2 = _mm512_unpackhi_epi8(input0, _mm512_setzero_si512()); - let a3 = _mm512_unpackhi_epi8(input1, _mm512_setzero_si512()); - - let lo_16 = _mm512_add_epi16(a0, a1); - let hi_16 = _mm512_add_epi16(a2, a3); - - let vj0 = _mm512_mulhrs_epi16(_mm512_slli_epi16::<6>(lo_16), weight); - let vj1 = _mm512_mulhrs_epi16(_mm512_slli_epi16::<6>(hi_16), weight); - - ( - _mm512_add_epi16(accumulator.0, vj0), - _mm512_add_epi16(accumulator.1, vj1), - ) + unsafe { + let a0 = _mm512_unpacklo_epi8(input0, _mm512_setzero_si512()); + let a1 = _mm512_unpacklo_epi8(input1, _mm512_setzero_si512()); + let a2 = _mm512_unpackhi_epi8(input0, _mm512_setzero_si512()); + let a3 = _mm512_unpackhi_epi8(input1, _mm512_setzero_si512()); + + let lo_16 = _mm512_add_epi16(a0, a1); + let hi_16 = _mm512_add_epi16(a2, a3); + + let vj0 = _mm512_mulhrs_epi16(_mm512_slli_epi16::<6>(lo_16), weight); + let vj1 = _mm512_mulhrs_epi16(_mm512_slli_epi16::<6>(hi_16), weight); + + ( + _mm512_add_epi16(accumulator.0, vj0), + _mm512_add_epi16(accumulator.1, vj1), + ) + } } #[inline] #[target_feature(enable = "avx2")] -pub(crate) unsafe fn _mm256_pack_epi32_x4_epi8(store: (__m256i, __m256i)) -> __m256i { +pub(crate) fn _mm256_pack_epi32_x4_epi8(store: (__m256i, __m256i)) -> __m256i { let rnd = _mm256_set1_epi16((1 << 5) - 1); _mm256_packus_epi16( _mm256_srai_epi16::<6>(_mm256_add_epi16(store.0, rnd)), @@ -185,7 +193,7 @@ pub(crate) unsafe fn _mm256_pack_epi32_x4_epi8(store: (__m256i, __m256i)) -> __m #[inline] #[target_feature(enable = "avx512bw")] -pub(crate) unsafe fn _mm512_pack_epi32_x4_epi8(store: (__m512i, __m512i)) -> __m512i { +pub(crate) fn _mm512_pack_epi32_x4_epi8(store: (__m512i, __m512i)) -> __m512i { let rnd = _mm512_set1_epi16((1 << 5) - 1); _mm512_packus_epi16( _mm512_srai_epi16::<6>(_mm512_add_epi16(store.0, rnd)), diff --git a/src/filter1d/avx512/v_load.rs b/src/filter1d/avx512/v_load.rs index a5b554ec..6788a506 100644 --- a/src/filter1d/avx512/v_load.rs +++ b/src/filter1d/avx512/v_load.rs @@ -32,101 +32,125 @@ use std::arch::x86_64::*; #[inline(always)] pub(crate) unsafe fn _mm256_load_pack_x4(ptr: *const u8) -> (__m256i, __m256i, __m256i, __m256i) { - let row0 = _mm256_loadu_si256(ptr as *const __m256i); - let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); - let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); - let row3 = _mm256_loadu_si256(ptr.add(96) as *const __m256i); - (row0, row1, row2, row3) + unsafe { + let row0 = _mm256_loadu_si256(ptr as *const __m256i); + let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); + let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); + let row3 = _mm256_loadu_si256(ptr.add(96) as *const __m256i); + (row0, row1, row2, row3) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_pack_x3(ptr: *const u8) -> (__m256i, __m256i, __m256i) { - let row0 = _mm256_loadu_si256(ptr as *const __m256i); - let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); - let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); - (row0, row1, row2) + unsafe { + let row0 = _mm256_loadu_si256(ptr as *const __m256i); + let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); + let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); + (row0, row1, row2) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_pack_x2(ptr: *const u8) -> (__m256i, __m256i) { - let row0 = _mm256_loadu_si256(ptr as *const __m256i); - let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); - (row0, row1) + unsafe { + let row0 = _mm256_loadu_si256(ptr as *const __m256i); + let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); + (row0, row1) + } } #[inline(always)] pub(crate) unsafe fn _mm512_load_pack_x2(ptr: *const u8) -> (__m512i, __m512i) { - let row0 = _mm512_loadu_si512(ptr as *const _); - let row1 = _mm512_loadu_si512(ptr.add(64) as *const _); - (row0, row1) + unsafe { + let row0 = _mm512_loadu_si512(ptr as *const _); + let row1 = _mm512_loadu_si512(ptr.add(64) as *const _); + (row0, row1) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_deinterleave_rgb(ptr: *const u8) -> (__m256i, __m256i, __m256i) { - let row0 = _mm256_loadu_si256(ptr as *const __m256i); - let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); - let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); - _mm256_deinterleave_rgb(row0, row1, row2) + unsafe { + let row0 = _mm256_loadu_si256(ptr as *const __m256i); + let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); + let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); + _mm256_deinterleave_rgb(row0, row1, row2) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_deinterleave_rgba( ptr: *const u8, ) -> (__m256i, __m256i, __m256i, __m256i) { - let row0 = _mm256_loadu_si256(ptr as *const __m256i); - let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); - let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); - let row3 = _mm256_loadu_si256(ptr.add(96) as *const __m256i); - _mm256_deinterleave_rgba_epi8(row0, row1, row2, row3) + unsafe { + let row0 = _mm256_loadu_si256(ptr as *const __m256i); + let row1 = _mm256_loadu_si256(ptr.add(32) as *const __m256i); + let row2 = _mm256_loadu_si256(ptr.add(64) as *const __m256i); + let row3 = _mm256_loadu_si256(ptr.add(96) as *const __m256i); + _mm256_deinterleave_rgba_epi8(row0, row1, row2, row3) + } } #[inline(always)] pub(crate) unsafe fn _mm512_load_pack_ps_x4(ptr: *const f32) -> (__m512, __m512, __m512, __m512) { - let row0 = _mm512_loadu_ps(ptr); - let row1 = _mm512_loadu_ps(ptr.add(16)); - let row2 = _mm512_loadu_ps(ptr.add(32)); - let row3 = _mm512_loadu_ps(ptr.add(48)); - (row0, row1, row2, row3) + unsafe { + let row0 = _mm512_loadu_ps(ptr); + let row1 = _mm512_loadu_ps(ptr.add(16)); + let row2 = _mm512_loadu_ps(ptr.add(32)); + let row3 = _mm512_loadu_ps(ptr.add(48)); + (row0, row1, row2, row3) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_pack_ps_x4(ptr: *const f32) -> (__m256, __m256, __m256, __m256) { - let row0 = _mm256_loadu_ps(ptr); - let row1 = _mm256_loadu_ps(ptr.add(8)); - let row2 = _mm256_loadu_ps(ptr.add(16)); - let row3 = _mm256_loadu_ps(ptr.add(24)); - (row0, row1, row2, row3) + unsafe { + let row0 = _mm256_loadu_ps(ptr); + let row1 = _mm256_loadu_ps(ptr.add(8)); + let row2 = _mm256_loadu_ps(ptr.add(16)); + let row3 = _mm256_loadu_ps(ptr.add(24)); + (row0, row1, row2, row3) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_pack_ps_x2(ptr: *const f32) -> (__m256, __m256) { - let row0 = _mm256_loadu_ps(ptr); - let row1 = _mm256_loadu_ps(ptr.add(8)); - (row0, row1) + unsafe { + let row0 = _mm256_loadu_ps(ptr); + let row1 = _mm256_loadu_ps(ptr.add(8)); + (row0, row1) + } } #[inline(always)] pub(crate) unsafe fn _mm512_load_pack_ps_x2(ptr: *const f32) -> (__m512, __m512) { - let row0 = _mm512_loadu_ps(ptr); - let row1 = _mm512_loadu_ps(ptr.add(16)); - (row0, row1) + unsafe { + let row0 = _mm512_loadu_ps(ptr); + let row1 = _mm512_loadu_ps(ptr.add(16)); + (row0, row1) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_deinterleave_rgb_ps(ptr: *const f32) -> (__m256, __m256, __m256) { - let row0 = _mm256_loadu_ps(ptr); - let row1 = _mm256_loadu_ps(ptr.add(8)); - let row2 = _mm256_loadu_ps(ptr.add(16)); - _mm256_deinterleave_rgb_ps((row0, row1, row2)) + unsafe { + let row0 = _mm256_loadu_ps(ptr); + let row1 = _mm256_loadu_ps(ptr.add(8)); + let row2 = _mm256_loadu_ps(ptr.add(16)); + _mm256_deinterleave_rgb_ps((row0, row1, row2)) + } } #[inline(always)] pub(crate) unsafe fn _mm256_load_deinterleave_rgba_ps( ptr: *const f32, ) -> (__m256, __m256, __m256, __m256) { - let row0 = _mm256_loadu_ps(ptr); - let row1 = _mm256_loadu_ps(ptr.add(8)); - let row2 = _mm256_loadu_ps(ptr.add(16)); - let row3 = _mm256_loadu_ps(ptr.add(24)); - _mm256_deinterleave_rgba_ps((row0, row1, row2, row3)) + unsafe { + let row0 = _mm256_loadu_ps(ptr); + let row1 = _mm256_loadu_ps(ptr.add(8)); + let row2 = _mm256_loadu_ps(ptr.add(16)); + let row3 = _mm256_loadu_ps(ptr.add(24)); + _mm256_deinterleave_rgba_ps((row0, row1, row2, row3)) + } } diff --git a/src/filter1d/avx512/v_store.rs b/src/filter1d/avx512/v_store.rs index a6d7390e..aa5ebcd0 100644 --- a/src/filter1d/avx512/v_store.rs +++ b/src/filter1d/avx512/v_store.rs @@ -32,42 +32,54 @@ use std::arch::x86_64::*; #[inline(always)] pub(crate) unsafe fn _mm256_store_pack_x4(ptr: *mut u8, v: (__m256i, __m256i, __m256i, __m256i)) { - _mm256_storeu_si256(ptr as *mut __m256i, v.0); - _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); - _mm256_storeu_si256(ptr.add(64) as *mut __m256i, v.2); - _mm256_storeu_si256(ptr.add(96) as *mut __m256i, v.3); + unsafe { + _mm256_storeu_si256(ptr as *mut __m256i, v.0); + _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); + _mm256_storeu_si256(ptr.add(64) as *mut __m256i, v.2); + _mm256_storeu_si256(ptr.add(96) as *mut __m256i, v.3); + } } #[inline(always)] pub(crate) unsafe fn _mm256_store_pack_x2(ptr: *mut u8, v: (__m256i, __m256i)) { - _mm256_storeu_si256(ptr as *mut __m256i, v.0); - _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); + unsafe { + _mm256_storeu_si256(ptr as *mut __m256i, v.0); + _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); + } } #[inline(always)] pub(crate) unsafe fn _mm512_store_pack_x2(ptr: *mut u8, v: (__m512i, __m512i)) { - _mm512_storeu_si512(ptr as *mut __m512i, v.0); - _mm512_storeu_si512(ptr.add(64) as *mut __m512i, v.1); + unsafe { + _mm512_storeu_si512(ptr as *mut __m512i, v.0); + _mm512_storeu_si512(ptr.add(64) as *mut __m512i, v.1); + } } #[inline(always)] pub(crate) unsafe fn _mm256_store_pack_x3(ptr: *mut u8, v: (__m256i, __m256i, __m256i)) { - _mm256_storeu_si256(ptr as *mut __m256i, v.0); - _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); - _mm256_storeu_si256(ptr.add(64) as *mut __m256i, v.2); + unsafe { + _mm256_storeu_si256(ptr as *mut __m256i, v.0); + _mm256_storeu_si256(ptr.add(32) as *mut __m256i, v.1); + _mm256_storeu_si256(ptr.add(64) as *mut __m256i, v.2); + } } #[inline(always)] pub(crate) unsafe fn _mm256_store_pack_x3_ps(ptr: *mut f32, v: (__m256, __m256, __m256)) { - _mm256_storeu_ps(ptr, v.0); - _mm256_storeu_ps(ptr.add(8), v.1); - _mm256_storeu_ps(ptr.add(16), v.2); + unsafe { + _mm256_storeu_ps(ptr, v.0); + _mm256_storeu_ps(ptr.add(8), v.1); + _mm256_storeu_ps(ptr.add(16), v.2); + } } #[inline(always)] pub(crate) unsafe fn _mm256_store_interleave_rgb(ptr: *mut u8, v: (__m256i, __m256i, __m256i)) { - let (v0, v1, v2) = _mm256_interleave_rgb(v.0, v.1, v.2); - _mm256_store_pack_x3(ptr, (v0, v1, v2)); + unsafe { + let (v0, v1, v2) = _mm256_interleave_rgb(v.0, v.1, v.2); + _mm256_store_pack_x3(ptr, (v0, v1, v2)); + } } #[inline(always)] @@ -75,42 +87,54 @@ pub(crate) unsafe fn _mm256_store_interleave_rgba( ptr: *mut u8, v: (__m256i, __m256i, __m256i, __m256i), ) { - let (v0, v1, v2, v3) = _mm256_interleave_rgba_epi8((v.0, v.1, v.2, v.3)); - _mm256_store_pack_x4(ptr, (v0, v1, v2, v3)); + unsafe { + let (v0, v1, v2, v3) = _mm256_interleave_rgba_epi8((v.0, v.1, v.2, v.3)); + _mm256_store_pack_x4(ptr, (v0, v1, v2, v3)); + } } #[inline(always)] pub(crate) unsafe fn _mm256_store_pack_ps_x4(ptr: *mut f32, v: (__m256, __m256, __m256, __m256)) { - _mm256_storeu_ps(ptr, v.0); - _mm256_storeu_ps(ptr.add(8), v.1); - _mm256_storeu_ps(ptr.add(16), v.2); - _mm256_storeu_ps(ptr.add(24), v.3); + unsafe { + _mm256_storeu_ps(ptr, v.0); + _mm256_storeu_ps(ptr.add(8), v.1); + _mm256_storeu_ps(ptr.add(16), v.2); + _mm256_storeu_ps(ptr.add(24), v.3); + } } #[inline(always)] pub(crate) unsafe fn _mm512_store_pack_ps_x4(ptr: *mut f32, v: (__m512, __m512, __m512, __m512)) { - _mm512_storeu_ps(ptr, v.0); - _mm512_storeu_ps(ptr.add(16), v.1); - _mm512_storeu_ps(ptr.add(32), v.2); - _mm512_storeu_ps(ptr.add(48), v.3); + unsafe { + _mm512_storeu_ps(ptr, v.0); + _mm512_storeu_ps(ptr.add(16), v.1); + _mm512_storeu_ps(ptr.add(32), v.2); + _mm512_storeu_ps(ptr.add(48), v.3); + } } #[inline(always)] pub(crate) unsafe fn _mm256_store_pack_ps_x2(ptr: *mut f32, v: (__m256, __m256)) { - _mm256_storeu_ps(ptr, v.0); - _mm256_storeu_ps(ptr.add(8), v.1); + unsafe { + _mm256_storeu_ps(ptr, v.0); + _mm256_storeu_ps(ptr.add(8), v.1); + } } #[inline(always)] pub(crate) unsafe fn _mm512_store_pack_ps_x2(ptr: *mut f32, v: (__m512, __m512)) { - _mm512_storeu_ps(ptr, v.0); - _mm512_storeu_ps(ptr.add(16), v.1); + unsafe { + _mm512_storeu_ps(ptr, v.0); + _mm512_storeu_ps(ptr.add(16), v.1); + } } #[inline(always)] pub(crate) unsafe fn _mm256_store_interleave_rgb_ps(ptr: *mut f32, v: (__m256, __m256, __m256)) { - let (v0, v1, v2) = _mm256_interleave_rgb_ps(v); - _mm256_store_pack_x3_ps(ptr, (v0, v1, v2)); + unsafe { + let (v0, v1, v2) = _mm256_interleave_rgb_ps(v); + _mm256_store_pack_x3_ps(ptr, (v0, v1, v2)); + } } #[inline(always)] @@ -118,6 +142,8 @@ pub(crate) unsafe fn _mm256_store_interleave_rgba_ps( ptr: *mut f32, v: (__m256, __m256, __m256, __m256), ) { - let (v0, v1, v2, v3) = _mm256_interleave_rgba_ps(v); - _mm256_store_pack_ps_x4(ptr, (v0, v1, v2, v3)); + unsafe { + let (v0, v1, v2, v3) = _mm256_interleave_rgba_ps(v); + _mm256_store_pack_ps_x4(ptr, (v0, v1, v2, v3)); + } } diff --git a/src/filter1d/filter.rs b/src/filter1d/filter.rs index 7268095d..614e7126 100644 --- a/src/filter1d/filter.rs +++ b/src/filter1d/filter.rs @@ -27,7 +27,7 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::edge_mode::clamp_edge; -use crate::filter1d::arena::{make_arena_columns, make_arena_row, write_arena_row, Arena}; +use crate::filter1d::arena::{Arena, make_arena_columns, make_arena_row, write_arena_row}; use crate::filter1d::filter_1d_column_handler::{ Filter1DColumnHandler, Filter1DColumnHandlerMultipleRows, }; diff --git a/src/filter1d/filter_1d_approx.rs b/src/filter1d/filter_1d_approx.rs index cf64962a..8b709102 100644 --- a/src/filter1d/filter_1d_approx.rs +++ b/src/filter1d/filter_1d_approx.rs @@ -27,23 +27,20 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #![forbid(unsafe_code)] -use crate::edge_mode::{clamp_edge, BorderHandle}; -use crate::filter1d::arena::{make_arena_columns, make_arena_row, write_arena_row, Arena}; +use crate::edge_mode::{BorderHandle, clamp_edge}; +use crate::filter1d::arena::{Arena, make_arena_columns, make_arena_row, write_arena_row}; use crate::filter1d::filter::create_brows; -use crate::filter1d::filter_1d_column_handler_approx::{ - Filter1DColumnHandlerApprox, Filter1DColumnMultipleRowsApprox, -}; +use crate::filter1d::filter_1d_column_handler_approx::BuildColumnHandlerApprox; use crate::filter1d::filter_1d_row_handler_approx::Filter1DRowHandlerApprox; use crate::filter1d::filter_element::KernelShape; use crate::filter1d::filter_scan::{is_symmetric_1d, scan_se_1d}; use crate::filter1d::region::FilterRegion; -use crate::filter1d::row_handler_small_approx::{ - Filter1DRowHandlerBInterpolateApr, RowsHolder, RowsHolderMut, -}; +use crate::filter1d::row_handler_small_approx::{BuildRowHandlerBInter, RowsHolder, RowsHolderMut}; use crate::filter1d::to_approx_storage::{ApproxLevel, ToApproxStorage}; use crate::primitives::PrimitiveCast; use crate::safe_math::{SafeAdd, SafeMul}; use crate::to_storage::ToStorage; +use crate::util::{CowScratch, ScratchBuffer}; use crate::{BlurError, BlurImage, BlurImageMut, EdgeMode, EdgeMode2D, Scalar, ThreadingPolicy}; use novtb::{ParallelZonedIterator, TbSliceMut}; use num_traits::{Float, MulAdd}; @@ -86,9 +83,8 @@ where + Send + Sync + Filter1DRowHandlerApprox - + Filter1DColumnHandlerApprox - + Filter1DColumnMultipleRowsApprox - + Filter1DRowHandlerBInterpolateApr + + BuildColumnHandlerApprox + + BuildRowHandlerBInter + Debug, F: ToStorage + Mul + MulAdd + Send + Sync + PrimitiveCast + Float, I: Copy @@ -214,7 +210,7 @@ where let bottom_pad = column_arena_k.bottom_pad.as_slice(); let pad_h = column_kernel_shape.height / 2; - let column_handler = T::get_column_handler(is_column_kernel_symmetric); + let column_handler = T::build_column_handler(is_column_kernel_symmetric, &scaled_column_kernel); let src_stride = image_size.width * N; let dst_stride = destination.row_stride() as usize; @@ -223,9 +219,7 @@ where let mut _processed_y = 0usize; #[cfg(all(target_arch = "aarch64", feature = "neon"))] - if let Some(handler) = - T::get_column_multiple_rows(is_column_kernel_symmetric, &scanned_column_kernel) - { + if column_handler.has_multiple_rows() { _dest_slice .tb_par_chunks_exact_mut(dst_stride * 3) .for_each_enumerated(&pool, |y, row| { @@ -272,13 +266,12 @@ where let brows = FilterBrows { brows: brows_vec }; - handler( + column_handler.multiple_rows( Arena::new(image_size.width, pad_h, 0, pad_h, N), brows, row, image_size, dst_stride, - scanned_column_kernel_slice, ); }); _processed_y = _dest_slice.chunks_exact_mut(dst_stride * 3).len() * 3; @@ -288,9 +281,7 @@ where } #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] - if let Some(handler) = - T::get_column_multiple_rows(is_column_kernel_symmetric, &scanned_column_kernel) - { + if column_handler.has_multiple_rows() { _dest_slice .tb_par_chunks_exact_mut(dst_stride * 2) .for_each_enumerated(&pool, |y, row| { @@ -325,13 +316,12 @@ where let brows = FilterBrows { brows: brows_vec }; - handler( + column_handler.multiple_rows( Arena::new(image_size.width, pad_h, 0, pad_h, N), brows, row, image_size, dst_stride, - scanned_column_kernel_slice, ); }); _processed_y = _dest_slice.chunks_exact_mut(dst_stride * 2).len() * 2; @@ -357,13 +347,12 @@ where let brows_slice = brows.as_slice(); let row = &mut row[..image_size.width * N]; - column_handler( + column_handler.single_row( Arena::new(image_size.width, pad_h, 0, pad_h, N), brows_slice, row, image_size, FilterRegion::new(y, y + 1), - scanned_column_kernel_slice, ); }); @@ -386,9 +375,8 @@ where + Send + Sync + Filter1DRowHandlerApprox - + Filter1DColumnHandlerApprox - + Filter1DColumnMultipleRowsApprox - + Filter1DRowHandlerBInterpolateApr + + BuildColumnHandlerApprox + + BuildRowHandlerBInter + Debug, F: ToStorage + Mul + MulAdd + Send + Sync + PrimitiveCast + Float, I: Copy @@ -408,6 +396,7 @@ where i64: PrimitiveCast + PrimitiveCast, f64: PrimitiveCast, { + const MAX_STACK_KERNEL: usize = 35; image.check_layout_channels(N)?; destination.check_layout_channels(N, Some(image))?; image.only_size_matches_mut(destination)?; @@ -442,8 +431,6 @@ where let scanned_row_kernel = scan_se_1d::(&scaled_row_kernel); let scanned_row_kernel_slice = scanned_row_kernel.as_slice(); - let scanned_column_kernel = scan_se_1d::(&scaled_column_kernel); - let scanned_column_kernel_slice = scanned_column_kernel.as_slice(); let is_column_kernel_symmetric = is_symmetric_1d(column_kernel); let is_row_kernel_symmetric = is_symmetric_1d(row_kernel); @@ -456,9 +443,9 @@ where let tile_size = (image_size.height as u32 / thread_count).clamp(1, image_size.height as u32); let row_handler_binter = - T::get_row_handler_binter_apr::(is_row_kernel_symmetric, &scanned_row_kernel); + T::build_row_handler_binter::(is_row_kernel_symmetric, &scaled_row_kernel); let row_handler = T::get_row_handler_apr::(is_row_kernel_symmetric); - let column_handler = T::get_column_handler(is_column_kernel_symmetric); + let column_handler = T::build_column_handler(is_column_kernel_symmetric, &scaled_column_kernel); let row_stride = image_size.width * N; let src_row_stride = image.row_stride() as usize; @@ -472,42 +459,39 @@ where .tb_par_chunks_mut(dest_stride * tile_size as usize) .for_each_enumerated(&pool, |cy, dst_rows| { let source_y = cy * tile_size as usize; - let mut buffer = vec![T::default(); row_stride * scanned_column_kernel.len()]; + let mut scratch_buffer = + ScratchBuffer::::new(row_stride * scaled_column_kernel.len()); + let buffer = scratch_buffer.as_mut_slice(); - let mut row_buffer: Vec = vec![]; + let mut row_buffer: CowScratch = CowScratch::new(); - let column_kernel_len = scanned_column_kernel.len(); + let column_kernel_len = scaled_column_kernel.len(); let half_kernel = column_kernel_len / 2; // preload top edge if source_y == 0 { - #[allow(clippy::unnecessary_unwrap)] - if row_kernel.len() < B_INTER_CUTOFF - && row_handler_binter.is_some() + if let Some(handler) = row_handler_binter.as_ref() + && row_kernel.len() < B_INTER_CUTOFF && edge_modes.horizontal != EdgeMode::Constant { - let handler = row_handler_binter.unwrap(); - handler( + handler.handle_row( BorderHandle { edge_mode: edge_modes.horizontal, scalar: border_constant, }, &RowsHolder { - holder: vec![&image.data.as_ref()[..src_row_stride]], + holder: [&image.data.as_ref()[..src_row_stride]], }, &mut RowsHolderMut { - holder: vec![&mut buffer[..row_stride]], + holder: [&mut buffer[..row_stride]], }, image_size, - scanned_row_kernel_slice, ); } else { let pad_w = scanned_row_kernel.len() / 2; - if row_buffer.is_empty() { - row_buffer = vec![T::default(); image_size.width * N + pad_w * 2 * N]; - } + let row = row_buffer.get_or_init(image_size.width * N + pad_w * 2 * N); write_arena_row::( - &mut row_buffer, + row, image, 0, KernelShape::new(row_kernel.len(), 0), @@ -517,7 +501,7 @@ where .unwrap(); row_handler( Arena::new(image_size.width, 1, row_kernel.len() / 2, 0, N), - &row_buffer, + row, &mut buffer[..row_stride], image_size, FilterRegion::new(0, 1), @@ -539,39 +523,31 @@ where 0i64, image_size.height as i64 ); - #[allow(clippy::unnecessary_unwrap)] - if row_kernel.len() < B_INTER_CUTOFF - && row_handler_binter.is_some() + if let Some(handler) = row_handler_binter.as_ref() + && row_kernel.len() < B_INTER_CUTOFF && edge_modes.horizontal != EdgeMode::Constant { - let handler = row_handler_binter.unwrap(); - handler( + handler.handle_row( BorderHandle { edge_mode: edge_modes.horizontal, scalar: border_constant, }, &RowsHolder { - holder: vec![ - &image.data.as_ref() - [s_y * src_row_stride..(s_y + 1) * src_row_stride], - ], + holder: [&image.data.as_ref() + [s_y * src_row_stride..(s_y + 1) * src_row_stride]], }, &mut RowsHolderMut { - holder: vec![ - &mut buffer[src_y * row_stride..(src_y + 1) * row_stride], + holder: [ + &mut buffer[src_y * row_stride..(src_y + 1) * row_stride] ], }, image_size, - scanned_row_kernel_slice, ); } else { let pad_w = scanned_row_kernel.len() / 2; - if row_buffer.is_empty() { - row_buffer = - vec![T::default(); image_size.width * N + pad_w * 2 * N]; - } + let row = row_buffer.get_or_init(image_size.width * N + pad_w * 2 * N); write_arena_row::( - &mut row_buffer, + row, image, s_y, KernelShape::new(row_kernel.len(), 0), @@ -581,7 +557,7 @@ where .unwrap(); row_handler( Arena::new(image_size.width, 1, row_kernel.len() / 2, 0, N), - &row_buffer, + row, &mut buffer[src_y * row_stride..(src_y + 1) * row_stride], image_size, FilterRegion::new(0, 1), @@ -611,34 +587,30 @@ where ) }; - #[allow(clippy::unnecessary_unwrap)] - if row_kernel.len() < B_INTER_CUTOFF - && row_handler_binter.is_some() + if let Some(handler) = row_handler_binter.as_ref() + && row_kernel.len() < B_INTER_CUTOFF && edge_modes.horizontal != EdgeMode::Constant { - let handler = row_handler_binter.unwrap(); - handler( + handler.handle_row( BorderHandle { edge_mode: edge_modes.horizontal, scalar: border_constant, }, &RowsHolder { - holder: vec![ - &image.data.as_ref() - [new_y * src_row_stride..(new_y + 1) * src_row_stride], - ], + holder: [&image.data.as_ref() + [new_y * src_row_stride..(new_y + 1) * src_row_stride]], }, &mut RowsHolderMut { - holder: vec![ - &mut buffer[start_ky * row_stride..(start_ky + 1) * row_stride], + holder: [ + &mut buffer[start_ky * row_stride..(start_ky + 1) * row_stride] ], }, image_size, - scanned_row_kernel_slice, ); } else { + let row = row_buffer.get_or_unwrap_mut(); write_arena_row::( - &mut row_buffer, + row, image, new_y, KernelShape::new(row_kernel.len(), 0), @@ -648,7 +620,7 @@ where .unwrap(); row_handler( Arena::new(image_size.width, 1, row_kernel.len() / 2, 0, N), - &row_buffer, + row, &mut buffer[start_ky * row_stride..(start_ky + 1) * row_stride], image_size, FilterRegion::new(0, 1), @@ -657,9 +629,10 @@ where } if dy >= half_kernel { - let mut brows = vec![image.data.as_ref(); column_kernel_len]; - - for (i, brow) in brows.iter_mut().enumerate() { + let mut brows_stack = + ScratchBuffer::<&[T], MAX_STACK_KERNEL>::new(column_kernel_len); + let brows_storage = brows_stack.as_mut_slice(); + for (i, brow) in brows_storage.iter_mut().enumerate() { let ky = (i + start_ky + 1) % column_kernel_len; *brow = &buffer[ky * row_stride..(ky + 1) * row_stride]; } @@ -668,13 +641,12 @@ where let dst = &mut dst_rows[dy * dest_stride..(dy + 1) * dest_stride]; - column_handler( + column_handler.single_row( Arena::new(image_size.width, half_kernel, 0, half_kernel, N), - &brows, + brows_storage, dst, image_size, FilterRegion::new(0, 1), - scanned_column_kernel_slice, ); } @@ -683,38 +655,35 @@ where } }); } else { - let mut buffer = vec![T::default(); row_stride * scanned_column_kernel.len()]; + let mut scratch_buffer = + ScratchBuffer::::new(row_stride * scaled_column_kernel.len()); + let buffer = scratch_buffer.as_mut_slice(); - let mut row_buffer: Vec = vec![]; + let mut row_buffer: CowScratch = CowScratch::new(); - #[allow(clippy::unnecessary_unwrap)] // preload top edge - if row_kernel.len() < B_INTER_CUTOFF - && row_handler_binter.is_some() + if let Some(handler) = row_handler_binter.as_ref() + && row_kernel.len() < B_INTER_CUTOFF && edge_modes.horizontal != EdgeMode::Constant { - let handler = row_handler_binter.unwrap(); - handler( + handler.handle_row( BorderHandle { edge_mode: edge_modes.horizontal, scalar: border_constant, }, &RowsHolder { - holder: vec![&image.data.as_ref()[..src_row_stride]], + holder: [&image.data.as_ref()[..src_row_stride]], }, &mut RowsHolderMut { - holder: vec![&mut buffer[..row_stride]], + holder: [&mut buffer[..row_stride]], }, image_size, - scanned_row_kernel_slice, ); } else { let pad_w = scanned_row_kernel.len() / 2; - if row_buffer.is_empty() { - row_buffer = vec![T::default(); image_size.width * N + pad_w * 2 * N]; - } + let row = row_buffer.get_or_init(image_size.width * N + pad_w * 2 * N); write_arena_row::( - &mut row_buffer, + row, image, 0, KernelShape::new(row_kernel.len(), 0), @@ -723,7 +692,7 @@ where )?; row_handler( Arena::new(image_size.width, 1, row_kernel.len() / 2, 0, N), - &row_buffer, + row, &mut buffer[..row_stride], image_size, FilterRegion::new(0, 1), @@ -731,7 +700,7 @@ where ); } - let column_kernel_len = scanned_column_kernel.len(); + let column_kernel_len = scaled_column_kernel.len(); let half_kernel = column_kernel_len / 2; @@ -758,34 +727,28 @@ where ) }; - #[allow(clippy::unnecessary_unwrap)] - if row_kernel.len() < B_INTER_CUTOFF - && row_handler_binter.is_some() + if let Some(handler) = row_handler_binter.as_ref() + && row_kernel.len() < B_INTER_CUTOFF && edge_modes.horizontal != EdgeMode::Constant { - let handler = row_handler_binter.unwrap(); - handler( + handler.handle_row( BorderHandle { edge_mode: edge_modes.horizontal, scalar: border_constant, }, &RowsHolder { - holder: vec![ - &image.data.as_ref() - [new_y * src_row_stride..(new_y + 1) * src_row_stride], - ], + holder: [&image.data.as_ref() + [new_y * src_row_stride..(new_y + 1) * src_row_stride]], }, &mut RowsHolderMut { - holder: vec![ - &mut buffer[start_ky * row_stride..(start_ky + 1) * row_stride], - ], + holder: [&mut buffer[start_ky * row_stride..(start_ky + 1) * row_stride]], }, image_size, - scanned_row_kernel_slice, ); } else { + let row = row_buffer.get_or_unwrap_mut(); write_arena_row::( - &mut row_buffer, + row, image, new_y, KernelShape::new(row_kernel.len(), 0), @@ -794,7 +757,7 @@ where )?; row_handler( Arena::new(image_size.width, 1, row_kernel.len() / 2, 0, N), - &row_buffer, + row, &mut buffer[start_ky * row_stride..(start_ky + 1) * row_stride], image_size, FilterRegion::new(0, 1), @@ -803,9 +766,10 @@ where } if y >= half_kernel { - let mut brows = vec![image.data.as_ref(); column_kernel_len]; - - for (i, brow) in brows.iter_mut().enumerate() { + let mut brows_stack = + ScratchBuffer::<&[T], MAX_STACK_KERNEL>::new(column_kernel_len); + let brows_storage = brows_stack.as_mut_slice(); + for (i, brow) in brows_storage.iter_mut().enumerate() { let ky = (i + start_ky + 1) % column_kernel_len; *brow = &buffer[ky * row_stride..(ky + 1) * row_stride]; } @@ -815,13 +779,12 @@ where let dst = &mut destination.data.borrow_mut()[dy * dest_stride..(dy + 1) * dest_stride]; - column_handler( + column_handler.single_row( Arena::new(image_size.width, half_kernel, 0, half_kernel, N), - &brows, + brows_storage, dst, image_size, FilterRegion::new(0, 1), - scanned_column_kernel_slice, ); } diff --git a/src/filter1d/filter_1d_column_handler.rs b/src/filter1d/filter_1d_column_handler.rs index a4c876df..bee6f141 100644 --- a/src/filter1d/filter_1d_column_handler.rs +++ b/src/filter1d/filter_1d_column_handler.rs @@ -26,6 +26,7 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; use crate::filter1d::arena::Arena; #[cfg(all(target_arch = "x86_64", feature = "avx"))] use crate::filter1d::avx::{ @@ -45,7 +46,6 @@ use crate::filter1d::sse::{ filter_column_sse_f32_f32, filter_column_sse_u8_f32, filter_column_sse_u8_i16, filter_column_symm_sse_u8_f32, filter_column_symm_sse_u8_i16, }; -use crate::ImageSize; #[cfg(feature = "nightly_f16")] use core::f16; diff --git a/src/filter1d/filter_1d_column_handler_approx.rs b/src/filter1d/filter_1d_column_handler_approx.rs index 11589f8d..a8e7ec83 100644 --- a/src/filter1d/filter_1d_column_handler_approx.rs +++ b/src/filter1d/filter_1d_column_handler_approx.rs @@ -26,277 +26,388 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; use crate::filter1d::arena::Arena; -#[cfg(all(target_arch = "x86_64", feature = "avx"))] -use crate::filter1d::avx::{filter_column_avx_symm_u8_i32_app, filter_column_avx_u8_i32_app}; use crate::filter1d::filter_1d_column_handler::FilterBrows; use crate::filter1d::filter_column_approx::filter_column_approx; use crate::filter1d::filter_column_approx_symmetric::filter_column_symmetric_approx; -use crate::filter1d::filter_scan::ScanPoint1d; -#[cfg(all(target_arch = "aarch64", feature = "neon"))] -use crate::filter1d::neon::{filter_column_neon_u8_i32_app, filter_column_symm_neon_u8_i32_app}; use crate::filter1d::region::FilterRegion; -#[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] -use crate::filter1d::sse::{filter_column_sse_u8_i32_app, filter_column_symm_u8_i32_app}; -use crate::ImageSize; +use std::sync::Arc; -pub trait Filter1DColumnHandlerApprox { - fn get_column_handler( - is_kernel_symmetric: bool, - ) -> fn( +pub trait ResolveColumnHandlerApprox: Send + Sync { + fn single_row( + &self, arena: Arena, - &[&[T]], + rows: &[&[T]], dst: &mut [T], image_size: ImageSize, - FilterRegion, - scanned_kernel: &[ScanPoint1d], + region: FilterRegion, ); -} -pub trait Filter1DColumnMultipleRowsApprox { - fn get_column_multiple_rows( - is_symmetric_kernel: bool, - kernel: &[ScanPoint1d], - ) -> Option< - fn( - arena: Arena, - FilterBrows, - dst: &mut [T], - image_size: ImageSize, - dst_stride: usize, - scanned_kernel: &[ScanPoint1d], - ), - >; + fn multiple_rows( + &self, + arena: Arena, + rows: FilterBrows, + dst: &mut [T], + image_size: ImageSize, + dst_stride: usize, + ) -> bool; // returns false if no multi-row impl available + fn has_multiple_rows(&self) -> bool; } -macro_rules! default_1d_column_handler { - ($store:ty, $intermediate:ty) => { - impl Filter1DColumnHandlerApprox<$store, $intermediate> for $store { - fn get_column_handler( - is_kernel_symmetric: bool, - ) -> fn( - Arena, - &[&[$store]], - &mut [$store], - ImageSize, - FilterRegion, - &[ScanPoint1d<$intermediate>], - ) { - if is_kernel_symmetric { - filter_column_symmetric_approx - } else { - filter_column_approx - } - } +#[allow(unused)] +pub(crate) fn make_q07_weights(weights: &[i32]) -> Vec { + let mut shifted = weights + .iter() + .map(|&x| (x >> 8).min(i8::MAX as i32) as i8) + .collect::>(); + let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); + if sum > 128 { + let half = shifted.len() / 2; + while sum > 128 { + shifted[half] = shifted[half].saturating_sub(1); + sum -= 1; } - }; + } else if sum < 128 { + let half = shifted.len() / 2; + while sum < 128 { + shifted[half] = shifted[half].saturating_add(1); + sum += 1; + } + } + shifted } -impl Filter1DColumnHandlerApprox for u16 { - #[cfg(all(target_arch = "aarch64", feature = "neon"))] - fn get_column_handler( - is_kernel_symmetric: bool, - ) -> fn(Arena, &[&[u16]], &mut [u16], ImageSize, FilterRegion, &[ScanPoint1d]) { - if is_kernel_symmetric { - use crate::filter1d::neon::filter_column_symm_neon_uq15_u16; - filter_column_symm_neon_uq15_u16 - } else { - filter_column_approx +#[cfg(all(target_arch = "aarch64", feature = "neon"))] +#[allow(unused)] +pub(crate) fn make_uq07_weights(weights: &[i32]) -> Vec { + let mut shifted = weights + .iter() + .map(|&x| (x >> 8).min(i8::MAX as i32) as u8) + .collect::>(); + let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); + if sum > 128 { + let half = shifted.len() / 2; + while sum > 128 { + shifted[half] = shifted[half].saturating_sub(1); + sum -= 1; + } + } else if sum < 128 { + let half = shifted.len() / 2; + while sum < 128 { + shifted[half] = shifted[half].saturating_add(1); + sum += 1; } } + shifted +} - #[cfg(not(any( - all(target_arch = "aarch64", feature = "neon"), - any(target_arch = "x86_64", target_arch = "x86") - )))] - fn get_column_handler( - is_kernel_symmetric: bool, - ) -> fn(Arena, &[&[u16]], &mut [u16], ImageSize, FilterRegion, &[ScanPoint1d]) { - if is_kernel_symmetric { - filter_column_symmetric_approx - } else { - filter_column_approx - } +struct ColumnHandlerApprox { + single_row: fn(Arena, &[&[T]], &mut [T], ImageSize, FilterRegion, &[F]), + multiple_rows: Option, &mut [T], ImageSize, usize, &[F])>, + kernel: Vec, +} + +impl ResolveColumnHandlerApprox for ColumnHandlerApprox { + fn single_row( + &self, + arena: Arena, + rows: &[&[T]], + dst: &mut [T], + image_size: ImageSize, + region: FilterRegion, + ) { + (self.single_row)(arena, rows, dst, image_size, region, &self.kernel) } - #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] - fn get_column_handler( - is_kernel_symmetric: bool, - ) -> fn(Arena, &[&[u16]], &mut [u16], ImageSize, FilterRegion, &[ScanPoint1d]) { - #[cfg(all(target_arch = "x86_64", feature = "avx"))] - if std::arch::is_x86_feature_detected!("avx2") && is_kernel_symmetric { - use crate::filter1d::avx::filter_column_avx_symm_uq15_u16; - return filter_column_avx_symm_uq15_u16; - } - #[cfg(feature = "sse")] - { - if std::arch::is_x86_feature_detected!("sse4.1") && is_kernel_symmetric { - use crate::filter1d::sse::filter_column_sse_symm_uq15_u16; - return filter_column_sse_symm_uq15_u16; + fn multiple_rows( + &self, + arena: Arena, + rows: FilterBrows, + dst: &mut [T], + image_size: ImageSize, + dst_stride: usize, + ) -> bool { + match self.multiple_rows { + Some(f) => { + f(arena, rows, dst, image_size, dst_stride, &self.kernel); + true } - } - if is_kernel_symmetric { - filter_column_symmetric_approx - } else { - filter_column_approx + None => false, } } -} -impl Filter1DColumnHandlerApprox for u8 { - #[cfg(not(any( - all(target_arch = "aarch64", feature = "neon"), - any(target_arch = "x86_64", target_arch = "x86") - )))] - fn get_column_handler( - is_kernel_symmetric: bool, - ) -> fn(Arena, &[&[u8]], &mut [u8], ImageSize, FilterRegion, &[ScanPoint1d]) { - if is_kernel_symmetric { - filter_column_symmetric_approx - } else { - filter_column_approx - } + fn has_multiple_rows(&self) -> bool { + self.multiple_rows.is_some() } +} - #[cfg(all(target_arch = "aarch64", feature = "neon"))] - fn get_column_handler( +pub trait BuildColumnHandlerApprox { + fn build_column_handler( is_kernel_symmetric: bool, - ) -> fn(Arena, &[&[u8]], &mut [u8], ImageSize, FilterRegion, &[ScanPoint1d]) { - if is_kernel_symmetric { - #[cfg(feature = "rdm")] - { - if std::arch::is_aarch64_feature_detected!("rdm") { - use crate::filter1d::neon::filter_column_symm_neon_u8_i32_rdm; - return filter_column_symm_neon_u8_i32_rdm; - } - } - filter_column_symm_neon_u8_i32_app - } else { - #[cfg(feature = "rdm")] - { - if std::arch::is_aarch64_feature_detected!("rdm") { - use crate::filter1d::neon::filter_column_neon_u8_i32_i16_qrdm_app; - return filter_column_neon_u8_i32_i16_qrdm_app; - } + kernel: &[F], + ) -> Arc + Send + Sync>; +} + +macro_rules! default_1d_column_handler { + ($store:ty, $intermediate:ty) => { + impl BuildColumnHandlerApprox<$store, $intermediate> for $store { + fn build_column_handler( + is_kernel_symmetric: bool, + kernel: &[$intermediate], + ) -> Arc + Send + Sync> { + Arc::new(ColumnHandlerApprox { + single_row: if is_kernel_symmetric { + filter_column_symmetric_approx + } else { + filter_column_approx + }, + multiple_rows: None, + kernel: kernel.to_vec(), + }) } - filter_column_neon_u8_i32_app } - } + }; +} - #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] - fn get_column_handler( +impl BuildColumnHandlerApprox for u8 { + fn build_column_handler( is_kernel_symmetric: bool, - ) -> fn(Arena, &[&[u8]], &mut [u8], ImageSize, FilterRegion, &[ScanPoint1d]) { - #[cfg(all(target_arch = "x86_64", feature = "nightly_avx512"))] + kernel: &[i32], + ) -> Arc + Send + Sync> { + let _single_row: fn(Arena, &[&[u8]], &mut [u8], ImageSize, FilterRegion, &[i32]); + let _multiple_rows: Option< + fn(Arena, FilterBrows, &mut [u8], ImageSize, usize, &[i32]), + >; + + #[cfg(all(target_arch = "x86_64", feature = "avx512"))] if is_kernel_symmetric && std::arch::is_x86_feature_detected!("avx512bw") { use crate::filter1d::avx512::filter_column_avx512_symm_u8_i32_app; - return filter_column_avx512_symm_u8_i32_app; + let single_row = filter_column_avx512_symm_u8_i32_app; + if is_kernel_symmetric && kernel.len() <= 7 && kernel.iter().all(|&x| x > 0) { + let q07_weights = make_q07_weights(kernel); + use crate::filter1d::avx512::filter_column_avx512_symm_u8_uq0_7; + let v_prepared = q07_weights + .iter() + .map(|&x| { + let z = x.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[0], z[0], z[0]]) + }) + .collect::>(); + return Arc::new(ColumnHandlerApprox { + single_row: filter_column_avx512_symm_u8_uq0_7, + multiple_rows: None, + kernel: v_prepared, + }); + } + let v_packed_i16 = kernel + .iter() + .map(|&x| { + let z = x.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) + }) + .collect::>(); + return Arc::new(ColumnHandlerApprox { + single_row, + multiple_rows: None, + kernel: v_packed_i16, + }); } + #[cfg(all(target_arch = "x86_64", feature = "avx"))] if std::arch::is_x86_feature_detected!("avx2") { - if is_kernel_symmetric { - return filter_column_avx_symm_u8_i32_app; + use crate::filter1d::avx::{ + filter_column_avx_symm_u8_i32_app, filter_column_avx_symm_u8_i32_app_x2, + filter_column_avx_u8_i32_app, + }; + if is_kernel_symmetric && kernel.len() <= 7 && kernel.iter().all(|&x| x > 0) { + let q07_weights = make_q07_weights(kernel); + let v_prepared = q07_weights + .iter() + .map(|&x| { + let z = x.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[0], z[0], z[0]]) + }) + .collect::>(); + use crate::filter1d::avx::filter_column_avx_symm_u8_uq0_7; + return Arc::new(ColumnHandlerApprox { + single_row: filter_column_avx_symm_u8_uq0_7, + multiple_rows: None, + kernel: v_prepared, + }); } - return filter_column_avx_u8_i32_app; - } - #[cfg(feature = "sse")] - if std::arch::is_x86_feature_detected!("sse4.1") { + let v_packed_i16 = kernel + .iter() + .map(|&x| { + let z = x.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) + }) + .collect::>(); if is_kernel_symmetric { - return filter_column_symm_u8_i32_app; + _single_row = filter_column_avx_symm_u8_i32_app; + _multiple_rows = Some(filter_column_avx_symm_u8_i32_app_x2); + } else { + _single_row = filter_column_avx_u8_i32_app; + _multiple_rows = None; } - return filter_column_sse_u8_i32_app; - } - if is_kernel_symmetric { - filter_column_symmetric_approx - } else { - filter_column_approx + return Arc::new(ColumnHandlerApprox { + single_row: _single_row, + multiple_rows: _multiple_rows, + kernel: v_packed_i16, + }); } - } -} -impl Filter1DColumnMultipleRowsApprox for u8 { - #[cfg(all(target_arch = "aarch64", feature = "neon"))] - fn get_column_multiple_rows( - is_symmetric_kernel: bool, - _: &[ScanPoint1d], - ) -> Option, &mut [u8], ImageSize, usize, &[ScanPoint1d])> { - if is_symmetric_kernel { - #[cfg(feature = "rdm")] - { - if std::arch::is_aarch64_feature_detected!("rdm") { - use crate::filter1d::neon::filter_column_symm_neon_u8_i32_rdm_x3; - return Some(filter_column_symm_neon_u8_i32_rdm_x3); - } + #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] + if std::arch::is_x86_feature_detected!("sse4.1") { + use crate::filter1d::sse::{ + filter_column_sse_symm_u8_uq0_7, filter_column_sse_u8_i32_app, + filter_column_symm_u8_i32_app, + }; + if is_kernel_symmetric && kernel.len() <= 7 && kernel.iter().all(|&x| x > 0) { + let q07_weights = make_q07_weights(kernel); + let v_prepared = q07_weights + .iter() + .map(|&x| { + let z = x.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[0], z[0], z[0]]) + }) + .collect::>(); + return Arc::new(ColumnHandlerApprox { + single_row: filter_column_sse_symm_u8_uq0_7, + multiple_rows: None, + kernel: v_prepared, + }); } + let v_packed_i16 = kernel + .iter() + .map(|&x| { + let z = x.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) + }) + .collect::>(); + let single_row = if is_kernel_symmetric { + filter_column_symm_u8_i32_app + } else { + filter_column_sse_u8_i32_app + }; + return Arc::new(ColumnHandlerApprox { + single_row, + multiple_rows: None, + kernel: v_packed_i16, + }); } - None - } - #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] - fn get_column_multiple_rows( - _is_symmetric: bool, - _: &[ScanPoint1d], - ) -> Option, &mut [u8], ImageSize, usize, &[ScanPoint1d])> { - #[cfg(all(target_arch = "x86_64", feature = "avx"))] + #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - if _is_symmetric && std::arch::is_x86_feature_detected!("avx2") { - use crate::filter1d::avx::filter_column_avx_symm_u8_i32_app_x2; - return Some(filter_column_avx_symm_u8_i32_app_x2); + use crate::filter1d::neon::{ + filter_column_neon_u8_i32_app, filter_column_symm_neon_u8_i32_app, + }; + if is_kernel_symmetric { + if kernel.len() <= 7 && kernel.iter().all(|&x| x > 0) { + let q07_weights = make_uq07_weights(kernel); + use crate::filter1d::neon::filter_column_symm_neon_u8_uq0_7; + return Arc::new(ColumnHandlerApprox { + single_row: filter_column_symm_neon_u8_uq0_7, + multiple_rows: None, + kernel: q07_weights, + }); + } + + #[cfg(feature = "rdm")] + if std::arch::is_aarch64_feature_detected!("rdm") { + use crate::filter1d::neon::{ + filter_column_symm_neon_u8_i32_rdm, filter_column_symm_neon_u8_i32_rdm_x3, + }; + return Arc::new(ColumnHandlerApprox { + single_row: filter_column_symm_neon_u8_i32_rdm, + multiple_rows: Some(filter_column_symm_neon_u8_i32_rdm_x3), + kernel: kernel.to_vec(), + }); + } + Arc::new(ColumnHandlerApprox { + single_row: filter_column_symm_neon_u8_i32_app, + multiple_rows: None, + kernel: kernel.to_vec(), + }) + } else { + #[cfg(feature = "rdm")] + if std::arch::is_aarch64_feature_detected!("rdm") { + use crate::filter1d::neon::filter_column_neon_u8_i32_i16_qrdm_app; + return Arc::new(ColumnHandlerApprox { + single_row: filter_column_neon_u8_i32_i16_qrdm_app, + multiple_rows: None, + kernel: kernel.to_vec(), + }); + } + Arc::new(ColumnHandlerApprox { + single_row: filter_column_neon_u8_i32_app, + multiple_rows: None, + kernel: kernel.to_vec(), + }) } } - None - } - #[cfg(not(any( - all(target_arch = "aarch64", feature = "neon"), - any(target_arch = "x86_64", target_arch = "x86") - )))] - fn get_column_multiple_rows( - _: bool, - _: &[ScanPoint1d], - ) -> Option, &mut [u8], ImageSize, usize, &[ScanPoint1d])> { - None + #[cfg(not(all(target_arch = "aarch64", feature = "neon")))] + Arc::new(ColumnHandlerApprox { + single_row: if is_kernel_symmetric { + filter_column_symmetric_approx + } else { + filter_column_approx + }, + multiple_rows: None, + kernel: kernel.to_vec(), + }) } } -impl Filter1DColumnMultipleRowsApprox for u16 { - #[cfg(all(target_arch = "aarch64", feature = "neon"))] - fn get_column_multiple_rows( - is_symmetric_kernel: bool, - _: &[ScanPoint1d], - ) -> Option, &mut [u16], ImageSize, usize, &[ScanPoint1d])> - { - if is_symmetric_kernel { - use crate::filter1d::neon::filter_symm_column_neon_uq15_u16_x3; - return Some(filter_symm_column_neon_uq15_u16_x3); +impl BuildColumnHandlerApprox for u16 { + fn build_column_handler( + is_kernel_symmetric: bool, + kernel: &[u32], + ) -> Arc + Send + Sync> { + #[cfg(all(target_arch = "x86_64", feature = "avx"))] + if std::arch::is_x86_feature_detected!("avx2") && is_kernel_symmetric { + use crate::filter1d::avx::{ + filter_column_avx_symm_uq15_u16, filter_column_avx_symm_uq15_u16_x2, + }; + return Arc::new(ColumnHandlerApprox { + single_row: filter_column_avx_symm_uq15_u16, + multiple_rows: Some(filter_column_avx_symm_uq15_u16_x2), + kernel: kernel.to_vec(), + }); } - None - } - #[cfg(not(any( - all(target_arch = "aarch64", feature = "neon"), - any(target_arch = "x86_64", target_arch = "x86") - )))] - fn get_column_multiple_rows( - _: bool, - _: &[ScanPoint1d], - ) -> Option, &mut [u16], ImageSize, usize, &[ScanPoint1d])> - { - None - } + #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] + if std::arch::is_x86_feature_detected!("sse4.1") && is_kernel_symmetric { + use crate::filter1d::sse::filter_column_sse_symm_uq15_u16; + return Arc::new(ColumnHandlerApprox { + single_row: filter_column_sse_symm_uq15_u16, + multiple_rows: None, + kernel: kernel.to_vec(), + }); + } - #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] - fn get_column_multiple_rows( - _is_kernel_symmetric: bool, - _: &[ScanPoint1d], - ) -> Option, &mut [u16], ImageSize, usize, &[ScanPoint1d])> - { - #[cfg(all(target_arch = "x86_64", feature = "avx"))] - if std::arch::is_x86_feature_detected!("avx2") && _is_kernel_symmetric { - use crate::filter1d::avx::filter_column_avx_symm_uq15_u16_x2; - return Some(filter_column_avx_symm_uq15_u16_x2); + #[cfg(all(target_arch = "aarch64", feature = "neon"))] + if is_kernel_symmetric { + use crate::filter1d::neon::{ + filter_column_symm_neon_uq15_u16, filter_symm_column_neon_uq15_u16_x3, + }; + return Arc::new(ColumnHandlerApprox { + single_row: filter_column_symm_neon_uq15_u16, + multiple_rows: Some(filter_symm_column_neon_uq15_u16_x3), + kernel: kernel.to_vec(), + }); } - None + + Arc::new(ColumnHandlerApprox { + single_row: if is_kernel_symmetric { + filter_column_symmetric_approx + } else { + filter_column_approx + }, + multiple_rows: None, + kernel: kernel.to_vec(), + }) } } @@ -313,39 +424,3 @@ default_1d_column_handler!(u16, i64); default_1d_column_handler!(u16, u64); default_1d_column_handler!(i16, i32); default_1d_column_handler!(i16, i64); - -macro_rules! default_1d_column_multiple_rows { - ($store:ty, $intermediate:ty) => { - impl Filter1DColumnMultipleRowsApprox<$store, $intermediate> for $store { - fn get_column_multiple_rows( - _: bool, - _: &[ScanPoint1d<$intermediate>], - ) -> Option< - fn( - Arena, - FilterBrows<$store>, - &mut [$store], - ImageSize, - usize, - &[ScanPoint1d<$intermediate>], - ), - > { - None - } - } - }; -} - -default_1d_column_multiple_rows!(u8, i64); -default_1d_column_multiple_rows!(u8, u16); -default_1d_column_multiple_rows!(u8, i16); -default_1d_column_multiple_rows!(u8, u32); -default_1d_column_multiple_rows!(u8, u64); -default_1d_column_multiple_rows!(i8, i32); -default_1d_column_multiple_rows!(i8, i64); -default_1d_column_multiple_rows!(i8, i16); -default_1d_column_multiple_rows!(u16, i32); -default_1d_column_multiple_rows!(u16, i64); -default_1d_column_multiple_rows!(u16, u64); -default_1d_column_multiple_rows!(i16, i32); -default_1d_column_multiple_rows!(i16, i64); diff --git a/src/filter1d/filter_1d_row_handler.rs b/src/filter1d/filter_1d_row_handler.rs index 26347852..cdd96d90 100644 --- a/src/filter1d/filter_1d_row_handler.rs +++ b/src/filter1d/filter_1d_row_handler.rs @@ -26,13 +26,13 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; use crate::filter1d::arena::Arena; use crate::filter1d::filter_row_symmetric::filter_row_symmetrical; use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] use crate::filter1d::sse::filter_row_sse_f32_f32; -use crate::ImageSize; #[cfg(feature = "nightly_f16")] use core::f16; diff --git a/src/filter1d/filter_1d_row_handler_approx.rs b/src/filter1d/filter_1d_row_handler_approx.rs index 3b7d8343..79292571 100644 --- a/src/filter1d/filter_1d_row_handler_approx.rs +++ b/src/filter1d/filter_1d_row_handler_approx.rs @@ -26,12 +26,12 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; use crate::filter1d::arena::Arena; use crate::filter1d::filter_row_approx::filter_row_approx; use crate::filter1d::filter_row_symmetric_approx::filter_row_symmetric_approx; use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; -use crate::ImageSize; pub trait Filter1DRowHandlerApprox { #[allow(clippy::type_complexity)] @@ -158,7 +158,7 @@ impl Filter1DRowHandlerApprox for u16 { fn get_row_handler_apr( is_kernel_symmetric: bool, ) -> fn(Arena, &[u16], &mut [u16], ImageSize, FilterRegion, &[ScanPoint1d]) { - #[cfg(all(target_arch = "x86_64", feature = "nightly_avx512"))] + #[cfg(all(target_arch = "x86_64", feature = "avx512"))] if std::arch::is_x86_feature_detected!("avx512bw") && is_kernel_symmetric { use crate::filter1d::avx512::filter_row_avx512_symm_uq15_u16; return filter_row_avx512_symm_uq15_u16::; diff --git a/src/filter1d/filter_column_approx.rs b/src/filter1d/filter_column_approx.rs index b5603ee9..a7aa5c1a 100644 --- a/src/filter1d/filter_column_approx.rs +++ b/src/filter1d/filter_column_approx.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; @@ -40,7 +39,7 @@ pub(crate) fn filter_column_approx( dst: &mut [T], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[I], ) where T: Copy + AsPrimitive + Default, I: Copy @@ -63,28 +62,28 @@ pub(crate) fn filter_column_approx( while cx + 4 < dst_stride { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - let mut k0 = v_src.get_unchecked(0).as_().mul(coeff.weight); - let mut k1 = v_src.get_unchecked(1).as_().mul(coeff.weight); - let mut k2 = v_src.get_unchecked(2).as_().mul(coeff.weight); - let mut k3 = v_src.get_unchecked(3).as_().mul(coeff.weight); + let mut k0 = v_src.get_unchecked(0).as_().mul(coeff); + let mut k1 = v_src.get_unchecked(1).as_().mul(coeff); + let mut k2 = v_src.get_unchecked(2).as_().mul(coeff); + let mut k3 = v_src.get_unchecked(3).as_().mul(coeff); for i in 1..length { let coeff = *scanned_kernel.get_unchecked(i); k0 = (*arena_src.get_unchecked(i).get_unchecked(cx)) .as_() - .mul(coeff.weight) + .mul(coeff) .add(k0); k1 = (*arena_src.get_unchecked(i).get_unchecked(cx + 1)) .as_() - .mul(coeff.weight) + .mul(coeff) .add(k1); k2 = (*arena_src.get_unchecked(i).get_unchecked(cx + 2)) .as_() - .mul(coeff.weight) + .mul(coeff) .add(k2); k3 = (*arena_src.get_unchecked(i).get_unchecked(cx + 3)) .as_() - .mul(coeff.weight) + .mul(coeff) .add(k3); } @@ -98,13 +97,13 @@ pub(crate) fn filter_column_approx( for x in cx..dst_stride { let v_src = arena_src.get_unchecked(0).get_unchecked(x..); - let mut k0 = v_src.get_unchecked(0).as_().mul(coeff.weight); + let mut k0 = v_src.get_unchecked(0).as_().mul(coeff); for i in 1..length { let coeff = *scanned_kernel.get_unchecked(i); k0 = (*arena_src.get_unchecked(i).get_unchecked(x)) .as_() - .mul(coeff.weight) + .mul(coeff) .add(k0); } diff --git a/src/filter1d/filter_column_approx_symmetric.rs b/src/filter1d/filter_column_approx_symmetric.rs index 5eccc610..156532f2 100644 --- a/src/filter1d/filter_column_approx_symmetric.rs +++ b/src/filter1d/filter_column_approx_symmetric.rs @@ -28,7 +28,6 @@ */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; @@ -41,7 +40,7 @@ pub(crate) fn filter_column_symmetric_approx( dst: &mut [T], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[I], ) where T: Copy + AsPrimitive + Default, I: Copy @@ -62,7 +61,7 @@ pub(crate) fn filter_column_symmetric_approx( let ref0 = arena_src[half_len]; - let coeff = scanned_kernel[half_len].weight; + let coeff = scanned_kernel[half_len]; while cx + 32 < dst_stride { let mut store: [I; 32] = [I::default(); 32]; @@ -73,13 +72,13 @@ pub(crate) fn filter_column_symmetric_approx( *dst = src.as_().mul(coeff); } - for (i, coeff) in scanned_kernel.iter().take(half_len).enumerate() { + for (i, &coeff) in scanned_kernel.iter().take(half_len).enumerate() { let rollback = length - i - 1; let fw = &arena_src[i][cx..(cx + 32)]; let bw = &arena_src[rollback][cx..(cx + 32)]; for ((dst, fw), bw) in store.iter_mut().zip(fw).zip(bw) { - *dst = *dst + fw.as_().add(bw.as_()).mul(coeff.weight); + *dst = *dst + fw.as_().add(bw.as_()).mul(coeff); } } @@ -99,13 +98,13 @@ pub(crate) fn filter_column_symmetric_approx( *dst = src.as_().mul(coeff); } - for (i, coeff) in scanned_kernel.iter().take(half_len).enumerate() { + for (i, &coeff) in scanned_kernel.iter().take(half_len).enumerate() { let rollback = length - i - 1; let fw = &arena_src[i][cx..(cx + 16)]; let bw = &arena_src[rollback][cx..(cx + 16)]; for ((dst, fw), bw) in store.iter_mut().zip(fw).zip(bw) { - *dst = *dst + fw.as_().add(bw.as_()).mul(coeff.weight); + *dst = *dst + fw.as_().add(bw.as_()).mul(coeff); } } @@ -124,14 +123,14 @@ pub(crate) fn filter_column_symmetric_approx( let mut k2 = v_src[2].as_().mul(coeff); let mut k3 = v_src[3].as_().mul(coeff); - for (i, coeff) in scanned_kernel.iter().take(half_len).enumerate() { + for (i, &coeff) in scanned_kernel.iter().take(half_len).enumerate() { let rollback = length - i - 1; let fw = &arena_src[i][cx..(cx + 4)]; let bw = &arena_src[rollback][cx..(cx + 4)]; - k0 = fw[0].as_().add(bw[0].as_()).mul(coeff.weight).add(k0); - k1 = fw[1].as_().add(bw[1].as_()).mul(coeff.weight).add(k1); - k2 = fw[2].as_().add(bw[2].as_()).mul(coeff.weight).add(k2); - k3 = fw[3].as_().add(bw[3].as_()).mul(coeff.weight).add(k3); + k0 = fw[0].as_().add(bw[0].as_()).mul(coeff).add(k0); + k1 = fw[1].as_().add(bw[1].as_()).mul(coeff).add(k1); + k2 = fw[2].as_().add(bw[2].as_()).mul(coeff).add(k2); + k3 = fw[3].as_().add(bw[3].as_()).mul(coeff).add(k3); } *dst.get_unchecked_mut(cx) = k0.to_approx_(); @@ -147,11 +146,11 @@ pub(crate) fn filter_column_symmetric_approx( let mut k0 = v_src.as_().mul(coeff); - for (i, coeff) in scanned_kernel.iter().take(half_len).enumerate() { + for (i, &coeff) in scanned_kernel.iter().take(half_len).enumerate() { let rollback = length - i - 1; let fw = arena_src[i][x]; let bw = arena_src[rollback][x]; - k0 = fw.as_().add(bw.as_()).mul(coeff.weight).add(k0); + k0 = fw.as_().add(bw.as_()).mul(coeff).add(k0); } *dst.get_unchecked_mut(x) = k0.to_approx_(); diff --git a/src/filter1d/filter_complex.rs b/src/filter1d/filter_complex.rs index 4061ecb8..f3f2a4c8 100644 --- a/src/filter1d/filter_complex.rs +++ b/src/filter1d/filter_complex.rs @@ -27,7 +27,7 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::edge_mode::clamp_edge; -use crate::filter1d::arena::{make_arena_columns, make_arena_row, write_arena_row, Arena}; +use crate::filter1d::arena::{Arena, make_arena_columns, make_arena_row, write_arena_row}; use crate::filter1d::filter::create_brows; use crate::filter1d::filter_complex_dispatch::ComplexDispatch; use crate::filter1d::filter_element::KernelShape; diff --git a/src/filter1d/filter_complex_dispatch.rs b/src/filter1d/filter_complex_dispatch.rs index f0157ea8..5d275707 100644 --- a/src/filter1d/filter_complex_dispatch.rs +++ b/src/filter1d/filter_complex_dispatch.rs @@ -26,10 +26,10 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; +use crate::filter1d::Arena; use crate::filter1d::filter_column_complex::filter_column_complex; use crate::filter1d::filter_row_complex::filter_row_complex; -use crate::filter1d::Arena; -use crate::ImageSize; use num_complex::Complex; pub trait ComplexDispatch { diff --git a/src/filter1d/filter_complex_dispatch_q.rs b/src/filter1d/filter_complex_dispatch_q.rs index 0565c6c4..62757703 100644 --- a/src/filter1d/filter_complex_dispatch_q.rs +++ b/src/filter1d/filter_complex_dispatch_q.rs @@ -26,8 +26,8 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -use crate::filter1d::Arena; use crate::ImageSize; +use crate::filter1d::Arena; use num_complex::Complex; pub trait ComplexDispatchQ { diff --git a/src/filter1d/filter_complex_q.rs b/src/filter1d/filter_complex_q.rs index 89be7d46..82beeb84 100644 --- a/src/filter1d/filter_complex_q.rs +++ b/src/filter1d/filter_complex_q.rs @@ -27,7 +27,7 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::edge_mode::clamp_edge; -use crate::filter1d::arena::{make_arena_columns, make_arena_row, write_arena_row, Arena}; +use crate::filter1d::arena::{Arena, make_arena_columns, make_arena_row, write_arena_row}; use crate::filter1d::filter::create_brows; use crate::filter1d::filter_complex_dispatch_q::ComplexDispatchQ; use crate::filter1d::filter_element::KernelShape; diff --git a/src/filter1d/mod.rs b/src/filter1d/mod.rs index a312f576..10d07813 100644 --- a/src/filter1d/mod.rs +++ b/src/filter1d/mod.rs @@ -31,7 +31,7 @@ mod arena; mod arena_roi; #[cfg(all(target_arch = "x86_64", feature = "avx"))] mod avx; -#[cfg(all(target_arch = "x86_64", feature = "nightly_avx512"))] +#[cfg(all(target_arch = "x86_64", feature = "avx512"))] mod avx512; mod filter; mod filter_1d_approx; @@ -67,7 +67,7 @@ pub(crate) mod sse; mod to_approx_storage; mod to_approx_storage_complex; -pub(crate) use arena::{make_arena, Arena, ArenaPads}; +pub(crate) use arena::{Arena, ArenaPads, make_arena}; pub use filter::filter_1d_exact; pub use filter_1d_approx::filter_1d_approx; pub use filter_complex::filter_1d_complex; diff --git a/src/filter1d/neon/filter_column_approx.rs b/src/filter1d/neon/filter_column_approx.rs index 13af7af8..aedcb5f5 100644 --- a/src/filter1d/neon/filter_column_approx.rs +++ b/src/filter1d/neon/filter_column_approx.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::{ vfmlaq_u8_s16, vmullq_u8_by_i16, vqmovnq_s32_u8, xvld1q_u8_x2, xvld1q_u8_x3, xvld1q_u8_x4, xvst1q_u8_x2, xvst1q_u8_x3, xvst1q_u8_x4, @@ -44,7 +43,7 @@ pub(crate) fn filter_column_neon_u8_i32_app( dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -53,7 +52,7 @@ pub(crate) fn filter_column_neon_u8_i32_app( let mut cx = 0usize; - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(0).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(0) as i16); while cx + 64 < image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); @@ -66,7 +65,7 @@ pub(crate) fn filter_column_neon_u8_i32_app( let mut k3 = vmullq_u8_by_i16(source.3, coeff); for i in 1..length { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let v_source = xvld1q_u8_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = vfmlaq_u8_s16(k0, v_source.0, coeff); @@ -97,7 +96,7 @@ pub(crate) fn filter_column_neon_u8_i32_app( let mut k2 = vmullq_u8_by_i16(source.2, coeff); for i in 1..length { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let v_source = xvld1q_u8_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = vfmlaq_u8_s16(k0, v_source.0, coeff); @@ -121,7 +120,7 @@ pub(crate) fn filter_column_neon_u8_i32_app( let mut k1 = vmullq_u8_by_i16(source.1, coeff); for i in 1..length { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let v_source = xvld1q_u8_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = vfmlaq_u8_s16(k0, v_source.0, coeff); @@ -143,7 +142,7 @@ pub(crate) fn filter_column_neon_u8_i32_app( let mut k0 = vmullq_u8_by_i16(source, coeff); for i in 1..length { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let v_source = vld1q_u8(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = vfmlaq_u8_s16(k0, v_source, coeff); } @@ -158,24 +157,24 @@ pub(crate) fn filter_column_neon_u8_i32_app( while cx + 4 < image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - let mut k0 = (*v_src.get_unchecked(0) as i32).mul(coeff.weight); - let mut k1 = (*v_src.get_unchecked(1) as i32).mul(coeff.weight); - let mut k2 = (*v_src.get_unchecked(2) as i32).mul(coeff.weight); - let mut k3 = (*v_src.get_unchecked(3) as i32).mul(coeff.weight); + let mut k0 = (*v_src.get_unchecked(0) as i32).mul(coeff); + let mut k1 = (*v_src.get_unchecked(1) as i32).mul(coeff); + let mut k2 = (*v_src.get_unchecked(2) as i32).mul(coeff); + let mut k3 = (*v_src.get_unchecked(3) as i32).mul(coeff); for i in 1..length { let coeff = *scanned_kernel.get_unchecked(i); k0 = ((*arena_src.get_unchecked(i).get_unchecked(cx)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k0); k1 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k1); k2 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k2); k3 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k3); } @@ -189,12 +188,12 @@ pub(crate) fn filter_column_neon_u8_i32_app( for x in cx..image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff); for i in 1..length { let coeff = *scanned_kernel.get_unchecked(i); k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k0); } diff --git a/src/filter1d/neon/filter_column_approx_rdm.rs b/src/filter1d/neon/filter_column_approx_rdm.rs index 65655213..8ca940db 100644 --- a/src/filter1d/neon/filter_column_approx_rdm.rs +++ b/src/filter1d/neon/filter_column_approx_rdm.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::{ vmlaq_hi_u8_s16, vmullq_expand_i16, vqmovnq_s16x2_u8, xvld1q_u8_x2, xvld1q_u8_x3, xvld1q_u8_x4, xvst1q_u8_x2, xvst1q_u8_x3, xvst1q_u8_x4, @@ -44,7 +43,7 @@ pub(crate) fn filter_column_neon_u8_i32_i16_qrdm_app( dst: &mut [u8], image_size: ImageSize, r: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { executor_unit(arena, arena_src, dst, image_size, r, scanned_kernel); @@ -58,7 +57,7 @@ unsafe fn executor_unit( dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -67,9 +66,9 @@ unsafe fn executor_unit( let mut cx = 0usize; - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(0).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(0) as i16); - while cx + 64 < image_width { + while cx + 64 <= image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let source = xvld1q_u8_x4(v_src.as_ptr()); @@ -80,7 +79,7 @@ unsafe fn executor_unit( let mut k3 = vmullq_expand_i16(source.3, coeff); for i in 1..length { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let v_source = xvld1q_u8_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = vmlaq_hi_u8_s16(k0, v_source.0, coeff); @@ -102,7 +101,7 @@ unsafe fn executor_unit( cx += 64; } - while cx + 48 < image_width { + while cx + 48 <= image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let source = xvld1q_u8_x3(v_src.as_ptr()); @@ -111,7 +110,7 @@ unsafe fn executor_unit( let mut k2 = vmullq_expand_i16(source.2, coeff); for i in 1..length { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let v_source = xvld1q_u8_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = vmlaq_hi_u8_s16(k0, v_source.0, coeff); @@ -131,7 +130,7 @@ unsafe fn executor_unit( cx += 48; } - while cx + 32 < image_width { + while cx + 32 <= image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let source = xvld1q_u8_x2(v_src.as_ptr()); @@ -139,7 +138,7 @@ unsafe fn executor_unit( let mut k1 = vmullq_expand_i16(source.1, coeff); for i in 1..length { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let v_source = xvld1q_u8_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = vmlaq_hi_u8_s16(k0, v_source.0, coeff); @@ -154,14 +153,14 @@ unsafe fn executor_unit( cx += 32; } - while cx + 16 < image_width { + while cx + 16 <= image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let source = vld1q_u8(v_src.as_ptr()); let mut k0 = vmullq_expand_i16(source, coeff); for i in 1..length { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let v_source = vld1q_u8(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = vmlaq_hi_u8_s16(k0, v_source, coeff); } @@ -173,27 +172,27 @@ unsafe fn executor_unit( let coeff = *scanned_kernel.get_unchecked(0); - while cx + 4 < image_width { + while cx + 4 <= image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - let mut k0 = (*v_src.get_unchecked(0) as i32).mul(coeff.weight); - let mut k1 = (*v_src.get_unchecked(1) as i32).mul(coeff.weight); - let mut k2 = (*v_src.get_unchecked(2) as i32).mul(coeff.weight); - let mut k3 = (*v_src.get_unchecked(3) as i32).mul(coeff.weight); + let mut k0 = (*v_src.get_unchecked(0) as i32).mul(coeff); + let mut k1 = (*v_src.get_unchecked(1) as i32).mul(coeff); + let mut k2 = (*v_src.get_unchecked(2) as i32).mul(coeff); + let mut k3 = (*v_src.get_unchecked(3) as i32).mul(coeff); for i in 1..length { let coeff = *scanned_kernel.get_unchecked(i); k0 = ((*arena_src.get_unchecked(i).get_unchecked(cx)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k0); k1 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k1); k2 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k2); k3 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k3); } @@ -207,12 +206,12 @@ unsafe fn executor_unit( for x in cx..image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff); for i in 1..length { let coeff = *scanned_kernel.get_unchecked(i); k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k0); } diff --git a/src/filter1d/neon/filter_column_complex_u16_i32.rs b/src/filter1d/neon/filter_column_complex_u16_i32.rs index ae63d11b..fc7f5bcd 100644 --- a/src/filter1d/neon/filter_column_complex_u16_i32.rs +++ b/src/filter1d/neon/filter_column_complex_u16_i32.rs @@ -32,28 +32,32 @@ use num_complex::Complex; use std::arch::aarch64::*; #[inline(always)] -pub(crate) unsafe fn vq_complex_mla_q( +pub(crate) fn vq_complex_mla_q( acc_r: (int32x4_t, int32x4_t), r0: int32x4_t, i0: int32x4_t, r1: int32x4_t, i1: int32x4_t, ) -> (int32x4_t, int32x4_t) { - let re = vmlsq_s32(vmlaq_s32(acc_r.0, r0, r1), i0, i1); - let im = vmlaq_s32(vmlaq_s32(acc_r.1, r0, i1), i0, r1); - (re, im) + unsafe { + let re = vmlsq_s32(vmlaq_s32(acc_r.0, r0, r1), i0, i1); + let im = vmlaq_s32(vmlaq_s32(acc_r.1, r0, i1), i0, r1); + (re, im) + } } #[inline(always)] -pub(crate) unsafe fn vq_complex_mul_q( +pub(crate) fn vq_complex_mul_q( r0: int32x4_t, i0: int32x4_t, r1: int32x4_t, i1: int32x4_t, ) -> (int32x4_t, int32x4_t) { - let re = vmlsq_s32(vmulq_s32(r0, r1), i0, i1); - let im = vmlaq_s32(vmulq_s32(r0, i1), i0, r1); - (re, im) + unsafe { + let re = vmlsq_s32(vmulq_s32(r0, r1), i0, i1); + let im = vmlaq_s32(vmulq_s32(r0, i1), i0, r1); + (re, im) + } } pub(crate) fn filter_column_complex_u16_i32( @@ -73,7 +77,7 @@ pub(crate) fn filter_column_complex_u16_i32( let c_re = vdupq_n_s32(kernel.get_unchecked(0).re as i32); let c_im = vdupq_n_s32(kernel.get_unchecked(0).im as i32); - while cx + 16 < full_width { + while cx + 16 <= full_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let values0 = vld2q_s32(v_src.as_ptr().cast()); @@ -137,7 +141,7 @@ pub(crate) fn filter_column_complex_u16_i32( cx += 16; } - while cx + 8 < full_width { + while cx + 8 <= full_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let values0 = vld2q_s32(v_src.as_ptr().cast()); @@ -177,7 +181,7 @@ pub(crate) fn filter_column_complex_u16_i32( cx += 8; } - while cx + 4 < full_width { + while cx + 4 <= full_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let values = vld2q_s32(v_src.as_ptr().cast()); diff --git a/src/filter1d/neon/filter_column_complex_u8_f32.rs b/src/filter1d/neon/filter_column_complex_u8_f32.rs index 75f9c35c..4222cfe7 100644 --- a/src/filter1d/neon/filter_column_complex_u8_f32.rs +++ b/src/filter1d/neon/filter_column_complex_u8_f32.rs @@ -33,53 +33,61 @@ use num_complex::Complex; use std::arch::aarch64::*; #[inline(always)] -pub(crate) unsafe fn v_complex_mul( +pub(crate) fn v_complex_mul( r0: float32x2_t, i0: float32x2_t, r1: float32x2_t, i1: float32x2_t, ) -> (float32x2_t, float32x2_t) { - let re = vfms_f32(vmul_f32(r0, r1), i0, i1); - let im = vfma_f32(vmul_f32(r0, i1), i0, r1); - (re, im) + unsafe { + let re = vfms_f32(vmul_f32(r0, r1), i0, i1); + let im = vfma_f32(vmul_f32(r0, i1), i0, r1); + (re, im) + } } #[inline(always)] -pub(crate) unsafe fn v_complex_mla( +pub(crate) fn v_complex_mla( acc_r: (float32x2_t, float32x2_t), r0: float32x2_t, i0: float32x2_t, r1: float32x2_t, i1: float32x2_t, ) -> (float32x2_t, float32x2_t) { - let re = vfms_f32(vfma_f32(acc_r.0, r0, r1), i0, i1); - let im = vfma_f32(vfma_f32(acc_r.1, r0, i1), i0, r1); - (re, im) + unsafe { + let re = vfms_f32(vfma_f32(acc_r.0, r0, r1), i0, i1); + let im = vfma_f32(vfma_f32(acc_r.1, r0, i1), i0, r1); + (re, im) + } } #[inline(always)] -pub(crate) unsafe fn vq_complex_mla( +pub(crate) fn vq_complex_mla( acc_r: (float32x4_t, float32x4_t), r0: float32x4_t, i0: float32x4_t, r1: float32x4_t, i1: float32x4_t, ) -> (float32x4_t, float32x4_t) { - let re = vfmsq_f32(vfmaq_f32(acc_r.0, r0, r1), i0, i1); - let im = vfmaq_f32(vfmaq_f32(acc_r.1, r0, i1), i0, r1); - (re, im) + unsafe { + let re = vfmsq_f32(vfmaq_f32(acc_r.0, r0, r1), i0, i1); + let im = vfmaq_f32(vfmaq_f32(acc_r.1, r0, i1), i0, r1); + (re, im) + } } #[inline(always)] -pub(crate) unsafe fn vq_complex_mul( +pub(crate) fn vq_complex_mul( r0: float32x4_t, i0: float32x4_t, r1: float32x4_t, i1: float32x4_t, ) -> (float32x4_t, float32x4_t) { - let re = vfmsq_f32(vmulq_f32(r0, r1), i0, i1); - let im = vfmaq_f32(vmulq_f32(r0, i1), i0, r1); - (re, im) + unsafe { + let re = vfmsq_f32(vmulq_f32(r0, r1), i0, i1); + let im = vfmaq_f32(vmulq_f32(r0, i1), i0, r1); + (re, im) + } } pub(crate) fn filter_column_complex_u8_f32( diff --git a/src/filter1d/neon/filter_column_complex_u8_f32_fcma.rs b/src/filter1d/neon/filter_column_complex_u8_f32_fcma.rs index e2af64f2..36de1ec4 100644 --- a/src/filter1d/neon/filter_column_complex_u8_f32_fcma.rs +++ b/src/filter1d/neon/filter_column_complex_u8_f32_fcma.rs @@ -44,15 +44,17 @@ pub(crate) fn filter_column_complex_u8_f32_fcma( } } -#[inline] -pub(crate) unsafe fn vqpermf(v0: float32x4_t, v1: float32x4_t, q: uint8x16_t) -> float32x4_t { - let a0 = vreinterpretq_f32_u8(vqtbl1q_u8(vreinterpretq_u8_f32(v0), q)); - let a1 = vreinterpretq_f32_u8(vqtbl1q_u8(vreinterpretq_u8_f32(v1), q)); - vcombine_f32(vget_low_f32(a0), vget_low_f32(a1)) +#[inline(always)] +pub(crate) fn vqpermf(v0: float32x4_t, v1: float32x4_t, q: uint8x16_t) -> float32x4_t { + unsafe { + let a0 = vreinterpretq_f32_u8(vqtbl1q_u8(vreinterpretq_u8_f32(v0), q)); + let a1 = vreinterpretq_f32_u8(vqtbl1q_u8(vreinterpretq_u8_f32(v1), q)); + vcombine_f32(vget_low_f32(a0), vget_low_f32(a1)) + } } #[target_feature(enable = "fcma")] -unsafe fn filter_column_complex_u8_f32_impl_fc( +fn filter_column_complex_u8_f32_impl_fc( arena: Arena, arena_src: &[&[Complex]], dst: &mut [u8], @@ -79,7 +81,7 @@ unsafe fn filter_column_complex_u8_f32_impl_fc( .as_ptr(), ); - while cx + 8 < full_width { + while cx + 8 <= full_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let values0 = vld1q_f32(v_src.as_ptr().cast()); @@ -148,7 +150,7 @@ unsafe fn filter_column_complex_u8_f32_impl_fc( cx += 8; } - while cx + 4 < full_width { + while cx + 4 <= full_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let values0 = vld1q_f32(v_src.as_ptr().cast()); diff --git a/src/filter1d/neon/filter_column_complex_u8_i32.rs b/src/filter1d/neon/filter_column_complex_u8_i32.rs index 0336e68e..f0b9fc3d 100644 --- a/src/filter1d/neon/filter_column_complex_u8_i32.rs +++ b/src/filter1d/neon/filter_column_complex_u8_i32.rs @@ -32,75 +32,83 @@ use num_complex::Complex; use std::arch::aarch64::*; #[inline(always)] -pub(crate) unsafe fn vq_complex_mla_q( +pub(crate) fn vq_complex_mla_q( acc_r: ((int32x4_t, int32x4_t), (int32x4_t, int32x4_t)), r0: int16x8_t, i0: int16x8_t, r1: int16x8_t, i1: int16x8_t, ) -> ((int32x4_t, int32x4_t), (int32x4_t, int32x4_t)) { - let acc0 = acc_r.0; - let acc1 = acc_r.1; - let re0 = vmlsl_s16( - vmlal_s16(acc0.0, vget_low_s16(r0), vget_low_s16(r1)), - vget_low_s16(i0), - vget_low_s16(i1), - ); - let im0 = vmlal_s16( - vmlal_s16(acc0.1, vget_low_s16(r0), vget_low_s16(i1)), - vget_low_s16(i0), - vget_low_s16(r1), - ); - let re1 = vmlsl_high_s16(vmlal_high_s16(acc1.0, r0, r1), i0, i1); - let im1 = vmlal_high_s16(vmlal_high_s16(acc1.1, r0, i1), i0, r1); - ((re0, im0), (re1, im1)) + unsafe { + let acc0 = acc_r.0; + let acc1 = acc_r.1; + let re0 = vmlsl_s16( + vmlal_s16(acc0.0, vget_low_s16(r0), vget_low_s16(r1)), + vget_low_s16(i0), + vget_low_s16(i1), + ); + let im0 = vmlal_s16( + vmlal_s16(acc0.1, vget_low_s16(r0), vget_low_s16(i1)), + vget_low_s16(i0), + vget_low_s16(r1), + ); + let re1 = vmlsl_high_s16(vmlal_high_s16(acc1.0, r0, r1), i0, i1); + let im1 = vmlal_high_s16(vmlal_high_s16(acc1.1, r0, i1), i0, r1); + ((re0, im0), (re1, im1)) + } } #[inline(always)] -pub(crate) unsafe fn v_complex_mla_q( +pub(crate) fn v_complex_mla_q( acc: (int32x4_t, int32x4_t), r0: int16x4_t, i0: int16x4_t, r1: int16x4_t, i1: int16x4_t, ) -> (int32x4_t, int32x4_t) { - let re0 = vmlsl_s16(vmlal_s16(acc.0, r0, r1), i0, i1); - let im0 = vmlal_s16(vmlal_s16(acc.1, r0, i1), i0, r1); - (re0, im0) + unsafe { + let re0 = vmlsl_s16(vmlal_s16(acc.0, r0, r1), i0, i1); + let im0 = vmlal_s16(vmlal_s16(acc.1, r0, i1), i0, r1); + (re0, im0) + } } #[inline(always)] -pub(crate) unsafe fn vq_complex_mul_q( +pub(crate) fn vq_complex_mul_q( r0: int16x8_t, i0: int16x8_t, r1: int16x8_t, i1: int16x8_t, ) -> ((int32x4_t, int32x4_t), (int32x4_t, int32x4_t)) { - let re0 = vmlsl_s16( - vmull_s16(vget_low_s16(r0), vget_low_s16(r1)), - vget_low_s16(i0), - vget_low_s16(i1), - ); - let im0 = vmlal_s16( - vmull_s16(vget_low_s16(r0), vget_low_s16(i1)), - vget_low_s16(i0), - vget_low_s16(r1), - ); - let re1 = vmlsl_high_s16(vmull_high_s16(r0, r1), i0, i1); - let im1 = vmlal_high_s16(vmull_high_s16(r0, i1), i0, r1); - ((re0, im0), (re1, im1)) + unsafe { + let re0 = vmlsl_s16( + vmull_s16(vget_low_s16(r0), vget_low_s16(r1)), + vget_low_s16(i0), + vget_low_s16(i1), + ); + let im0 = vmlal_s16( + vmull_s16(vget_low_s16(r0), vget_low_s16(i1)), + vget_low_s16(i0), + vget_low_s16(r1), + ); + let re1 = vmlsl_high_s16(vmull_high_s16(r0, r1), i0, i1); + let im1 = vmlal_high_s16(vmull_high_s16(r0, i1), i0, r1); + ((re0, im0), (re1, im1)) + } } #[inline(always)] -pub(crate) unsafe fn v_complex_mul_q( +pub(crate) fn v_complex_mul_q( r0: int16x4_t, i0: int16x4_t, r1: int16x4_t, i1: int16x4_t, ) -> (int32x4_t, int32x4_t) { - let re0 = vmlsl_s16(vmull_s16(r0, r1), i0, i1); - let im0 = vmlal_s16(vmull_s16(r0, i1), i0, r1); - (re0, im0) + unsafe { + let re0 = vmlsl_s16(vmull_s16(r0, r1), i0, i1); + let im0 = vmlal_s16(vmull_s16(r0, i1), i0, r1); + (re0, im0) + } } pub(crate) fn filter_column_complex_u8_i32( diff --git a/src/filter1d/neon/filter_column_symm_approx.rs b/src/filter1d/neon/filter_column_symm_approx.rs index 6cd69f59..d8a6054d 100644 --- a/src/filter1d/neon/filter_column_symm_approx.rs +++ b/src/filter1d/neon/filter_column_symm_approx.rs @@ -27,8 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; -use crate::filter1d::neon::filter_column_symm_approx_uq0_7::filter_column_symm_neon_u8_uq0_7; use crate::filter1d::neon::utils::{ vfmlaq_symm_u8_s16, vmullq_u8_by_i16, vqmovnq_s32_u8, xvld1q_u8_x2, xvld1q_u8_x3, xvld1q_u8_x4, xvst1q_u8_x2, xvst1q_u8_x3, xvst1q_u8_x4, @@ -44,33 +42,20 @@ pub(crate) fn filter_column_symm_neon_u8_i32_app( arena_src: &[&[u8]], dst: &mut [u8], image_size: ImageSize, - zfr: FilterRegion, - scanned_kernel: &[ScanPoint1d], + _: FilterRegion, + kernel: &[i32], ) { - if scanned_kernel.len() <= 7 { - let is_all_positive = scanned_kernel.iter().all(|&x| x.weight > 0); - if is_all_positive { - return filter_column_symm_neon_u8_uq0_7( - arena, - arena_src, - dst, - image_size, - zfr, - scanned_kernel, - ); - } - } unsafe { let image_width = image_size.width * arena.components; - let length = scanned_kernel.len(); + let length = kernel.len(); let half_len = length / 2; let mut cx = 0usize; let ref0 = arena_src.get_unchecked(half_len); - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(half_len).weight as i16); + let coeff = vdupq_n_s16(*kernel.get_unchecked(half_len) as i16); while cx + 64 < image_width { let v_src = ref0.get_unchecked(cx..); @@ -83,7 +68,7 @@ pub(crate) fn filter_column_symm_neon_u8_i32_app( for i in 0..half_len { let rollback = length - i - 1; - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*kernel.get_unchecked(i) as i16); let v_source0 = xvld1q_u8_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = xvld1q_u8_x4( @@ -120,7 +105,7 @@ pub(crate) fn filter_column_symm_neon_u8_i32_app( let mut k2 = vmullq_u8_by_i16(source.2, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = xvld1q_u8_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); @@ -151,7 +136,7 @@ pub(crate) fn filter_column_symm_neon_u8_i32_app( let mut k1 = vmullq_u8_by_i16(source.1, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = xvld1q_u8_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); @@ -180,7 +165,7 @@ pub(crate) fn filter_column_symm_neon_u8_i32_app( let mut k0 = vmullq_u8_by_i16(source, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = vld1q_u8(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = vld1q_u8( @@ -197,34 +182,34 @@ pub(crate) fn filter_column_symm_neon_u8_i32_app( cx += 16; } - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); while cx + 4 < image_width { let v_src = ref0.get_unchecked(cx..); - let mut k0 = (*v_src.get_unchecked(0) as i32).mul(coeff.weight); - let mut k1 = (*v_src.get_unchecked(1) as i32).mul(coeff.weight); - let mut k2 = (*v_src.get_unchecked(2) as i32).mul(coeff.weight); - let mut k3 = (*v_src.get_unchecked(3) as i32).mul(coeff.weight); + let mut k0 = (*v_src.get_unchecked(0) as i32).mul(coeff); + let mut k1 = (*v_src.get_unchecked(1) as i32).mul(coeff); + let mut k2 = (*v_src.get_unchecked(2) as i32).mul(coeff); + let mut k3 = (*v_src.get_unchecked(3) as i32).mul(coeff); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; k0 = ((*arena_src.get_unchecked(i).get_unchecked(cx)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(cx)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k0); k1 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(cx + 1)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k1); k2 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(cx + 2)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k2); k3 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(cx + 3)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k3); } @@ -238,14 +223,14 @@ pub(crate) fn filter_column_symm_neon_u8_i32_app( for x in cx..image_width { let v_src = ref0.get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k0); } diff --git a/src/filter1d/neon/filter_column_symm_approx_rdm.rs b/src/filter1d/neon/filter_column_symm_approx_rdm.rs index 89b2e231..27ac2b12 100644 --- a/src/filter1d/neon/filter_column_symm_approx_rdm.rs +++ b/src/filter1d/neon/filter_column_symm_approx_rdm.rs @@ -27,12 +27,10 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; -use crate::filter1d::neon::filter_column_symm_approx_uq0_7::filter_column_symm_neon_u8_uq0_7; use crate::filter1d::neon::utils::{ vmla_symm_hi_u8_s16, vmlaq_symm_hi_u8_s16, vmull_expand_i16, vmullq_expand_i16, - vqmovnq_s16x2_u8, xvld1q_u8_x2, xvld1q_u8_x3, xvld1q_u8_x4, xvld4u8, xvst1q_u8_x2, - xvst1q_u8_x3, xvst1q_u8_x4, xvst_u8x4_q15, + vqmovnq_s16x2_u8, xvld1q_u8_x2, xvld1q_u8_x3, xvld1q_u8_x4, xvld4u8, xvst_u8x4_q15, + xvst1q_u8_x2, xvst1q_u8_x3, xvst1q_u8_x4, }; use crate::filter1d::region::FilterRegion; use crate::filter1d::to_approx_storage::ToApproxStorage; @@ -46,22 +44,9 @@ pub(crate) fn filter_column_symm_neon_u8_i32_rdm( dst: &mut [u8], image_size: ImageSize, r: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { - if scanned_kernel.len() <= 7 { - let is_all_positive = scanned_kernel.iter().all(|&x| x.weight > 0); - if is_all_positive { - return filter_column_symm_neon_u8_uq0_7( - arena, - arena_src, - dst, - image_size, - r, - scanned_kernel, - ); - } - } executor_unit(arena, arena_src, dst, image_size, r, scanned_kernel); } } @@ -73,7 +58,7 @@ unsafe fn executor_unit( dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -84,7 +69,7 @@ unsafe fn executor_unit( let mut cx = 0usize; let ref0 = arena_src.get_unchecked(half_len); - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(half_len).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(half_len) as i16); while cx + 64 < image_width { let v_src = ref0.get_unchecked(cx..); @@ -97,7 +82,7 @@ unsafe fn executor_unit( for i in 0..half_len { let rollback = length - i - 1; - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let v_source0 = xvld1q_u8_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = xvld1q_u8_x4( @@ -134,7 +119,7 @@ unsafe fn executor_unit( let mut k2 = vmullq_expand_i16(source.2, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = xvld1q_u8_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); @@ -169,7 +154,7 @@ unsafe fn executor_unit( let mut k1 = vmullq_expand_i16(source.1, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = xvld1q_u8_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); @@ -198,7 +183,7 @@ unsafe fn executor_unit( let mut k0 = vmullq_expand_i16(source, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = vld1q_u8(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = vld1q_u8( @@ -222,7 +207,7 @@ unsafe fn executor_unit( let mut k0 = vmull_expand_i16(source, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = xvld4u8(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = xvld4u8( @@ -244,14 +229,14 @@ unsafe fn executor_unit( for x in cx..image_width { let v_src = ref0.get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff); for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k0); } diff --git a/src/filter1d/neon/filter_column_symm_approx_rdm_x3.rs b/src/filter1d/neon/filter_column_symm_approx_rdm_x3.rs index e67d4cb1..add64c7e 100644 --- a/src/filter1d/neon/filter_column_symm_approx_rdm_x3.rs +++ b/src/filter1d/neon/filter_column_symm_approx_rdm_x3.rs @@ -28,10 +28,9 @@ */ use crate::filter1d::arena::Arena; use crate::filter1d::filter_1d_column_handler::FilterBrows; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::{ vmla_symm_hi_u8_s16, vmlaq_symm_hi_u8_s16, vmull_expand_i16, vmullq_expand_i16, - vqmovnq_s16x2_u8, xvld1q_u8_x2, xvld4u8, xvst1q_u8_x2, xvst_u8x4_q15, + vqmovnq_s16x2_u8, xvld1q_u8_x2, xvld4u8, xvst_u8x4_q15, xvst1q_u8_x2, }; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; @@ -44,7 +43,7 @@ pub(crate) fn filter_column_symm_neon_u8_i32_rdm_x3( dst: &mut [u8], image_size: ImageSize, dst_stride: usize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { executor_unit(arena, brows, dst, image_size, dst_stride, scanned_kernel); @@ -58,7 +57,7 @@ unsafe fn executor_unit( dst: &mut [u8], image_size: ImageSize, dst_stride: usize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -77,7 +76,7 @@ unsafe fn executor_unit( let ref1 = brows1.get_unchecked(half_len); let ref2 = brows2.get_unchecked(half_len); - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(half_len).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(half_len) as i16); let mut cx = 0usize; @@ -100,7 +99,7 @@ unsafe fn executor_unit( let mut k1_2 = vmullq_expand_i16(source2.1, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0_0 = xvld1q_u8_x2(brows0.get_unchecked(i).get_unchecked(cx..).as_ptr()); @@ -157,7 +156,7 @@ unsafe fn executor_unit( let mut k2 = vmullq_expand_i16(source2, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0_0 = vld1q_u8(brows0.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1_0 = @@ -197,7 +196,7 @@ unsafe fn executor_unit( let mut k2 = vmull_expand_i16(source2, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0_0 = xvld4u8(brows0.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1_0 = @@ -230,26 +229,26 @@ unsafe fn executor_unit( let v_src1 = ref1.get_unchecked(x..); let v_src2 = ref2.get_unchecked(x..); - let mut k0 = ((*v_src0.get_unchecked(0)) as i32).mul(coeff.weight); - let mut k1 = ((*v_src1.get_unchecked(0)) as i32).mul(coeff.weight); - let mut k2 = ((*v_src2.get_unchecked(0)) as i32).mul(coeff.weight); + let mut k0 = ((*v_src0.get_unchecked(0)) as i32).mul(coeff); + let mut k1 = ((*v_src1.get_unchecked(0)) as i32).mul(coeff); + let mut k2 = ((*v_src2.get_unchecked(0)) as i32).mul(coeff); for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; k0 = ((*brows0.get_unchecked(i).get_unchecked(x)) as i32) .add((*brows0.get_unchecked(rollback).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k0); k1 = ((*brows1.get_unchecked(i).get_unchecked(x)) as i32) .add((*brows1.get_unchecked(rollback).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k1); k2 = ((*brows2.get_unchecked(i).get_unchecked(x)) as i32) .add((*brows2.get_unchecked(rollback).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k2); } diff --git a/src/filter1d/neon/filter_column_symm_approx_uq0_7.rs b/src/filter1d/neon/filter_column_symm_approx_uq0_7.rs index d5843507..acdf5748 100644 --- a/src/filter1d/neon/filter_column_symm_approx_uq0_7.rs +++ b/src/filter1d/neon/filter_column_symm_approx_uq0_7.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::{xvld1q_u8_x2, xvst1q_u8_x2}; use crate::filter1d::region::FilterRegion; use crate::img_size::ImageSize; @@ -40,50 +39,35 @@ pub(crate) fn filter_column_symm_neon_u8_uq0_7( dst: &mut [u8], image_size: ImageSize, zf: FilterRegion, - scanned_kernel: &[ScanPoint1d], + kernel: &[u8], ) { - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8) as u8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } + unsafe { + executor_unit(arena, arena_src, dst, image_size, zf, kernel); } - executor_unit(arena, arena_src, dst, image_size, zf, &shifted); } +#[target_feature(enable = "neon")] fn executor_unit( arena: Arena, arena_src: &[&[u8]], dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[u8], + kernel: &[u8], ) { unsafe { let image_width = image_size.width * arena.components; - let length = scanned_kernel.len(); + let length = kernel.len(); let half_len = length / 2; let mut cx = 0usize; let ref0 = arena_src.get_unchecked(half_len); - let coeff = vdupq_n_u8(*scanned_kernel.get_unchecked(half_len)); + let coeff = vdupq_n_u8(*kernel.get_unchecked(half_len)); - while cx + 32 < image_width { + while cx + 32 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = xvld1q_u8_x2(v_src.as_ptr()); @@ -93,7 +77,7 @@ fn executor_unit( let mut k3 = vmull_high_u8(source.1, coeff); for i in 0..half_len { - let coeff = vdupq_n_u16(*scanned_kernel.get_unchecked(i) as u16); + let coeff = vdupq_n_u16(*kernel.get_unchecked(i) as u16); let rollback = length - i - 1; let v_source0 = xvld1q_u8_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); @@ -128,7 +112,7 @@ fn executor_unit( cx += 32; } - while cx + 16 < image_width { + while cx + 16 <= image_width { let v_src = ref0.get_unchecked(cx..); let source = vld1q_u8(v_src.as_ptr()); @@ -136,7 +120,7 @@ fn executor_unit( let mut k1 = vmull_high_u8(source, coeff); for i in 0..half_len { - let coeff = vdupq_n_u16(*scanned_kernel.get_unchecked(i) as u16); + let coeff = vdupq_n_u16(*kernel.get_unchecked(i) as u16); let rollback = length - i - 1; let v_source0 = vld1q_u8(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = vld1q_u8( @@ -161,8 +145,8 @@ fn executor_unit( cx += 16; } - while cx + 4 < image_width { - let coeff = *scanned_kernel.get_unchecked(half_len); + while cx + 4 <= image_width { + let coeff = *kernel.get_unchecked(half_len); let v_src = ref0.get_unchecked(cx..); @@ -172,7 +156,7 @@ fn executor_unit( let mut k3 = (*v_src.get_unchecked(3) as u16).mul(coeff as u16); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; k0 = ((*arena_src.get_unchecked(i).get_unchecked(cx)) as u16) .add((*arena_src.get_unchecked(rollback).get_unchecked(cx)) as u16) @@ -200,14 +184,14 @@ fn executor_unit( } for x in cx..image_width { - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); let v_src = ref0.get_unchecked(x..); let mut k0 = ((*v_src.get_unchecked(0)) as u16).mul(coeff as u16); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as u16) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as u16) diff --git a/src/filter1d/neon/filter_column_symm_u15_u16_x3.rs b/src/filter1d/neon/filter_column_symm_u15_u16_x3.rs index ab7e877f..0be66fae 100644 --- a/src/filter1d/neon/filter_column_symm_u15_u16_x3.rs +++ b/src/filter1d/neon/filter_column_symm_u15_u16_x3.rs @@ -28,7 +28,6 @@ */ use crate::filter1d::arena::Arena; use crate::filter1d::filter_1d_column_handler::FilterBrows; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::{ vfmlaq_symm_u16_u16, vmullq_u16_by_u16, vqmovnq_u32_u16, xvld1q_u16_x2, xvst1q_u16_x2, }; @@ -44,7 +43,7 @@ pub(crate) fn filter_symm_column_neon_uq15_u16_x3( dst: &mut [u16], image_size: ImageSize, dst_stride: usize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[u32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -65,7 +64,7 @@ pub(crate) fn filter_symm_column_neon_uq15_u16_x3( let mut cx = 0usize; - let coeff = vdupq_n_u16(scanned_kernel.get_unchecked(half_len).weight as u16); + let coeff = vdupq_n_u16(*scanned_kernel.get_unchecked(half_len) as u16); while cx + 16 < image_width { let v_src0 = ref0.get_unchecked(cx..); @@ -86,7 +85,7 @@ pub(crate) fn filter_symm_column_neon_uq15_u16_x3( let mut k1_2 = vmullq_u16_by_u16(source2.1, coeff); for i in 0..half_len { - let coeff = vdupq_n_u32(scanned_kernel.get_unchecked(i).weight); + let coeff = vdupq_n_u32(*scanned_kernel.get_unchecked(i)); let rollback = length - i - 1; let v_source0_0 = @@ -147,7 +146,7 @@ pub(crate) fn filter_symm_column_neon_uq15_u16_x3( let mut k2 = vmullq_u16_by_u16(source2, coeff); for i in 0..half_len { - let coeff = vdupq_n_u32(scanned_kernel.get_unchecked(i).weight); + let coeff = vdupq_n_u32(*scanned_kernel.get_unchecked(i)); let rollback = length - i - 1; let v_source0_0 = vld1q_u16(brows0.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1_0 = @@ -191,7 +190,7 @@ pub(crate) fn filter_symm_column_neon_uq15_u16_x3( let mut k2 = vmull_u16(source2, vget_low_u16(coeff)); for i in 0..half_len { - let coeff = vdupq_n_u32(scanned_kernel.get_unchecked(i).weight); + let coeff = vdupq_n_u32(*scanned_kernel.get_unchecked(i)); let rollback = length - i - 1; let v_source0_0 = vld1_u16(brows0.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1_0 = @@ -221,7 +220,7 @@ pub(crate) fn filter_symm_column_neon_uq15_u16_x3( cx += 4; } - let coeff = scanned_kernel.get_unchecked(half_len).weight; + let coeff = scanned_kernel.get_unchecked(half_len); for x in cx..dst_stride { let mut k0 = (*ref0.get_unchecked(x) as u32).mul(coeff); @@ -235,21 +234,21 @@ pub(crate) fn filter_symm_column_neon_uq15_u16_x3( k0, ((*brows0.get_unchecked(i).get_unchecked(x)) as u32) .add(*brows0.get_unchecked(rollback).get_unchecked(x) as u32), - coeff.weight, + coeff, ); k1 = mlaf( k1, ((*brows1.get_unchecked(i).get_unchecked(x)) as u32) .add(*brows1.get_unchecked(rollback).get_unchecked(x) as u32), - coeff.weight, + coeff, ); k2 = mlaf( k2, ((*brows2.get_unchecked(i).get_unchecked(x)) as u32) .add(*brows2.get_unchecked(rollback).get_unchecked(x) as u32), - coeff.weight, + coeff, ); } diff --git a/src/filter1d/neon/filter_column_symm_uq15_u16.rs b/src/filter1d/neon/filter_column_symm_uq15_u16.rs index 546ff113..76c22ccc 100644 --- a/src/filter1d/neon/filter_column_symm_uq15_u16.rs +++ b/src/filter1d/neon/filter_column_symm_uq15_u16.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::{ vfmlaq_symm_u16_u16, vmullq_u16_by_u16, vqmovnq_u32_u16, xvld1q_u16_x2, xvld1q_u16_x3, xvld1q_u16_x4, xvst1q_u16_x2, xvst1q_u16_x3, xvst1q_u16_x4, @@ -44,7 +43,7 @@ pub(crate) fn filter_column_symm_neon_uq15_u16( dst: &mut [u16], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[u32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -56,7 +55,7 @@ pub(crate) fn filter_column_symm_neon_uq15_u16( let ref0 = arena_src.get_unchecked(half_len); - let coeff = vdupq_n_u16(scanned_kernel.get_unchecked(half_len).weight as u16); + let coeff = vdupq_n_u16(*scanned_kernel.get_unchecked(half_len) as u16); while cx + 32 < image_width { let v_src = ref0.get_unchecked(cx..); @@ -69,7 +68,7 @@ pub(crate) fn filter_column_symm_neon_uq15_u16( for i in 0..half_len { let rollback = length - i - 1; - let coeff = vdupq_n_u32(scanned_kernel.get_unchecked(i).weight); + let coeff = vdupq_n_u32(*scanned_kernel.get_unchecked(i)); let v_source0 = xvld1q_u16_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = xvld1q_u16_x4( @@ -106,7 +105,7 @@ pub(crate) fn filter_column_symm_neon_uq15_u16( let mut k2 = vmullq_u16_by_u16(source.2, coeff); for i in 0..half_len { - let coeff = vdupq_n_u32(scanned_kernel.get_unchecked(i).weight); + let coeff = vdupq_n_u32(*scanned_kernel.get_unchecked(i)); let rollback = length - i - 1; let v_source0 = xvld1q_u16_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); @@ -141,7 +140,7 @@ pub(crate) fn filter_column_symm_neon_uq15_u16( let mut k1 = vmullq_u16_by_u16(source.1, coeff); for i in 0..half_len { - let coeff = vdupq_n_u32(scanned_kernel.get_unchecked(i).weight); + let coeff = vdupq_n_u32(*scanned_kernel.get_unchecked(i)); let rollback = length - i - 1; let v_source0 = xvld1q_u16_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); @@ -170,7 +169,7 @@ pub(crate) fn filter_column_symm_neon_uq15_u16( let mut k0 = vmullq_u16_by_u16(source, coeff); for i in 0..half_len { - let coeff = vdupq_n_u32(scanned_kernel.get_unchecked(i).weight); + let coeff = vdupq_n_u32(*scanned_kernel.get_unchecked(i)); let rollback = length - i - 1; let v_source0 = vld1q_u16(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = vld1q_u16( @@ -194,7 +193,7 @@ pub(crate) fn filter_column_symm_neon_uq15_u16( let mut k0 = vmull_u16(source, vget_low_u16(coeff)); for i in 0..half_len { - let coeff = vdupq_n_u32(scanned_kernel.get_unchecked(i).weight); + let coeff = vdupq_n_u32(*scanned_kernel.get_unchecked(i)); let rollback = length - i - 1; let v_source0 = vld1_u16(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = vld1_u16( @@ -216,14 +215,14 @@ pub(crate) fn filter_column_symm_neon_uq15_u16( for x in cx..image_width { let v_src = ref0.get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as u32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as u32).mul(coeff); for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as u32) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as u32) - .mul(coeff.weight) + .mul(coeff) .add(k0); } diff --git a/src/filter1d/neon/filter_row_symm_approx_rdm.rs b/src/filter1d/neon/filter_row_symm_approx_rdm.rs index c22f3177..513fa68e 100644 --- a/src/filter1d/neon/filter_row_symm_approx_rdm.rs +++ b/src/filter1d/neon/filter_row_symm_approx_rdm.rs @@ -30,8 +30,8 @@ use crate::filter1d::arena::Arena; use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::{ vmla_symm_hi_u8_s16, vmlaq_symm_hi_u8_s16, vmull_expand_i16, vmullq_expand_i16, vqmovn_s16_u8, - vqmovnq_s16x2_u8, xvld1q_u8_x3, xvld1q_u8_x4, xvld4u8, xvst1q_u8_x3, xvst1q_u8_x4, - xvst_u8x4_q15, + vqmovnq_s16x2_u8, xvld1q_u8_x3, xvld1q_u8_x4, xvld4u8, xvst_u8x4_q15, xvst1q_u8_x3, + xvst1q_u8_x4, }; use crate::filter1d::region::FilterRegion; use crate::filter1d::to_approx_storage::ToApproxStorage; diff --git a/src/filter1d/neon/mod.rs b/src/filter1d/neon/mod.rs index c9cef3ca..9f5da546 100644 --- a/src/filter1d/neon/mod.rs +++ b/src/filter1d/neon/mod.rs @@ -94,14 +94,14 @@ pub(crate) use filter_column_approx_rdm::filter_column_neon_u8_i32_i16_qrdm_app; pub(crate) use filter_column_complex_f32_f32::filter_column_complex_f32_f32; #[cfg(feature = "nightly_fcma")] pub(crate) use filter_column_complex_f32_f32_fcma::filter_column_complex_f32_f32_fcma; -pub(crate) use filter_column_complex_u16_f32::filter_column_complex_u16_f32; -#[cfg(feature = "nightly_fcma")] -pub(crate) use filter_column_complex_u16_f32_fcma::filter_column_complex_u16_f32_fcma; -pub(crate) use filter_column_complex_u16_i32::filter_column_complex_u16_i32; pub(crate) use filter_column_complex_u8_f32::filter_column_complex_u8_f32; #[cfg(feature = "nightly_fcma")] pub(crate) use filter_column_complex_u8_f32_fcma::filter_column_complex_u8_f32_fcma; pub(crate) use filter_column_complex_u8_i32::filter_column_complex_u8_i32; +pub(crate) use filter_column_complex_u16_f32::filter_column_complex_u16_f32; +#[cfg(feature = "nightly_fcma")] +pub(crate) use filter_column_complex_u16_f32_fcma::filter_column_complex_u16_f32_fcma; +pub(crate) use filter_column_complex_u16_i32::filter_column_complex_u16_i32; pub(crate) use filter_column_f32::filter_column_neon_f32_f32; pub(crate) use filter_column_symm::filter_symm_column_neon_u8_f32; pub(crate) use filter_column_symm_approx::filter_column_symm_neon_u8_i32_app; @@ -109,12 +109,13 @@ pub(crate) use filter_column_symm_approx::filter_column_symm_neon_u8_i32_app; pub(crate) use filter_column_symm_approx_rdm::filter_column_symm_neon_u8_i32_rdm; #[cfg(feature = "rdm")] pub(crate) use filter_column_symm_approx_rdm_x3::filter_column_symm_neon_u8_i32_rdm_x3; +pub(crate) use filter_column_symm_approx_uq0_7::filter_column_symm_neon_u8_uq0_7; pub(crate) use filter_column_symm_f32::filter_column_neon_symm_f32_f32; pub(crate) use filter_column_symm_f32_f64::filter_column_neon_symm_f32_f64; +pub(crate) use filter_column_symm_u8_i16::filter_column_symm_neon_u8_i16; pub(crate) use filter_column_symm_u15_u16_x3::filter_symm_column_neon_uq15_u16_x3; pub(crate) use filter_column_symm_u16::filter_symm_column_neon_u16_f32; pub(crate) use filter_column_symm_u16_x3::filter_symm_column_neon_u16_f32_x3; -pub(crate) use filter_column_symm_u8_i16::filter_column_symm_neon_u8_i16; pub(crate) use filter_column_symm_uq15_u16::filter_column_symm_neon_uq15_u16; pub(crate) use filter_column_symm_x3::filter_symm_column_neon_u8_f32_x3; pub(crate) use filter_column_u8_i16::filter_column_neon_u8_i16; @@ -123,10 +124,10 @@ pub(crate) use filter_row_approx::filter_row_neon_u8_i32_app; #[cfg(feature = "rdm")] pub(crate) use filter_row_approx_rdm::filter_row_neon_u8_i32_rdm; pub(crate) use filter_row_complex_f32_f32::filter_row_complex_f32_f32; -pub(crate) use filter_row_complex_u16_f32::filter_row_complex_u16_f32; -pub(crate) use filter_row_complex_u16_i32::filter_row_complex_u16_i32_q; pub(crate) use filter_row_complex_u8_f32::filter_row_complex_u8_f32; pub(crate) use filter_row_complex_u8_i32::filter_row_complex_u8_i32_q; +pub(crate) use filter_row_complex_u16_f32::filter_row_complex_u16_f32; +pub(crate) use filter_row_complex_u16_i32::filter_row_complex_u16_i32_q; pub(crate) use filter_row_f32::filter_row_neon_f32_f32; pub(crate) use filter_row_symm::filter_row_symm_neon_u8_f32; pub(crate) use filter_row_symm_approx::filter_row_symm_neon_u8_i32; @@ -134,8 +135,8 @@ pub(crate) use filter_row_symm_approx::filter_row_symm_neon_u8_i32; pub(crate) use filter_row_symm_approx_rdm::filter_row_symm_neon_u8_i32_rdm; pub(crate) use filter_row_symm_f32::filter_row_neon_symm_f32_f32; pub(crate) use filter_row_symm_f32_f64::filter_row_neon_symm_f32_f64; -pub(crate) use filter_row_symm_u16::filter_row_symm_neon_u16_f32; pub(crate) use filter_row_symm_u8_i16::filter_rgb_row_symm_neon_u8_i16; +pub(crate) use filter_row_symm_u16::filter_row_symm_neon_u16_f32; pub(crate) use filter_row_symm_uq15_u16::filter_row_symm_neon_uq15_u16; pub(crate) use filter_row_u8_i16::filter_rgb_row_neon_u8_i16; pub(crate) use rom_symm_approx3::filter_row_symm_neon_binter_u8_uq0_7_k3; diff --git a/src/filter1d/neon/rom_symm_approx3.rs b/src/filter1d/neon/rom_symm_approx3.rs index 02d52935..8a7e661e 100644 --- a/src/filter1d/neon/rom_symm_approx3.rs +++ b/src/filter1d/neon/rom_symm_approx3.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::edge_mode::clamp_edge; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::rom_symm_approx5::xvst_u8x4_q0_7; use crate::filter1d::neon::utils::{ xvld1q_u8_x2, xvld1q_u8_x4, xvld4u8, xvst1q_u8_x2, xvst1q_u8_x4, @@ -41,28 +40,10 @@ pub(crate) fn filter_row_symm_neon_binter_u8_uq0_7_k3( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + kernel: &[u8], ) { assert!(N <= 4); - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8) as u8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } - executor_unit_3_q0_u8::(edge_mode, m_src, m_dst, image_size, &shifted); + executor_unit_3_q0_u8::(edge_mode, m_src, m_dst, image_size, kernel); } fn executor_unit_3_q0_u8( diff --git a/src/filter1d/neon/rom_symm_approx5.rs b/src/filter1d/neon/rom_symm_approx5.rs index 959c3088..edb9102d 100644 --- a/src/filter1d/neon/rom_symm_approx5.rs +++ b/src/filter1d/neon/rom_symm_approx5.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::edge_mode::clamp_edge; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::xvld4u8; use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::{BorderHandle, ImageSize}; @@ -38,34 +37,18 @@ pub(crate) fn filter_row_symm_neon_binter_u8_uq0_7_x5( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + kernel: &[u8], ) { assert!(N <= 4); - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8) as u8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } - executor_unit_5_q0_u8::(edge_mode, m_src, m_dst, image_size, &shifted); + executor_unit_5_q0_u8::(edge_mode, m_src, m_dst, image_size, kernel); } #[inline(always)] -pub(crate) unsafe fn xvst_u8x4_q0_7(a: *mut u8, v: uint16x8_t) { - let shifted = vqrshrn_n_u16::<7>(v); - vst1_lane_u32::<0>(a as *mut _, vreinterpret_u32_u8(shifted)); +pub(crate) fn xvst_u8x4_q0_7(a: *mut u8, v: uint16x8_t) { + unsafe { + let shifted = vqrshrn_n_u16::<7>(v); + vst1_lane_u32::<0>(a as *mut _, vreinterpret_u32_u8(shifted)); + } } fn executor_unit_5_q0_u8( diff --git a/src/filter1d/neon/row_symm_app_binter_rdm.rs b/src/filter1d/neon/row_symm_app_binter_rdm.rs index a5fbaac0..202bf1a4 100644 --- a/src/filter1d/neon/row_symm_app_binter_rdm.rs +++ b/src/filter1d/neon/row_symm_app_binter_rdm.rs @@ -26,17 +26,16 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::BorderHandle; use crate::edge_mode::{border_interpolate, clamp_edge}; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::{ vmla_symm_hi_u8_s16, vmlaq_symm_hi_u8_s16, vmull_expand_i16, vmullq_expand_i16, vqmovn_s16_u8, - vqmovnq_s16x2_u8, xvld1q_u8_x3, xvld1q_u8_x4, xvld4u8, xvst1q_u8_x3, xvst1q_u8_x4, - xvst_u8x4_q15, + vqmovnq_s16x2_u8, xvld1q_u8_x3, xvld1q_u8_x4, xvld4u8, xvst_u8x4_q15, xvst1q_u8_x3, + xvst1q_u8_x4, }; use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; -use crate::BorderHandle; use std::arch::aarch64::*; pub(crate) fn filter_row_symm_neon_binter_u8_i32_rdm( @@ -44,28 +43,28 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32_rdm( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { - executor_unit::(edge_mode, m_src, m_dst, image_size, scanned_kernel); + executor_unit::(edge_mode, m_src, m_dst, image_size, kernel); } } #[target_feature(enable = "rdm")] -unsafe fn executor_unit( +fn executor_unit( edge_mode: BorderHandle, m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { - let length = scanned_kernel.len(); + let length = kernel.len(); if length == 5 { - executor_unit_5::(edge_mode, m_src, m_dst, image_size, scanned_kernel); + executor_unit_5::(edge_mode, m_src, m_dst, image_size, kernel); } else { - executor_unit_any::(edge_mode, m_src, m_dst, image_size, scanned_kernel); + executor_unit_any::(edge_mode, m_src, m_dst, image_size, kernel); } } } @@ -76,12 +75,12 @@ unsafe fn executor_unit_5( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { let width = image_size.width; - let length = scanned_kernel.len(); + let length = kernel.len(); let half_len = length / 2; let min_left = half_len.min(width); @@ -91,9 +90,9 @@ unsafe fn executor_unit_5( let dst = &mut **dst; let mut f_cx = 0usize; - let c0 = scanned_kernel[0].weight; - let c1 = scanned_kernel[1].weight; - let c2 = scanned_kernel[2].weight; + let c0 = kernel[0]; + let c1 = kernel[1]; + let c2 = kernel[2]; while f_cx < min_left { let mx = f_cx as i64 - s_kernel; let e0 = clamp_edge!(edge_mode.edge_mode, mx, 0, width as i64); @@ -124,11 +123,11 @@ unsafe fn executor_unit_5( let m_right = width.saturating_sub(half_len); let max_width = m_right * N; - let c0 = vdupq_n_s16(scanned_kernel[0].weight as i16); - let c1 = vdupq_n_s16(scanned_kernel[1].weight as i16); - let c2 = vdupq_n_s16(scanned_kernel[2].weight as i16); + let c0 = vdupq_n_s16(kernel[0] as i16); + let c1 = vdupq_n_s16(kernel[1] as i16); + let c2 = vdupq_n_s16(kernel[2] as i16); - while m_cx + 64 < max_width { + while m_cx + 64 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -167,7 +166,7 @@ unsafe fn executor_unit_5( m_cx += 64; } - while m_cx + 48 < max_width { + while m_cx + 48 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -201,7 +200,7 @@ unsafe fn executor_unit_5( m_cx += 48; } - while m_cx + 16 < max_width { + while m_cx + 16 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -222,7 +221,7 @@ unsafe fn executor_unit_5( m_cx += 16; } - while m_cx + 8 < max_width { + while m_cx + 8 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -264,9 +263,9 @@ unsafe fn executor_unit_5( m_cx += 4; } - let c0 = scanned_kernel[0].weight; - let c1 = scanned_kernel[1].weight; - let c2 = scanned_kernel[2].weight; + let c0 = kernel[0]; + let c1 = kernel[1]; + let c2 = kernel[2]; for zx in m_cx..max_width { let x = zx - s_half; @@ -315,17 +314,17 @@ unsafe fn executor_unit_5( } #[target_feature(enable = "rdm")] -unsafe fn executor_unit_any( +fn executor_unit_any( edge_mode: BorderHandle, m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { let width = image_size.width; - let length = scanned_kernel.len(); + let length = kernel.len(); let half_len = length / 2; let min_left = half_len.min(width); @@ -338,14 +337,14 @@ unsafe fn executor_unit_any( let dst = &mut **dst; let mut f_cx = 0usize; - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); while f_cx < min_left { for c in 0..N { let mx = f_cx as i64 - s_kernel; - let mut k0: i32 = *src.get_unchecked(f_cx * N + c) as i32 * coeff.weight; + let mut k0: i32 = *src.get_unchecked(f_cx * N + c) as i32 * coeff; for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; let src0 = border_interpolate!( @@ -367,7 +366,7 @@ unsafe fn executor_unit_any( c ); - k0 += (src0 as i32 + src1 as i32) * coeff.weight; + k0 += (src0 as i32 + src1 as i32) * coeff; } *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); @@ -381,9 +380,9 @@ unsafe fn executor_unit_any( let m_right = width.saturating_sub(half_len); let max_width = m_right * N; - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(half_len).weight as i16); + let coeff = vdupq_n_s16(*kernel.get_unchecked(half_len) as i16); - while m_cx + 48 < max_width { + while m_cx + 48 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -394,7 +393,7 @@ unsafe fn executor_unit_any( let mut k2 = vmullq_expand_i16(source.2, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = xvld1q_u8_x3(shifted_src.get_unchecked((i * N)..).as_ptr()); let v_source1 = @@ -416,7 +415,7 @@ unsafe fn executor_unit_any( m_cx += 48; } - while m_cx + 16 < max_width { + while m_cx + 16 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -425,7 +424,7 @@ unsafe fn executor_unit_any( let mut k0 = vmullq_expand_i16(source, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = vld1q_u8(shifted_src.get_unchecked((i * N)..).as_ptr()); let v_source1 = vld1q_u8(shifted_src.get_unchecked((rollback * N)..).as_ptr()); @@ -437,7 +436,7 @@ unsafe fn executor_unit_any( m_cx += 16; } - while m_cx + 8 < max_width { + while m_cx + 8 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -446,7 +445,7 @@ unsafe fn executor_unit_any( let mut k0 = vmull_expand_i16(source, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = vld1_u8(shifted_src.get_unchecked((i * N)..).as_ptr()); let v_source1 = vld1_u8(shifted_src.get_unchecked((rollback * N)..).as_ptr()); @@ -458,7 +457,7 @@ unsafe fn executor_unit_any( m_cx += 8; } - while m_cx + 4 < max_width { + while m_cx + 4 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -467,7 +466,7 @@ unsafe fn executor_unit_any( let mut k0 = vmull_expand_i16(source, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = xvld4u8(shifted_src.get_unchecked((i * N)..).as_ptr()); let v_source1 = xvld4u8(shifted_src.get_unchecked((rollback * N)..).as_ptr()); @@ -479,22 +478,22 @@ unsafe fn executor_unit_any( m_cx += 4; } - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); for zx in m_cx..max_width { let x = zx - s_half; let shifted_src = src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; + let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff; for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; k0 += (*shifted_src.get_unchecked(i * N) as i16 + *shifted_src.get_unchecked(rollback * N) as i16) as i32 - * coeff.weight; + * coeff; } *dst.get_unchecked_mut(zx) = k0.to_approx_(); @@ -504,13 +503,13 @@ unsafe fn executor_unit_any( while f_cx < width { let mx = f_cx as i64 - s_kernel; - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); for c in 0..N { - let mut k0 = *src.get_unchecked(f_cx * N + c) as i32 * coeff.weight; + let mut k0 = *src.get_unchecked(f_cx * N + c) as i32 * coeff; for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; let src0 = border_interpolate!( @@ -532,7 +531,7 @@ unsafe fn executor_unit_any( c ); - k0 += (src0 as i32 + src1 as i32) * coeff.weight; + k0 += (src0 as i32 + src1 as i32) * coeff; } *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); diff --git a/src/filter1d/neon/row_symm_approx_binter.rs b/src/filter1d/neon/row_symm_approx_binter.rs index 97c80b0d..2420f51d 100644 --- a/src/filter1d/neon/row_symm_approx_binter.rs +++ b/src/filter1d/neon/row_symm_approx_binter.rs @@ -26,8 +26,8 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::BorderHandle; use crate::edge_mode::border_interpolate; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::{ vfmla_symm_u8_s16, vfmlaq_symm_u8_s16, vmull_u8_by_i16, vmullq_u8_by_i16, vqmovn_s32_u8, vqmovnq_s32_u8, xvld1q_u8_x2, xvld1q_u8_x4, xvst1q_u8_x2, xvst1q_u8_x4, @@ -35,7 +35,6 @@ use crate::filter1d::neon::utils::{ use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; -use crate::BorderHandle; use std::arch::aarch64::*; pub(crate) fn filter_row_symm_neon_binter_u8_i32( @@ -43,7 +42,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { let width = image_size.width; @@ -61,7 +60,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( while f_cx < min_left { for c in 0..N { let mx = f_cx as i64 - s_kernel; - let mut k0: i32 = *src.get_unchecked(f_cx * N + c) as i32 * coeff.weight; + let mut k0: i32 = *src.get_unchecked(f_cx * N + c) as i32 * coeff; for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); @@ -86,7 +85,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( c ); - k0 += (src0 as i32 + src1 as i32) * coeff.weight; + k0 += (src0 as i32 + src1 as i32) * coeff; } *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); @@ -100,9 +99,9 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( let m_right = width.saturating_sub(half_len); let max_width = m_right * N; - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(half_len).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(half_len) as i16); - while m_cx + 64 < max_width { + while m_cx + 64 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -113,7 +112,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( let mut k3 = vmullq_u8_by_i16(source.3, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = xvld1q_u8_x4(shifted_src.get_unchecked((i * N)..).as_ptr()); let v_source1 = @@ -137,8 +136,8 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( m_cx += 64; } - while m_cx + 32 < max_width { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(half_len).weight as i16); + while m_cx + 32 <= max_width { + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(half_len) as i16); let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -148,7 +147,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( let mut k1 = vmullq_u8_by_i16(source.1, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = xvld1q_u8_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); let v_source1 = @@ -165,7 +164,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( m_cx += 32; } - while m_cx + 16 < max_width { + while m_cx + 16 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -173,7 +172,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( let mut k0 = vmullq_u8_by_i16(source, coeff); for i in 0..half_len { - let coeff = vdupq_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdupq_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = vld1q_u8(shifted_src.get_unchecked((i * N)..).as_ptr()); let v_source1 = vld1q_u8(shifted_src.get_unchecked((rollback * N)..).as_ptr()); @@ -185,7 +184,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( m_cx += 16; } - while m_cx + 8 < max_width { + while m_cx + 8 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); @@ -193,7 +192,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( let mut k0 = vmull_u8_by_i16(source, vget_low_s16(coeff)); for i in 0..half_len { - let coeff = vdup_n_s16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = vdup_n_s16(*scanned_kernel.get_unchecked(i) as i16); let rollback = length - i - 1; let v_source0 = vld1_u8(shifted_src.get_unchecked((i * N)..).as_ptr()); let v_source1 = vld1_u8(shifted_src.get_unchecked((rollback * N)..).as_ptr()); @@ -207,13 +206,13 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( let coeff = *scanned_kernel.get_unchecked(half_len); - while m_cx + 4 < max_width { + while m_cx + 4 <= max_width { let cx = m_cx - s_half; let shifted_src = src.get_unchecked(cx..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; - let mut k1 = *shifted_src.get_unchecked(half_len * N + 1) as i32 * coeff.weight; - let mut k2 = *shifted_src.get_unchecked(half_len * N + 2) as i32 * coeff.weight; - let mut k3 = *shifted_src.get_unchecked(half_len * N + 3) as i32 * coeff.weight; + let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff; + let mut k1 = *shifted_src.get_unchecked(half_len * N + 1) as i32 * coeff; + let mut k2 = *shifted_src.get_unchecked(half_len * N + 2) as i32 * coeff; + let mut k3 = *shifted_src.get_unchecked(half_len * N + 3) as i32 * coeff; for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); @@ -222,22 +221,22 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( k0 += (*shifted_src.get_unchecked(i * N) as i16 + *shifted_src.get_unchecked(rollback * N) as i16) as i32 - * coeff.weight; + * coeff; k1 += (*shifted_src.get_unchecked(i * N + 1) as i16 + *shifted_src.get_unchecked(rollback * N + 1) as i16) as i32 - * coeff.weight; + * coeff; k2 += (*shifted_src.get_unchecked(i * N + 2) as i16 + *shifted_src.get_unchecked(rollback * N + 2) as i16) as i32 - * coeff.weight; + * coeff; k3 += (*shifted_src.get_unchecked(i * N + 3) as i16 + *shifted_src.get_unchecked(rollback * N + 3) as i16) as i32 - * coeff.weight; + * coeff; } *dst.get_unchecked_mut(m_cx) = k0.to_approx_(); @@ -250,7 +249,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( for zx in m_cx..max_width { let x = zx - s_half; let shifted_src = src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; + let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff; for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); @@ -259,7 +258,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( k0 += (*shifted_src.get_unchecked(i * N) as i16 + *shifted_src.get_unchecked(rollback * N) as i16) as i32 - * coeff.weight; + * coeff; } *dst.get_unchecked_mut(zx) = k0.to_approx_(); @@ -270,7 +269,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( while f_cx < width { for c in 0..N { let mx = f_cx as i64 - s_kernel; - let mut k0 = *src.get_unchecked(f_cx * N + c) as i32 * coeff.weight; + let mut k0 = *src.get_unchecked(f_cx * N + c) as i32 * coeff; for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); @@ -295,7 +294,7 @@ pub(crate) fn filter_row_symm_neon_binter_u8_i32( c ); - k0 += (src0 as i32 + src1 as i32) * coeff.weight; + k0 += (src0 as i32 + src1 as i32) * coeff; } *dst.get_unchecked_mut(f_cx * N + c) = k0.to_approx_(); diff --git a/src/filter1d/neon/row_symm_approx_binter_uq0_7.rs b/src/filter1d/neon/row_symm_approx_binter_uq0_7.rs index 29872a2a..c3693171 100644 --- a/src/filter1d/neon/row_symm_approx_binter_uq0_7.rs +++ b/src/filter1d/neon/row_symm_approx_binter_uq0_7.rs @@ -26,14 +26,13 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::BorderHandle; use crate::edge_mode::border_interpolate; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::neon::utils::{ xvld1q_u8_x2, xvld1q_u8_x3, xvld4u8, xvst1q_u8_x2, xvst1q_u8_x3, }; use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::img_size::ImageSize; -use crate::BorderHandle; use std::arch::aarch64::*; pub(crate) fn filter_row_symm_neon_binter_u8_u0_7( @@ -41,33 +40,17 @@ pub(crate) fn filter_row_symm_neon_binter_u8_u0_7( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + kernel: &[u8], ) { - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8) as u8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } - executor_unit::(edge_mode, m_src, m_dst, image_size, &shifted); + executor_unit::(edge_mode, m_src, m_dst, image_size, kernel); } #[inline(always)] -pub(crate) unsafe fn xvst_u8x4_q7(a: *mut u8, v: uint16x8_t) { - let shifted = vqrshrn_n_u16::<7>(v); - vst1_lane_u32::<0>(a as *mut _, vreinterpret_u32_u8(shifted)); +pub(crate) fn xvst_u8x4_q7(a: *mut u8, v: uint16x8_t) { + unsafe { + let shifted = vqrshrn_n_u16::<7>(v); + vst1_lane_u32::<0>(a as *mut _, vreinterpret_u32_u8(shifted)); + } } fn executor_unit( diff --git a/src/filter1d/neon/utils.rs b/src/filter1d/neon/utils.rs index c62ea112..43d261b4 100644 --- a/src/filter1d/neon/utils.rs +++ b/src/filter1d/neon/utils.rs @@ -29,21 +29,20 @@ use crate::neon::p_vfmaq_f32; use std::arch::aarch64::*; -#[inline(always)] +#[inline] +#[target_feature(enable = "rdm")] #[cfg(feature = "rdm")] -pub(crate) unsafe fn vmullq_expand_i16( - input: uint8x16_t, - weight: int16x8_t, -) -> (int16x8_t, int16x8_t) { +pub(crate) fn vmullq_expand_i16(input: uint8x16_t, weight: int16x8_t) -> (int16x8_t, int16x8_t) { let lo_16 = vreinterpretq_s16_u16(vshll_n_u8::<6>(vget_low_u8(input))); let hi_16 = vreinterpretq_s16_u16(vshll_high_n_u8::<6>(input)); (vqrdmulhq_s16(lo_16, weight), vqrdmulhq_s16(hi_16, weight)) } -#[inline(always)] +#[inline] #[cfg(feature = "rdm")] -pub(crate) unsafe fn vmull_expand_i16(input: uint8x8_t, weight: int16x8_t) -> int16x8_t { +#[target_feature(enable = "rdm")] +pub(crate) fn vmull_expand_i16(input: uint8x8_t, weight: int16x8_t) -> int16x8_t { let v = vreinterpretq_s16_u16(vshll_n_u8::<6>(input)); vqrdmulhq_s16(v, weight) } @@ -51,7 +50,7 @@ pub(crate) unsafe fn vmull_expand_i16(input: uint8x8_t, weight: int16x8_t) -> in #[cfg(feature = "rdm")] #[inline] #[target_feature(enable = "rdm")] -pub(crate) unsafe fn vmlaq_hi_u8_s16( +pub(crate) fn vmlaq_hi_u8_s16( store: (int16x8_t, int16x8_t), input: uint8x16_t, weight: int16x8_t, @@ -68,7 +67,7 @@ pub(crate) unsafe fn vmlaq_hi_u8_s16( #[inline] #[target_feature(enable = "rdm")] #[cfg(feature = "rdm")] -pub(crate) unsafe fn vmla_symm_hi_u8_s16( +pub(crate) fn vmla_symm_hi_u8_s16( store: int16x8_t, input0: uint8x8_t, input1: uint8x8_t, @@ -82,7 +81,7 @@ pub(crate) unsafe fn vmla_symm_hi_u8_s16( #[inline] #[target_feature(enable = "rdm")] #[cfg(feature = "rdm")] -pub(crate) unsafe fn vmlaq_symm_hi_u8_s16( +pub(crate) fn vmlaq_symm_hi_u8_s16( store: (int16x8_t, int16x8_t), input0: uint8x16_t, input1: uint8x16_t, @@ -100,499 +99,555 @@ pub(crate) unsafe fn vmlaq_symm_hi_u8_s16( } #[inline(always)] -pub(crate) unsafe fn vmullq_u8_by_i16( +pub(crate) fn vmullq_u8_by_i16( input: uint8x16_t, weight: int16x8_t, ) -> (int32x4_t, int32x4_t, int32x4_t, int32x4_t) { - let lo_16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(input))); - let hi_16 = vreinterpretq_s16_u16(vmovl_high_u8(input)); + unsafe { + let lo_16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(input))); + let hi_16 = vreinterpretq_s16_u16(vmovl_high_u8(input)); - ( - vmull_s16(vget_low_s16(lo_16), vget_low_s16(weight)), - vmull_high_s16(lo_16, weight), - vmull_s16(vget_low_s16(hi_16), vget_low_s16(weight)), - vmull_high_s16(hi_16, weight), - ) + ( + vmull_s16(vget_low_s16(lo_16), vget_low_s16(weight)), + vmull_high_s16(lo_16, weight), + vmull_s16(vget_low_s16(hi_16), vget_low_s16(weight)), + vmull_high_s16(hi_16, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vmullq_u16_by_u16( - input: uint16x8_t, - weight: uint16x8_t, -) -> (uint32x4_t, uint32x4_t) { - ( - vmull_u16(vget_low_u16(input), vget_low_u16(weight)), - vmull_high_u16(input, weight), - ) +pub(crate) fn vmullq_u16_by_u16(input: uint16x8_t, weight: uint16x8_t) -> (uint32x4_t, uint32x4_t) { + unsafe { + ( + vmull_u16(vget_low_u16(input), vget_low_u16(weight)), + vmull_high_u16(input, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vmull_u8_by_i16( - input: uint8x8_t, - weight: int16x4_t, -) -> (int32x4_t, int32x4_t) { - let lo_16 = vreinterpretq_s16_u16(vmovl_u8(input)); +pub(crate) fn vmull_u8_by_i16(input: uint8x8_t, weight: int16x4_t) -> (int32x4_t, int32x4_t) { + unsafe { + let lo_16 = vreinterpretq_s16_u16(vmovl_u8(input)); - ( - vmull_s16(vget_low_s16(lo_16), weight), - vmull_s16(vget_high_s16(lo_16), weight), - ) + ( + vmull_s16(vget_low_s16(lo_16), weight), + vmull_s16(vget_high_s16(lo_16), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vmulq_u8_by_i16( - input: uint8x16_t, - weight: int16x8_t, -) -> (int16x8_t, int16x8_t) { - let lw = vmovl_u8(vget_low_u8(input)); - let hw = vmovl_u8(vget_high_u8(input)); - ( - vmulq_s16(vreinterpretq_s16_u16(lw), weight), - vmulq_s16(vreinterpretq_s16_u16(hw), weight), - ) +pub(crate) fn vmulq_u8_by_i16(input: uint8x16_t, weight: int16x8_t) -> (int16x8_t, int16x8_t) { + unsafe { + let lw = vmovl_u8(vget_low_u8(input)); + let hw = vmovl_u8(vget_high_u8(input)); + ( + vmulq_s16(vreinterpretq_s16_u16(lw), weight), + vmulq_s16(vreinterpretq_s16_u16(hw), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vmulq_u8_by_f32( +pub(crate) fn vmulq_u8_by_f32( input: uint8x16_t, weight: float32x4_t, ) -> (float32x4_t, float32x4_t, float32x4_t, float32x4_t) { - let lo_16 = vmovl_u8(vget_low_u8(input)); - let hi_16 = vmovl_high_u8(input); - let o0 = vmovl_u16(vget_low_u16(lo_16)); - let o1 = vmovl_high_u16(lo_16); - let o2 = vmovl_u16(vget_low_u16(hi_16)); - let o3 = vmovl_high_u16(hi_16); - - let lo_lo = vcvtq_f32_u32(o0); - let lo_hi = vcvtq_f32_u32(o1); - let hi_lo = vcvtq_f32_u32(o2); - let hi_hi = vcvtq_f32_u32(o3); - - ( - vmulq_f32(lo_lo, weight), - vmulq_f32(lo_hi, weight), - vmulq_f32(hi_lo, weight), - vmulq_f32(hi_hi, weight), - ) -} - -#[inline(always)] -pub(crate) unsafe fn vmulq_u16_by_f32( + unsafe { + let lo_16 = vmovl_u8(vget_low_u8(input)); + let hi_16 = vmovl_high_u8(input); + let o0 = vmovl_u16(vget_low_u16(lo_16)); + let o1 = vmovl_high_u16(lo_16); + let o2 = vmovl_u16(vget_low_u16(hi_16)); + let o3 = vmovl_high_u16(hi_16); + + let lo_lo = vcvtq_f32_u32(o0); + let lo_hi = vcvtq_f32_u32(o1); + let hi_lo = vcvtq_f32_u32(o2); + let hi_hi = vcvtq_f32_u32(o3); + + ( + vmulq_f32(lo_lo, weight), + vmulq_f32(lo_hi, weight), + vmulq_f32(hi_lo, weight), + vmulq_f32(hi_hi, weight), + ) + } +} + +#[inline(always)] +pub(crate) fn vmulq_u16_by_f32( input: uint16x8_t, weight: float32x4_t, ) -> (float32x4_t, float32x4_t) { - let o0 = vmovl_u16(vget_low_u16(input)); - let o1 = vmovl_high_u16(input); + unsafe { + let o0 = vmovl_u16(vget_low_u16(input)); + let o1 = vmovl_high_u16(input); - let lo_lo = vcvtq_f32_u32(o0); - let lo_hi = vcvtq_f32_u32(o1); + let lo_lo = vcvtq_f32_u32(o0); + let lo_hi = vcvtq_f32_u32(o1); - (vmulq_f32(lo_lo, weight), vmulq_f32(lo_hi, weight)) + (vmulq_f32(lo_lo, weight), vmulq_f32(lo_hi, weight)) + } } #[inline(always)] -pub(crate) unsafe fn vmul_u8_by_f32( - input: uint8x8_t, - weight: float32x4_t, -) -> (float32x4_t, float32x4_t) { - let lo_16 = vmovl_u8(input); +pub(crate) fn vmul_u8_by_f32(input: uint8x8_t, weight: float32x4_t) -> (float32x4_t, float32x4_t) { + unsafe { + let lo_16 = vmovl_u8(input); - let o0 = vmovl_u16(vget_low_u16(lo_16)); - let o1 = vmovl_high_u16(lo_16); + let o0 = vmovl_u16(vget_low_u16(lo_16)); + let o1 = vmovl_high_u16(lo_16); - let lo_lo = vcvtq_f32_u32(o0); - let lo_hi = vcvtq_f32_u32(o1); + let lo_lo = vcvtq_f32_u32(o0); + let lo_hi = vcvtq_f32_u32(o1); - (vmulq_f32(lo_lo, weight), vmulq_f32(lo_hi, weight)) + (vmulq_f32(lo_lo, weight), vmulq_f32(lo_hi, weight)) + } } #[inline(always)] -pub(crate) unsafe fn vfmlaq_u8_f32( +pub(crate) fn vfmlaq_u8_f32( store: (float32x4_t, float32x4_t, float32x4_t, float32x4_t), input: uint8x16_t, weight: float32x4_t, ) -> (float32x4_t, float32x4_t, float32x4_t, float32x4_t) { - let lo_16 = vmovl_u8(vget_low_u8(input)); - let hi_16 = vmovl_high_u8(input); + unsafe { + let lo_16 = vmovl_u8(vget_low_u8(input)); + let hi_16 = vmovl_high_u8(input); - let o0 = vmovl_u16(vget_low_u16(lo_16)); - let o1 = vmovl_high_u16(lo_16); - let o2 = vmovl_u16(vget_low_u16(hi_16)); - let o3 = vmovl_high_u16(hi_16); + let o0 = vmovl_u16(vget_low_u16(lo_16)); + let o1 = vmovl_high_u16(lo_16); + let o2 = vmovl_u16(vget_low_u16(hi_16)); + let o3 = vmovl_high_u16(hi_16); - let lo_lo = vcvtq_f32_u32(o0); - let lo_hi = vcvtq_f32_u32(o1); - let hi_lo = vcvtq_f32_u32(o2); - let hi_hi = vcvtq_f32_u32(o3); + let lo_lo = vcvtq_f32_u32(o0); + let lo_hi = vcvtq_f32_u32(o1); + let hi_lo = vcvtq_f32_u32(o2); + let hi_hi = vcvtq_f32_u32(o3); - ( - p_vfmaq_f32(store.0, lo_lo, weight), - p_vfmaq_f32(store.1, lo_hi, weight), - p_vfmaq_f32(store.2, hi_lo, weight), - p_vfmaq_f32(store.3, hi_hi, weight), - ) + ( + p_vfmaq_f32(store.0, lo_lo, weight), + p_vfmaq_f32(store.1, lo_hi, weight), + p_vfmaq_f32(store.2, hi_lo, weight), + p_vfmaq_f32(store.3, hi_hi, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vfmlaq_symm_u8_f32( +pub(crate) fn vfmlaq_symm_u8_f32( store: (float32x4_t, float32x4_t, float32x4_t, float32x4_t), input0: uint8x16_t, input1: uint8x16_t, weight: float32x4_t, ) -> (float32x4_t, float32x4_t, float32x4_t, float32x4_t) { - let lo_16 = vaddl_u8(vget_low_u8(input0), vget_low_u8(input1)); - let hi_16 = vaddl_high_u8(input0, input1); + unsafe { + let lo_16 = vaddl_u8(vget_low_u8(input0), vget_low_u8(input1)); + let hi_16 = vaddl_high_u8(input0, input1); - let o0 = vmovl_u16(vget_low_u16(lo_16)); - let o1 = vmovl_high_u16(lo_16); - let o2 = vmovl_u16(vget_low_u16(hi_16)); - let o3 = vmovl_high_u16(hi_16); + let o0 = vmovl_u16(vget_low_u16(lo_16)); + let o1 = vmovl_high_u16(lo_16); + let o2 = vmovl_u16(vget_low_u16(hi_16)); + let o3 = vmovl_high_u16(hi_16); - let lo_lo = vcvtq_f32_u32(o0); - let lo_hi = vcvtq_f32_u32(o1); + let lo_lo = vcvtq_f32_u32(o0); + let lo_hi = vcvtq_f32_u32(o1); - let hi_lo = vcvtq_f32_u32(o2); - let hi_hi = vcvtq_f32_u32(o3); + let hi_lo = vcvtq_f32_u32(o2); + let hi_hi = vcvtq_f32_u32(o3); - ( - p_vfmaq_f32(store.0, lo_lo, weight), - p_vfmaq_f32(store.1, lo_hi, weight), - p_vfmaq_f32(store.2, hi_lo, weight), - p_vfmaq_f32(store.3, hi_hi, weight), - ) + ( + p_vfmaq_f32(store.0, lo_lo, weight), + p_vfmaq_f32(store.1, lo_hi, weight), + p_vfmaq_f32(store.2, hi_lo, weight), + p_vfmaq_f32(store.3, hi_hi, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vfmlaq_symm_u16_f32( +pub(crate) fn vfmlaq_symm_u16_f32( store: (float32x4_t, float32x4_t), input0: uint16x8_t, input1: uint16x8_t, weight: float32x4_t, ) -> (float32x4_t, float32x4_t) { - let lo_32 = vaddl_u16(vget_low_u16(input0), vget_low_u16(input1)); - let hi_32 = vaddl_high_u16(input0, input1); + unsafe { + let lo_32 = vaddl_u16(vget_low_u16(input0), vget_low_u16(input1)); + let hi_32 = vaddl_high_u16(input0, input1); - let ll = vcvtq_f32_u32(lo_32); - let hh = vcvtq_f32_u32(hi_32); - ( - p_vfmaq_f32(store.0, ll, weight), - p_vfmaq_f32(store.1, hh, weight), - ) + let ll = vcvtq_f32_u32(lo_32); + let hh = vcvtq_f32_u32(hi_32); + ( + p_vfmaq_f32(store.0, ll, weight), + p_vfmaq_f32(store.1, hh, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vdotq_exact_s16( +pub(crate) fn vdotq_exact_s16( store: (int16x8_t, int16x8_t), input: uint8x16_t, weight: int16x8_t, ) -> (int16x8_t, int16x8_t) { - let o0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(input))); - let o1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(input))); - ( - vmlaq_s16(store.0, o0, weight), - vmlaq_s16(store.1, o1, weight), - ) + unsafe { + let o0 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(input))); + let o1 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(input))); + ( + vmlaq_s16(store.0, o0, weight), + vmlaq_s16(store.1, o1, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vdotq_exact_symm_s16( +pub(crate) fn vdotq_exact_symm_s16( store: (int16x8_t, int16x8_t), input0: uint8x16_t, input1: uint8x16_t, weight: int16x8_t, ) -> (int16x8_t, int16x8_t) { - let wide_lo = vaddl_u8(vget_low_u8(input0), vget_low_u8(input1)); - let wide_hi = vaddl_u8(vget_high_u8(input0), vget_high_u8(input1)); - ( - vmlaq_s16(store.0, vreinterpretq_s16_u16(wide_lo), weight), - vmlaq_s16(store.1, vreinterpretq_s16_u16(wide_hi), weight), - ) + unsafe { + let wide_lo = vaddl_u8(vget_low_u8(input0), vget_low_u8(input1)); + let wide_hi = vaddl_u8(vget_high_u8(input0), vget_high_u8(input1)); + ( + vmlaq_s16(store.0, vreinterpretq_s16_u16(wide_lo), weight), + vmlaq_s16(store.1, vreinterpretq_s16_u16(wide_hi), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vfmlaq_u8_s16( +pub(crate) fn vfmlaq_u8_s16( store: (int32x4_t, int32x4_t, int32x4_t, int32x4_t), input: uint8x16_t, weight: int16x8_t, ) -> (int32x4_t, int32x4_t, int32x4_t, int32x4_t) { - let lo_16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(input))); - let hi_16 = vreinterpretq_s16_u16(vmovl_high_u8(input)); + unsafe { + let lo_16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(input))); + let hi_16 = vreinterpretq_s16_u16(vmovl_high_u8(input)); - ( - vmlal_s16(store.0, vget_low_s16(lo_16), vget_low_s16(weight)), - vmlal_high_s16(store.1, lo_16, weight), - vmlal_s16(store.2, vget_low_s16(hi_16), vget_low_s16(weight)), - vmlal_high_s16(store.3, hi_16, weight), - ) + ( + vmlal_s16(store.0, vget_low_s16(lo_16), vget_low_s16(weight)), + vmlal_high_s16(store.1, lo_16, weight), + vmlal_s16(store.2, vget_low_s16(hi_16), vget_low_s16(weight)), + vmlal_high_s16(store.3, hi_16, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vfmlaq_symm_u8_s16( +pub(crate) fn vfmlaq_symm_u8_s16( store: (int32x4_t, int32x4_t, int32x4_t, int32x4_t), input0: uint8x16_t, input1: uint8x16_t, weight: int16x8_t, ) -> (int32x4_t, int32x4_t, int32x4_t, int32x4_t) { - let lo_16 = vreinterpretq_s16_u16(vaddq_u16( - vmovl_u8(vget_low_u8(input0)), - vmovl_u8(vget_low_u8(input1)), - )); - let hi_16 = vreinterpretq_s16_u16(vaddl_u8(vget_high_u8(input0), vget_high_u8(input1))); + unsafe { + let lo_16 = vreinterpretq_s16_u16(vaddq_u16( + vmovl_u8(vget_low_u8(input0)), + vmovl_u8(vget_low_u8(input1)), + )); + let hi_16 = vreinterpretq_s16_u16(vaddl_u8(vget_high_u8(input0), vget_high_u8(input1))); - ( - vmlal_s16(store.0, vget_low_s16(lo_16), vget_low_s16(weight)), - vmlal_high_s16(store.1, lo_16, weight), - vmlal_s16(store.2, vget_low_s16(hi_16), vget_low_s16(weight)), - vmlal_high_s16(store.3, hi_16, weight), - ) + ( + vmlal_s16(store.0, vget_low_s16(lo_16), vget_low_s16(weight)), + vmlal_high_s16(store.1, lo_16, weight), + vmlal_s16(store.2, vget_low_s16(hi_16), vget_low_s16(weight)), + vmlal_high_s16(store.3, hi_16, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vfmlaq_symm_u16_u16( +pub(crate) fn vfmlaq_symm_u16_u16( store: (uint32x4_t, uint32x4_t), input0: uint16x8_t, input1: uint16x8_t, weight: uint32x4_t, ) -> (uint32x4_t, uint32x4_t) { - let lo_16 = vaddl_u16(vget_low_u16(input0), vget_low_u16(input1)); - let hi_16 = vaddl_high_u16(input0, input1); + unsafe { + let lo_16 = vaddl_u16(vget_low_u16(input0), vget_low_u16(input1)); + let hi_16 = vaddl_high_u16(input0, input1); - ( - vmlaq_u32(store.0, lo_16, weight), - vmlaq_u32(store.1, hi_16, weight), - ) + ( + vmlaq_u32(store.0, lo_16, weight), + vmlaq_u32(store.1, hi_16, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vfmla_u8_s16( +pub(crate) fn vfmla_u8_s16( store: (int32x4_t, int32x4_t), input: uint8x8_t, weight: int16x4_t, ) -> (int32x4_t, int32x4_t) { - let lo_16 = vreinterpretq_s16_u16(vmovl_u8(input)); + unsafe { + let lo_16 = vreinterpretq_s16_u16(vmovl_u8(input)); - ( - vmlal_s16(store.0, vget_low_s16(lo_16), weight), - vmlal_s16(store.1, vget_high_s16(lo_16), weight), - ) + ( + vmlal_s16(store.0, vget_low_s16(lo_16), weight), + vmlal_s16(store.1, vget_high_s16(lo_16), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vfmla_symm_u8_s16( +pub(crate) fn vfmla_symm_u8_s16( store: (int32x4_t, int32x4_t), input0: uint8x8_t, input1: uint8x8_t, weight: int16x4_t, ) -> (int32x4_t, int32x4_t) { - let lo_16 = vreinterpretq_s16_u16(vaddq_u16(vmovl_u8(input0), vmovl_u8(input1))); + unsafe { + let lo_16 = vreinterpretq_s16_u16(vaddq_u16(vmovl_u8(input0), vmovl_u8(input1))); - ( - vmlal_s16(store.0, vget_low_s16(lo_16), weight), - vmlal_s16(store.1, vget_high_s16(lo_16), weight), - ) + ( + vmlal_s16(store.0, vget_low_s16(lo_16), weight), + vmlal_s16(store.1, vget_high_s16(lo_16), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vfmla_symm_u8_f32( +pub(crate) fn vfmla_symm_u8_f32( store: (float32x4_t, float32x4_t), input0: uint8x8_t, input1: uint8x8_t, weight: float32x4_t, ) -> (float32x4_t, float32x4_t) { - let lo_16 = vaddl_u8(input0, input1); + unsafe { + let lo_16 = vaddl_u8(input0, input1); - let a0 = vmovl_u16(vget_low_u16(lo_16)); - let a1 = vmovl_high_u16(lo_16); + let a0 = vmovl_u16(vget_low_u16(lo_16)); + let a1 = vmovl_high_u16(lo_16); - let lo_lo = vcvtq_f32_u32(a0); - let lo_hi = vcvtq_f32_u32(a1); + let lo_lo = vcvtq_f32_u32(a0); + let lo_hi = vcvtq_f32_u32(a1); - ( - p_vfmaq_f32(store.0, lo_lo, weight), - p_vfmaq_f32(store.1, lo_hi, weight), - ) + ( + p_vfmaq_f32(store.0, lo_lo, weight), + p_vfmaq_f32(store.1, lo_hi, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn vqmovnq_f32_u8( +pub(crate) fn vqmovnq_f32_u8( store: (float32x4_t, float32x4_t, float32x4_t, float32x4_t), ) -> uint8x16_t { - let a0 = vcvtaq_u32_f32(store.0); - let a1 = vcvtaq_u32_f32(store.1); - let a2 = vcvtaq_u32_f32(store.2); - let a3 = vcvtaq_u32_f32(store.3); - let lo_s = vcombine_u16(vmovn_u32(a0), vmovn_u32(a1)); - let hi_s = vcombine_u16(vmovn_u32(a2), vmovn_u32(a3)); - vcombine_u8(vqmovn_u16(lo_s), vqmovn_u16(hi_s)) + unsafe { + let a0 = vcvtaq_u32_f32(store.0); + let a1 = vcvtaq_u32_f32(store.1); + let a2 = vcvtaq_u32_f32(store.2); + let a3 = vcvtaq_u32_f32(store.3); + let lo_s = vcombine_u16(vmovn_u32(a0), vmovn_u32(a1)); + let hi_s = vcombine_u16(vmovn_u32(a2), vmovn_u32(a3)); + vcombine_u8(vqmovn_u16(lo_s), vqmovn_u16(hi_s)) + } } #[inline(always)] -pub(crate) unsafe fn vqmovnq_f32_u16(store: (float32x4_t, float32x4_t)) -> uint16x8_t { - let a0 = vcvtaq_u32_f32(store.0); - let a1 = vcvtaq_u32_f32(store.1); - vcombine_u16(vqmovn_u32(a0), vqmovn_u32(a1)) +pub(crate) fn vqmovnq_f32_u16(store: (float32x4_t, float32x4_t)) -> uint16x8_t { + unsafe { + let a0 = vcvtaq_u32_f32(store.0); + let a1 = vcvtaq_u32_f32(store.1); + vcombine_u16(vqmovn_u32(a0), vqmovn_u32(a1)) + } } #[inline(always)] -pub(crate) unsafe fn vqmovnq_s32_u8( - store: (int32x4_t, int32x4_t, int32x4_t, int32x4_t), -) -> uint8x16_t { - let hi_s = vcombine_u16(vqrshrun_n_s32::<15>(store.2), vqrshrun_n_s32::<15>(store.3)); - let lo_s = vcombine_u16(vqrshrun_n_s32::<15>(store.0), vqrshrun_n_s32::<15>(store.1)); - vcombine_u8(vqmovn_u16(lo_s), vqmovn_u16(hi_s)) +pub(crate) fn vqmovnq_s32_u8(store: (int32x4_t, int32x4_t, int32x4_t, int32x4_t)) -> uint8x16_t { + unsafe { + let hi_s = vcombine_u16(vqrshrun_n_s32::<15>(store.2), vqrshrun_n_s32::<15>(store.3)); + let lo_s = vcombine_u16(vqrshrun_n_s32::<15>(store.0), vqrshrun_n_s32::<15>(store.1)); + vcombine_u8(vqmovn_u16(lo_s), vqmovn_u16(hi_s)) + } } #[inline(always)] -pub(crate) unsafe fn vqmovnq_u32_u16(store: (uint32x4_t, uint32x4_t)) -> uint16x8_t { - vcombine_u16(vqrshrn_n_u32::<15>(store.0), vqrshrn_n_u32::<15>(store.1)) +pub(crate) fn vqmovnq_u32_u16(store: (uint32x4_t, uint32x4_t)) -> uint16x8_t { + unsafe { vcombine_u16(vqrshrn_n_u32::<15>(store.0), vqrshrn_n_u32::<15>(store.1)) } } #[inline(always)] #[cfg(feature = "rdm")] -pub(crate) unsafe fn vqmovnq_s16x2_u8(store: (int16x8_t, int16x8_t)) -> uint8x16_t { - vcombine_u8(vqrshrun_n_s16::<6>(store.0), vqrshrun_n_s16::<6>(store.1)) +pub(crate) fn vqmovnq_s16x2_u8(store: (int16x8_t, int16x8_t)) -> uint8x16_t { + unsafe { vcombine_u8(vqrshrun_n_s16::<6>(store.0), vqrshrun_n_s16::<6>(store.1)) } } #[inline(always)] -pub(crate) unsafe fn vqmovn_s32_u8(store: (int32x4_t, int32x4_t)) -> uint8x8_t { - let lo_s = vcombine_u16(vqrshrun_n_s32::<15>(store.0), vqrshrun_n_s32::<15>(store.1)); - vqmovn_u16(lo_s) +pub(crate) fn vqmovn_s32_u8(store: (int32x4_t, int32x4_t)) -> uint8x8_t { + unsafe { + let lo_s = vcombine_u16(vqrshrun_n_s32::<15>(store.0), vqrshrun_n_s32::<15>(store.1)); + vqmovn_u16(lo_s) + } } #[inline(always)] #[cfg(feature = "rdm")] -pub(crate) unsafe fn vqmovn_s16_u8(store: int16x8_t) -> uint8x8_t { - vqrshrun_n_s16::<6>(store) +pub(crate) fn vqmovn_s16_u8(store: int16x8_t) -> uint8x8_t { + unsafe { vqrshrun_n_s16::<6>(store) } } #[inline(always)] -pub(crate) unsafe fn vqmovn_f32_u8(store: (float32x4_t, float32x4_t)) -> uint8x8_t { - let o0 = vcvtaq_u32_f32(store.0); - let o1 = vcvtaq_u32_f32(store.1); - let lo_s = vcombine_u16(vmovn_u32(o0), vmovn_u32(o1)); - vqmovn_u16(lo_s) +pub(crate) fn vqmovn_f32_u8(store: (float32x4_t, float32x4_t)) -> uint8x8_t { + unsafe { + let o0 = vcvtaq_u32_f32(store.0); + let o1 = vcvtaq_u32_f32(store.1); + let lo_s = vcombine_u16(vmovn_u32(o0), vmovn_u32(o1)); + vqmovn_u16(lo_s) + } } #[inline(always)] pub unsafe fn xvld1q_f32_x4(a: *const f32) -> float32x4x4_t { - float32x4x4_t( - vld1q_f32(a), - vld1q_f32(a.add(4)), - vld1q_f32(a.add(8)), - vld1q_f32(a.add(12)), - ) + unsafe { + float32x4x4_t( + vld1q_f32(a), + vld1q_f32(a.add(4)), + vld1q_f32(a.add(8)), + vld1q_f32(a.add(12)), + ) + } } #[inline(always)] pub unsafe fn xvld1q_f32_x2(a: *const f32) -> float32x4x2_t { - float32x4x2_t(vld1q_f32(a), vld1q_f32(a.add(4))) + unsafe { float32x4x2_t(vld1q_f32(a), vld1q_f32(a.add(4))) } } #[inline(always)] pub(crate) unsafe fn xvld1q_u8_x4(a: *const u8) -> uint8x16x4_t { - uint8x16x4_t( - vld1q_u8(a), - vld1q_u8(a.add(16)), - vld1q_u8(a.add(32)), - vld1q_u8(a.add(48)), - ) + unsafe { + uint8x16x4_t( + vld1q_u8(a), + vld1q_u8(a.add(16)), + vld1q_u8(a.add(32)), + vld1q_u8(a.add(48)), + ) + } } #[inline(always)] pub(crate) unsafe fn xvld1q_u8_x3(a: *const u8) -> uint8x16x3_t { - uint8x16x3_t(vld1q_u8(a), vld1q_u8(a.add(16)), vld1q_u8(a.add(32))) + unsafe { uint8x16x3_t(vld1q_u8(a), vld1q_u8(a.add(16)), vld1q_u8(a.add(32))) } } #[inline(always)] pub(crate) unsafe fn xvld1q_u8_x2(a: *const u8) -> uint8x16x2_t { - uint8x16x2_t(vld1q_u8(a), vld1q_u8(a.add(16))) + unsafe { uint8x16x2_t(vld1q_u8(a), vld1q_u8(a.add(16))) } } #[inline(always)] pub(crate) unsafe fn xvst1q_f32_x4(a: *mut f32, b: float32x4x4_t) { - vst1q_f32(a, b.0); - vst1q_f32(a.add(4), b.1); - vst1q_f32(a.add(8), b.2); - vst1q_f32(a.add(12), b.3); + unsafe { + vst1q_f32(a, b.0); + vst1q_f32(a.add(4), b.1); + vst1q_f32(a.add(8), b.2); + vst1q_f32(a.add(12), b.3); + } } #[inline(always)] pub(crate) unsafe fn xvld1q_u16_x4(a: *const u16) -> uint16x8x4_t { - uint16x8x4_t( - vld1q_u16(a), - vld1q_u16(a.add(8)), - vld1q_u16(a.add(16)), - vld1q_u16(a.add(24)), - ) + unsafe { + uint16x8x4_t( + vld1q_u16(a), + vld1q_u16(a.add(8)), + vld1q_u16(a.add(16)), + vld1q_u16(a.add(24)), + ) + } } #[inline(always)] pub(crate) unsafe fn xvld1q_u16_x3(a: *const u16) -> uint16x8x3_t { - uint16x8x3_t(vld1q_u16(a), vld1q_u16(a.add(8)), vld1q_u16(a.add(16))) + unsafe { uint16x8x3_t(vld1q_u16(a), vld1q_u16(a.add(8)), vld1q_u16(a.add(16))) } } #[inline(always)] pub(crate) unsafe fn xvld1q_u16_x2(a: *const u16) -> uint16x8x2_t { - uint16x8x2_t(vld1q_u16(a), vld1q_u16(a.add(8))) + unsafe { uint16x8x2_t(vld1q_u16(a), vld1q_u16(a.add(8))) } } #[inline(always)] pub(crate) unsafe fn xvst1q_f32_x2(a: *mut f32, b: float32x4x2_t) { - vst1q_f32(a, b.0); - vst1q_f32(a.add(4), b.1); + unsafe { + vst1q_f32(a, b.0); + vst1q_f32(a.add(4), b.1); + } } #[inline(always)] pub(crate) unsafe fn xvst1q_u8_x4(a: *mut u8, b: uint8x16x4_t) { - vst1q_u8(a, b.0); - vst1q_u8(a.add(16), b.1); - vst1q_u8(a.add(32), b.2); - vst1q_u8(a.add(48), b.3); + unsafe { + vst1q_u8(a, b.0); + vst1q_u8(a.add(16), b.1); + vst1q_u8(a.add(32), b.2); + vst1q_u8(a.add(48), b.3); + } } #[inline(always)] pub(crate) unsafe fn xvst1q_u8_x3(a: *mut u8, b: uint8x16x3_t) { - vst1q_u8(a, b.0); - vst1q_u8(a.add(16), b.1); - vst1q_u8(a.add(32), b.2); + unsafe { + vst1q_u8(a, b.0); + vst1q_u8(a.add(16), b.1); + vst1q_u8(a.add(32), b.2); + } } #[inline(always)] pub(crate) unsafe fn xvst1q_u8_x2(a: *mut u8, b: uint8x16x2_t) { - vst1q_u8(a, b.0); - vst1q_u8(a.add(16), b.1); + unsafe { + vst1q_u8(a, b.0); + vst1q_u8(a.add(16), b.1); + } } #[inline(always)] pub(crate) unsafe fn xvst1q_u16_x4(a: *mut u16, b: uint16x8x4_t) { - vst1q_u16(a, b.0); - vst1q_u16(a.add(8), b.1); - vst1q_u16(a.add(16), b.2); - vst1q_u16(a.add(24), b.3); + unsafe { + vst1q_u16(a, b.0); + vst1q_u16(a.add(8), b.1); + vst1q_u16(a.add(16), b.2); + vst1q_u16(a.add(24), b.3); + } } #[inline(always)] pub(crate) unsafe fn xvst1q_u16_x3(a: *mut u16, b: uint16x8x3_t) { - vst1q_u16(a, b.0); - vst1q_u16(a.add(8), b.1); - vst1q_u16(a.add(16), b.2); + unsafe { + vst1q_u16(a, b.0); + vst1q_u16(a.add(8), b.1); + vst1q_u16(a.add(16), b.2); + } } #[inline(always)] pub(crate) unsafe fn xvst1q_u16_x2(a: *mut u16, b: uint16x8x2_t) { - vst1q_u16(a, b.0); - vst1q_u16(a.add(8), b.1); + unsafe { + vst1q_u16(a, b.0); + vst1q_u16(a.add(8), b.1); + } } #[inline(always)] pub(crate) unsafe fn xvld4u8(a: *const u8) -> uint8x8_t { - vreinterpret_u8_u32(vld1_lane_u32::<0>(a as *const _, vdup_n_u32(0))) + unsafe { vreinterpret_u8_u32(vld1_lane_u32::<0>(a as *const _, vdup_n_u32(0))) } } #[cfg(feature = "rdm")] #[inline(always)] pub(crate) unsafe fn xvst_u8x4_q15(a: *mut u8, v: int16x8_t) { - let shifted = vqrshrun_n_s16::<6>(v); - vst1_lane_u32::<0>(a as *mut _, vreinterpret_u32_u8(shifted)); + unsafe { + let shifted = vqrshrun_n_s16::<6>(v); + vst1_lane_u32::<0>(a as *mut _, vreinterpret_u32_u8(shifted)); + } } diff --git a/src/filter1d/row_handler_small_approx.rs b/src/filter1d/row_handler_small_approx.rs index 950baaea..09f25b95 100644 --- a/src/filter1d/row_handler_small_approx.rs +++ b/src/filter1d/row_handler_small_approx.rs @@ -26,18 +26,18 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -use crate::edge_mode::BorderHandle; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::ImageSize; +use crate::edge_mode::BorderHandle; +use std::sync::Arc; #[allow(dead_code)] pub struct RowsHolder<'a, T> { - pub(crate) holder: Vec<&'a [T]>, + pub(crate) holder: [&'a [T]; 1], } #[allow(dead_code)] pub struct RowsHolderMut<'a, T> { - pub(crate) holder: Vec<&'a mut [T]>, + pub(crate) holder: [&'a mut [T]; 1], } type BInterpolateHandler = fn( @@ -45,136 +45,183 @@ type BInterpolateHandler = fn( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[F], ); -pub trait Filter1DRowHandlerBInterpolateApr { - fn get_row_handler_binter_apr( +pub trait ResolveRowHandlerBInter: Send + Sync { + fn handle_row( + &self, + edge_mode: BorderHandle, + src: &RowsHolder, + dst: &mut RowsHolderMut, + image_size: ImageSize, + ); +} + +struct RowHandlerBInter { + handler: BInterpolateHandler, + kernel: Vec, +} + +impl ResolveRowHandlerBInter for RowHandlerBInter { + fn handle_row( + &self, + edge_mode: BorderHandle, + src: &RowsHolder, + dst: &mut RowsHolderMut, + image_size: ImageSize, + ) { + (self.handler)(edge_mode, src, dst, image_size, &self.kernel) + } +} + +pub trait BuildRowHandlerBInter { + fn build_row_handler_binter( is_kernel_symmetric: bool, - kernel: &[ScanPoint1d], - ) -> Option>; + kernel: &[F], + ) -> Option + Send + Sync>>; +} + +// ── helper to wrap a found handler ─────────────────────────────────────────── +#[allow(unused)] +fn wrap_binter( + handler: BInterpolateHandler, + kernel: Vec, +) -> Option + Send + Sync>> { + Some(Arc::new(RowHandlerBInter { handler, kernel })) } -impl Filter1DRowHandlerBInterpolateApr for u8 { - fn get_row_handler_binter_apr( +// ── u8 / i32 ───────────────────────────────────────────────────────────────── +impl BuildRowHandlerBInter for u8 { + fn build_row_handler_binter( is_kernel_symmetric: bool, - _kernel: &[ScanPoint1d], - ) -> Option> { - if is_kernel_symmetric { - #[cfg(all(target_arch = "x86_64", feature = "nightly_avx512"))] - { - if std::arch::is_x86_feature_detected!("avx512bw") { - if _kernel.len() == 3 { - let all_positive = _kernel.iter().all(|&x| x.weight > 0); - if all_positive { - use crate::filter1d::avx512::filter_row_avx512_symm_u8_uq0_7_k3; - return Some(filter_row_avx512_symm_u8_uq0_7_k3::); - } - } - // if _kernel.len() == 5 { - // let all_positive = _kernel.iter().all(|&x| x.weight > 0); - // if all_positive { - // use crate::filter1d::avx::filter_row_avx_symm_u8_uq0_7_k5; - // return Some(filter_row_avx_symm_u8_uq0_7_k5::); - // } - // } - // if _kernel.len() < 9 { - // let all_positive = _kernel.iter().all(|&x| x.weight > 0); - // if all_positive { - // use crate::filter1d::avx::filter_row_avx_symm_u8_uq0_7_any; - // return Some(filter_row_avx_symm_u8_uq0_7_any::); - // } - // } - use crate::filter1d::avx512::filter_row_avx512_symm_u8_i32_app_binter; - return Some(filter_row_avx512_symm_u8_i32_app_binter::); - } + kernel: &[i32], + ) -> Option + Send + Sync>> { + if !is_kernel_symmetric { + return None; + } + + let _all_positive = kernel.iter().all(|&x| x > 0); + + #[cfg(all(target_arch = "x86_64", feature = "avx512"))] + if std::arch::is_x86_feature_detected!("avx512bw") { + if kernel.len() == 3 && _all_positive { + use crate::filter1d::avx512::filter_row_avx512_symm_u8_uq0_7_k3; + use crate::filter1d::filter_1d_column_handler_approx::make_q07_weights; + return wrap_binter( + filter_row_avx512_symm_u8_uq0_7_k3::, + make_q07_weights(kernel), + ); + } + use crate::filter1d::avx512::filter_row_avx512_symm_u8_i32_app_binter; + + let v_prepared = kernel + .iter() + .map(|&x| { + let z = x.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) + }) + .collect::>(); + return wrap_binter(filter_row_avx512_symm_u8_i32_app_binter::, v_prepared); + } + + #[cfg(all(target_arch = "x86_64", feature = "avx"))] + if std::arch::is_x86_feature_detected!("avx2") { + if kernel.len() == 3 && _all_positive { + use crate::filter1d::avx::filter_row_avx_symm_u8_uq0_7_k3; + use crate::filter1d::filter_1d_column_handler_approx::make_q07_weights; + return wrap_binter( + filter_row_avx_symm_u8_uq0_7_k3::, + make_q07_weights(kernel), + ); + } + if kernel.len() == 5 && _all_positive { + use crate::filter1d::avx::filter_row_avx_symm_u8_uq0_7_k5; + use crate::filter1d::filter_1d_column_handler_approx::make_q07_weights; + return wrap_binter( + filter_row_avx_symm_u8_uq0_7_k5::, + make_q07_weights(kernel), + ); } - #[cfg(all(target_arch = "x86_64", feature = "avx"))] - { - if std::arch::is_x86_feature_detected!("avx2") { - if _kernel.len() == 3 { - let all_positive = _kernel.iter().all(|&x| x.weight > 0); - if all_positive { - use crate::filter1d::avx::filter_row_avx_symm_u8_uq0_7_k3; - return Some(filter_row_avx_symm_u8_uq0_7_k3::); - } - } - if _kernel.len() == 5 { - let all_positive = _kernel.iter().all(|&x| x.weight > 0); - if all_positive { - use crate::filter1d::avx::filter_row_avx_symm_u8_uq0_7_k5; - return Some(filter_row_avx_symm_u8_uq0_7_k5::); - } - } - if _kernel.len() < 9 { - let all_positive = _kernel.iter().all(|&x| x.weight > 0); - if all_positive { - use crate::filter1d::avx::filter_row_avx_symm_u8_uq0_7_any; - return Some(filter_row_avx_symm_u8_uq0_7_any::); - } - } - use crate::filter1d::avx::filter_row_avx_symm_u8_i32_app_binter; - return Some(filter_row_avx_symm_u8_i32_app_binter::); - } + if kernel.len() < 9 && _all_positive { + use crate::filter1d::avx::filter_row_avx_symm_u8_uq0_7_any; + use crate::filter1d::filter_1d_column_handler_approx::make_q07_weights; + return wrap_binter( + filter_row_avx_symm_u8_uq0_7_any::, + make_q07_weights(kernel), + ); } - #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] - { - if _kernel.len() < 9 { - let all_positive = _kernel.iter().all(|&x| x.weight > 0); - if all_positive { - use crate::filter1d::sse::filter_row_sse_symm_u8_uq0_7_any; - return Some(filter_row_sse_symm_u8_uq0_7_any::); - } - } + use crate::filter1d::avx::filter_row_avx_symm_u8_i32_app_binter; + let v_prepared = kernel + .iter() + .map(|&x| { + let z = x.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) + }) + .collect::>(); + return wrap_binter(filter_row_avx_symm_u8_i32_app_binter::, v_prepared); + } + + #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] + if std::arch::is_x86_feature_detected!("sse4.1") && kernel.len() < 9 && _all_positive { + use crate::filter1d::filter_1d_column_handler_approx::make_q07_weights; + use crate::filter1d::sse::filter_row_sse_symm_u8_uq0_7_any; + let v_prepared = make_q07_weights(kernel) + .iter() + .map(|&x| { + let z = x.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[0], z[0], z[0]]) + }) + .collect::>(); + + return wrap_binter(filter_row_sse_symm_u8_uq0_7_any::, v_prepared); + } + + #[cfg(all(target_arch = "aarch64", feature = "neon"))] + { + use crate::filter1d::filter_1d_column_handler_approx::make_uq07_weights; + if kernel.len() == 3 && _all_positive { + use crate::filter1d::neon::filter_row_symm_neon_binter_u8_uq0_7_k3; + return wrap_binter( + filter_row_symm_neon_binter_u8_uq0_7_k3::, + make_uq07_weights(kernel), + ); } - #[cfg(all(target_arch = "aarch64", feature = "neon"))] - { - if _kernel.len() == 3 { - let all_positive = _kernel.iter().all(|&x| x.weight > 0); - if all_positive { - use crate::filter1d::neon::filter_row_symm_neon_binter_u8_uq0_7_k3; - return Some(filter_row_symm_neon_binter_u8_uq0_7_k3::); - } - } - if _kernel.len() == 5 { - let all_positive = _kernel.iter().all(|&x| x.weight > 0); - if all_positive { - use crate::filter1d::neon::filter_row_symm_neon_binter_u8_uq0_7_x5; - return Some(filter_row_symm_neon_binter_u8_uq0_7_x5::); - } - } - if _kernel.len() < 9 { - let all_positive = _kernel.iter().all(|&x| x.weight > 0); - if all_positive { - use crate::filter1d::neon::filter_row_symm_neon_binter_u8_u0_7; - return Some(filter_row_symm_neon_binter_u8_u0_7::); - } - } + if kernel.len() == 5 && _all_positive { + use crate::filter1d::neon::filter_row_symm_neon_binter_u8_uq0_7_x5; + return wrap_binter( + filter_row_symm_neon_binter_u8_uq0_7_x5::, + make_uq07_weights(kernel), + ); } - #[cfg(all(target_arch = "aarch64", feature = "rdm"))] - { - if std::arch::is_aarch64_feature_detected!("rdm") { - use crate::filter1d::neon::filter_row_symm_neon_binter_u8_i32_rdm; - return Some(filter_row_symm_neon_binter_u8_i32_rdm::); - } + if kernel.len() < 9 && _all_positive { + use crate::filter1d::neon::filter_row_symm_neon_binter_u8_u0_7; + return wrap_binter( + filter_row_symm_neon_binter_u8_u0_7::, + make_uq07_weights(kernel), + ); } - #[cfg(all(target_arch = "aarch64", feature = "neon"))] - { - use crate::filter1d::neon::filter_row_symm_neon_binter_u8_i32; - return Some(filter_row_symm_neon_binter_u8_i32::); + #[cfg(feature = "rdm")] + if std::arch::is_aarch64_feature_detected!("rdm") { + use crate::filter1d::neon::filter_row_symm_neon_binter_u8_i32_rdm; + return wrap_binter(filter_row_symm_neon_binter_u8_i32_rdm::, kernel.to_vec()); } + use crate::filter1d::neon::filter_row_symm_neon_binter_u8_i32; + wrap_binter(filter_row_symm_neon_binter_u8_i32::, kernel.to_vec()) } + + #[cfg(not(all(target_arch = "aarch64", feature = "neon")))] None } } macro_rules! d_binter { ($store:ty, $intermediate:ty) => { - impl Filter1DRowHandlerBInterpolateApr<$store, $intermediate> for $store { - fn get_row_handler_binter_apr( + impl BuildRowHandlerBInter<$store, $intermediate> for $store { + fn build_row_handler_binter( _: bool, - _: &[ScanPoint1d<$intermediate>], - ) -> Option> { + _: &[$intermediate], + ) -> Option + Send + Sync>> { None } } diff --git a/src/filter1d/row_symm_approx_binter.rs b/src/filter1d/row_symm_approx_binter.rs index 90e36d61..2d1e946c 100644 --- a/src/filter1d/row_symm_approx_binter.rs +++ b/src/filter1d/row_symm_approx_binter.rs @@ -27,7 +27,7 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #![allow(dead_code)] -use crate::edge_mode::{border_interpolate, BorderHandle}; +use crate::edge_mode::{BorderHandle, border_interpolate}; use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::to_approx_storage::ToApproxStorage; use crate::img_size::ImageSize; diff --git a/src/filter1d/sse/filter_column.rs b/src/filter1d/sse/filter_column.rs index 39bee388..6b91188d 100644 --- a/src/filter1d/sse/filter_column.rs +++ b/src/filter1d/sse/filter_column.rs @@ -71,7 +71,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "sse4.1")] - unsafe fn pass( + fn pass( &self, arena: Arena, arena_src: &[&[u8]], @@ -80,171 +80,173 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let image_width = image_size.width * arena.components; - - let length = scanned_kernel.len(); - - let mut cx = 0usize; - - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(0).weight); - - while cx + 64 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let source = _mm_load_pack_x4(v_src.as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); - let mut k3 = _mm_mul_epi8_by_ps_x4(source.3, coeff); - - for i in 1..length { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = - _mm_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source.0, coeff); - k1 = _mm_mul_add_epi8_by_ps_x4(k1, v_source.1, coeff); - k2 = _mm_mul_add_epi8_by_ps_x4(k2, v_source.2, coeff); - k3 = _mm_mul_add_epi8_by_ps_x4(k3, v_source.3, coeff); - } - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0, - ( - _mm_pack_ps_x4_epi8(k0), - _mm_pack_ps_x4_epi8(k1), - _mm_pack_ps_x4_epi8(k2), - _mm_pack_ps_x4_epi8(k3), - ), - ); - cx += 64; - } - - while cx + 48 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let source = _mm_load_pack_x3(v_src.as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); - - for i in 1..length { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = - _mm_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source.0, coeff); - k1 = _mm_mul_add_epi8_by_ps_x4(k1, v_source.1, coeff); - k2 = _mm_mul_add_epi8_by_ps_x4(k2, v_source.2, coeff); + unsafe { + let image_width = image_size.width * arena.components; + + let length = scanned_kernel.len(); + + let mut cx = 0usize; + + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(0).weight); + + while cx + 64 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let source = _mm_load_pack_x4(v_src.as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); + let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); + let mut k3 = _mm_mul_epi8_by_ps_x4(source.3, coeff); + + for i in 1..length { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = + _mm_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source.0, coeff); + k1 = _mm_mul_add_epi8_by_ps_x4(k1, v_source.1, coeff); + k2 = _mm_mul_add_epi8_by_ps_x4(k2, v_source.2, coeff); + k3 = _mm_mul_add_epi8_by_ps_x4(k3, v_source.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0, + ( + _mm_pack_ps_x4_epi8(k0), + _mm_pack_ps_x4_epi8(k1), + _mm_pack_ps_x4_epi8(k2), + _mm_pack_ps_x4_epi8(k3), + ), + ); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x3( - dst_ptr0, - ( - _mm_pack_ps_x4_epi8(k0), - _mm_pack_ps_x4_epi8(k1), - _mm_pack_ps_x4_epi8(k2), - ), - ); - cx += 48; - } - - while cx + 32 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let source = _mm_load_pack_x2(v_src.as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - - for i in 1..length { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = - _mm_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source.0, coeff); - k1 = _mm_mul_add_epi8_by_ps_x4(k1, v_source.1, coeff); + while cx + 48 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let source = _mm_load_pack_x3(v_src.as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); + let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); + + for i in 1..length { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = + _mm_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source.0, coeff); + k1 = _mm_mul_add_epi8_by_ps_x4(k1, v_source.1, coeff); + k2 = _mm_mul_add_epi8_by_ps_x4(k2, v_source.2, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x3( + dst_ptr0, + ( + _mm_pack_ps_x4_epi8(k0), + _mm_pack_ps_x4_epi8(k1), + _mm_pack_ps_x4_epi8(k2), + ), + ); + cx += 48; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2(dst_ptr0, (_mm_pack_ps_x4_epi8(k0), _mm_pack_ps_x4_epi8(k1))); + while cx + 32 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - cx += 32; - } + let source = _mm_load_pack_x2(v_src.as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - while cx + 16 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + for i in 1..length { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = + _mm_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source.0, coeff); + k1 = _mm_mul_add_epi8_by_ps_x4(k1, v_source.1, coeff); + } - let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi8_by_ps_x4(source_0, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2(dst_ptr0, (_mm_pack_ps_x4_epi8(k0), _mm_pack_ps_x4_epi8(k1))); - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - let v_source_0 = _mm_loadu_si128( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source_0, _mm_set1_ps(coeff.weight)); + cx += 32; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); - cx += 16; - } + while cx + 16 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - let coeff = *scanned_kernel.get_unchecked(0); + let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi8_by_ps_x4(source_0, coeff); - while cx + 4 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + let v_source_0 = + _mm_loadu_si128(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() + as *const __m128i); + k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source_0, _mm_set1_ps(coeff.weight)); + } - let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff.weight); - let mut k1 = (*v_src.get_unchecked(1) as f32).mul(coeff.weight); - let mut k2 = (*v_src.get_unchecked(2) as f32).mul(coeff.weight); - let mut k3 = (*v_src.get_unchecked(3) as f32).mul(coeff.weight); + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); + cx += 16; + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = mlaf( - k0, - (*arena_src.get_unchecked(i).get_unchecked(cx)) as f32, - coeff.weight, - ); - k1 = mlaf( - k1, - (*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as f32, - coeff.weight, - ); - k2 = mlaf( - k2, - (*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as f32, - coeff.weight, - ); - k3 = mlaf( - k3, - (*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as f32, - coeff.weight, - ); + let coeff = *scanned_kernel.get_unchecked(0); + + while cx + 4 <= image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff.weight); + let mut k1 = (*v_src.get_unchecked(1) as f32).mul(coeff.weight); + let mut k2 = (*v_src.get_unchecked(2) as f32).mul(coeff.weight); + let mut k3 = (*v_src.get_unchecked(3) as f32).mul(coeff.weight); + + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 = mlaf( + k0, + (*arena_src.get_unchecked(i).get_unchecked(cx)) as f32, + coeff.weight, + ); + k1 = mlaf( + k1, + (*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as f32, + coeff.weight, + ); + k2 = mlaf( + k2, + (*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as f32, + coeff.weight, + ); + k3 = mlaf( + k3, + (*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as f32, + coeff.weight, + ); + } + + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } + for x in cx..image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(x..); - for x in cx..image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(x..); + let mut k0 = ((*v_src.get_unchecked(0)) as f32).mul(coeff.weight); - let mut k0 = ((*v_src.get_unchecked(0)) as f32).mul(coeff.weight); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 = mlaf( + k0, + (*arena_src.get_unchecked(i).get_unchecked(cx)) as f32, + coeff.weight, + ); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = mlaf( - k0, - (*arena_src.get_unchecked(i).get_unchecked(cx)) as f32, - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/sse/filter_column_approx.rs b/src/filter1d/sse/filter_column_approx.rs index eee4e608..3a4deaa9 100644 --- a/src/filter1d/sse/filter_column_approx.rs +++ b/src/filter1d/sse/filter_column_approx.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::filter1d::sse::utils::{ _mm_mul_add_epi8_by_epi16_x4, _mm_mul_epi8_by_epi16_x4, _mm_pack_epi32_x2_epi8, @@ -50,7 +49,7 @@ pub(crate) fn filter_column_sse_u8_i32_app( dst: &mut [u8], image_size: ImageSize, filter_region: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { filter_column_sse_u8_i32_impl( @@ -71,7 +70,7 @@ unsafe fn filter_column_sse_u8_i32_impl( dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -80,7 +79,7 @@ unsafe fn filter_column_sse_u8_i32_impl( let mut cx = 0usize; - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(0).weight as i16); + let coeff = _mm_set1_epi16(*scanned_kernel.get_unchecked(0) as i16); while cx + 64 < image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); @@ -92,7 +91,7 @@ unsafe fn filter_column_sse_u8_i32_impl( let mut k3 = _mm_mul_epi8_by_epi16_x4(source.3, coeff); for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = _mm_set1_epi16(*scanned_kernel.get_unchecked(i) as i16); let v_source = _mm_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); @@ -123,7 +122,7 @@ unsafe fn filter_column_sse_u8_i32_impl( let mut k2 = _mm_mul_epi8_by_epi16_x4(source.2, coeff); for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = _mm_set1_epi16(*scanned_kernel.get_unchecked(i) as i16); let v_source = _mm_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); @@ -151,7 +150,7 @@ unsafe fn filter_column_sse_u8_i32_impl( let mut k1 = _mm_mul_epi8_by_epi16_x4(source.1, coeff); for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = _mm_set1_epi16(*scanned_kernel.get_unchecked(i) as i16); let v_source = _mm_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); @@ -173,7 +172,7 @@ unsafe fn filter_column_sse_u8_i32_impl( let mut k0 = _mm_mul_epi8_by_epi16_x4(source, coeff); for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let coeff = _mm_set1_epi16(*scanned_kernel.get_unchecked(i) as i16); let v_source = _mm_loadu_si128( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -190,24 +189,24 @@ unsafe fn filter_column_sse_u8_i32_impl( while cx + 4 < image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - let mut k0 = (*v_src.get_unchecked(0) as i32).mul(coeff.weight); - let mut k1 = (*v_src.get_unchecked(1) as i32).mul(coeff.weight); - let mut k2 = (*v_src.get_unchecked(2) as i32).mul(coeff.weight); - let mut k3 = (*v_src.get_unchecked(3) as i32).mul(coeff.weight); + let mut k0 = (*v_src.get_unchecked(0) as i32).mul(coeff); + let mut k1 = (*v_src.get_unchecked(1) as i32).mul(coeff); + let mut k2 = (*v_src.get_unchecked(2) as i32).mul(coeff); + let mut k3 = (*v_src.get_unchecked(3) as i32).mul(coeff); for i in 1..length { let coeff = *scanned_kernel.get_unchecked(i); k0 = ((*arena_src.get_unchecked(i).get_unchecked(cx)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k0); k1 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k1); k2 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k2); k3 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k3); } @@ -221,12 +220,12 @@ unsafe fn filter_column_sse_u8_i32_impl( for x in cx..image_width { let v_src = arena_src.get_unchecked(0).get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff); for i in 1..length { let coeff = *scanned_kernel.get_unchecked(i); k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(coeff) .add(k0); } diff --git a/src/filter1d/sse/filter_column_complex_u16_i32.rs b/src/filter1d/sse/filter_column_complex_u16_i32.rs index 9f14a885..dd3aa91c 100644 --- a/src/filter1d/sse/filter_column_complex_u16_i32.rs +++ b/src/filter1d/sse/filter_column_complex_u16_i32.rs @@ -40,22 +40,26 @@ use std::arch::x86::*; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn mq_complex_mla( +pub(crate) fn mq_complex_mla( acc_r: __m128i, r0i0: __m128i, r: __m128i, r_swop: __m128i, ) -> __m128i { - let c0 = _mm_mullo_epi32(r0i0, r); - let c1 = _mm_mullo_epi32(_mm_shuffle_epi32::<{ _shuffle(2, 3, 0, 1) }>(r0i0), r_swop); - _mm_add_epi32(acc_r, _mm_addsub_epi32(c0, c1)) + unsafe { + let c0 = _mm_mullo_epi32(r0i0, r); + let c1 = _mm_mullo_epi32(_mm_shuffle_epi32::<{ _shuffle(2, 3, 0, 1) }>(r0i0), r_swop); + _mm_add_epi32(acc_r, _mm_addsub_epi32(c0, c1)) + } } #[inline(always)] -pub(crate) unsafe fn mq_complex_mul(r0i0: __m128i, r: __m128i, r_swop: __m128i) -> __m128i { - let c0 = _mm_mullo_epi32(r0i0, r); - let c1 = _mm_mullo_epi32(_mm_shuffle_epi32::<{ _shuffle(2, 3, 0, 1) }>(r0i0), r_swop); - _mm_addsub_epi32(c0, c1) +pub(crate) fn mq_complex_mul(r0i0: __m128i, r: __m128i, r_swop: __m128i) -> __m128i { + unsafe { + let c0 = _mm_mullo_epi32(r0i0, r); + let c1 = _mm_mullo_epi32(_mm_shuffle_epi32::<{ _shuffle(2, 3, 0, 1) }>(r0i0), r_swop); + _mm_addsub_epi32(c0, c1) + } } pub(crate) fn filter_sse_column_complex_u16_i32( @@ -71,7 +75,7 @@ pub(crate) fn filter_sse_column_complex_u16_i32( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_sse_column_complex_u16_i32_impl( +fn filter_sse_column_complex_u16_i32_impl( arena: Arena, arena_src: &[&[Complex]], dst: &mut [u16], @@ -101,7 +105,7 @@ unsafe fn filter_sse_column_complex_u16_i32_impl( let shuf_table = _mm_set_epi8(29, 28, 25, 24, 21, 20, 17, 16, 13, 12, 9, 8, 5, 4, 1, 0); - while cx + 8 < full_width { + while cx + 8 <= full_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let values0 = _mm_loadu_si128(v_src.as_ptr().cast()); @@ -145,7 +149,7 @@ unsafe fn filter_sse_column_complex_u16_i32_impl( cx += 8; } - while cx + 4 < full_width { + while cx + 4 <= full_width { let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); let values0 = _mm_loadu_si128(v_src.as_ptr().cast()); diff --git a/src/filter1d/sse/filter_column_complex_u8_f32.rs b/src/filter1d/sse/filter_column_complex_u8_f32.rs index a3617e80..cd99b44c 100644 --- a/src/filter1d/sse/filter_column_complex_u8_f32.rs +++ b/src/filter1d/sse/filter_column_complex_u8_f32.rs @@ -37,28 +37,27 @@ use std::arch::x86::*; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn mq_complex_mla( - acc_r: __m128, - r0i0: __m128, - r: __m128, - r_swop: __m128, -) -> __m128 { - let c0 = _mm_mul_ps(r0i0, r); - let c1 = _mm_mul_ps( - _mm_shuffle_ps::<{ _shuffle(2, 3, 0, 1) }>(r0i0, r0i0), - r_swop, - ); - _mm_add_ps(acc_r, _mm_addsub_ps(c0, c1)) +pub(crate) fn mq_complex_mla(acc_r: __m128, r0i0: __m128, r: __m128, r_swop: __m128) -> __m128 { + unsafe { + let c0 = _mm_mul_ps(r0i0, r); + let c1 = _mm_mul_ps( + _mm_shuffle_ps::<{ _shuffle(2, 3, 0, 1) }>(r0i0, r0i0), + r_swop, + ); + _mm_add_ps(acc_r, _mm_addsub_ps(c0, c1)) + } } #[inline(always)] -pub(crate) unsafe fn mq_complex_mul(r0i0: __m128, r: __m128, r_swop: __m128) -> __m128 { - let c0 = _mm_mul_ps(r0i0, r); - let c1 = _mm_mul_ps( - _mm_shuffle_ps::<{ _shuffle(2, 3, 0, 1) }>(r0i0, r0i0), - r_swop, - ); - _mm_addsub_ps(c0, c1) +pub(crate) fn mq_complex_mul(r0i0: __m128, r: __m128, r_swop: __m128) -> __m128 { + unsafe { + let c0 = _mm_mul_ps(r0i0, r); + let c1 = _mm_mul_ps( + _mm_shuffle_ps::<{ _shuffle(2, 3, 0, 1) }>(r0i0, r0i0), + r_swop, + ); + _mm_addsub_ps(c0, c1) + } } pub(crate) fn filter_sse_column_complex_u8_f32( @@ -74,7 +73,7 @@ pub(crate) fn filter_sse_column_complex_u8_f32( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_sse_column_complex_u8_f32_impl( +fn filter_sse_column_complex_u8_f32_impl( arena: Arena, arena_src: &[&[Complex]], dst: &mut [u8], diff --git a/src/filter1d/sse/filter_column_complex_u8_i32.rs b/src/filter1d/sse/filter_column_complex_u8_i32.rs index b96ae1e9..bb88a5be 100644 --- a/src/filter1d/sse/filter_column_complex_u8_i32.rs +++ b/src/filter1d/sse/filter_column_complex_u8_i32.rs @@ -38,9 +38,11 @@ use std::arch::x86::*; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm_addsub_epi32(a: __m128i, b: __m128i) -> __m128i { - let c = _mm_sign_epi32(b, _mm_setr_epi32(-1, 1, -1, 1)); - _mm_add_epi32(a, c) +pub(crate) fn _mm_addsub_epi32(a: __m128i, b: __m128i) -> __m128i { + unsafe { + let c = _mm_sign_epi32(b, _mm_setr_epi32(-1, 1, -1, 1)); + _mm_add_epi32(a, c) + } } #[inline(always)] @@ -50,13 +52,15 @@ pub(crate) unsafe fn mq_complex_mla( r: __m128i, r_swop: __m128i, ) -> (__m128i, __m128i) { - let xc00 = _mm_madd_epi16(r0i0, r); - let xc10 = _mm_madd_epi16(r0i0, r_swop); + unsafe { + let xc00 = _mm_madd_epi16(r0i0, r); + let xc10 = _mm_madd_epi16(r0i0, r_swop); - let x_r = _mm_unpacklo_epi32(xc00, xc10); - let x_c = _mm_unpackhi_epi32(xc00, xc10); + let x_r = _mm_unpacklo_epi32(xc00, xc10); + let x_c = _mm_unpackhi_epi32(xc00, xc10); - (_mm_add_epi32(acc.0, x_r), _mm_add_epi32(acc.1, x_c)) + (_mm_add_epi32(acc.0, x_r), _mm_add_epi32(acc.1, x_c)) + } } pub(crate) fn filter_sse_column_complex_u8_i32( diff --git a/src/filter1d/sse/filter_column_symm.rs b/src/filter1d/sse/filter_column_symm.rs index 11ddf1c4..db6ae7c6 100644 --- a/src/filter1d/sse/filter_column_symm.rs +++ b/src/filter1d/sse/filter_column_symm.rs @@ -71,7 +71,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "sse4.1")] - unsafe fn pass( + fn pass( &self, arena: Arena, arena_src: &[&[u8]], @@ -80,214 +80,216 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let image_width = image_size.width * arena.components; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let mut cx = 0usize; - - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - - while cx + 64 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - - let source = _mm_load_pack_x4(v_src.as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); - let mut k3 = _mm_mul_epi8_by_ps_x4(source.3, coeff); - - for i in 0..half_len { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let rollback = length - i - 1; - let v_source0 = - _mm_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - let v_source1 = _mm_load_pack_x4( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr(), + unsafe { + let image_width = image_size.width * arena.components; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let mut cx = 0usize; + + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + + while cx + 64 <= image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + + let source = _mm_load_pack_x4(v_src.as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); + let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); + let mut k3 = _mm_mul_epi8_by_ps_x4(source.3, coeff); + + for i in 0..half_len { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let rollback = length - i - 1; + let v_source0 = + _mm_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + let v_source1 = _mm_load_pack_x4( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr(), + ); + k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi8_by_ps_x4(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm_mul_add_symm_epi8_by_ps_x4(k2, v_source0.2, v_source1.2, coeff); + k3 = _mm_mul_add_symm_epi8_by_ps_x4(k3, v_source0.3, v_source1.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0, + ( + _mm_pack_ps_x4_epi8(k0), + _mm_pack_ps_x4_epi8(k1), + _mm_pack_ps_x4_epi8(k2), + _mm_pack_ps_x4_epi8(k3), + ), ); - k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi8_by_ps_x4(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm_mul_add_symm_epi8_by_ps_x4(k2, v_source0.2, v_source1.2, coeff); - k3 = _mm_mul_add_symm_epi8_by_ps_x4(k3, v_source0.3, v_source1.3, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0, - ( - _mm_pack_ps_x4_epi8(k0), - _mm_pack_ps_x4_epi8(k1), - _mm_pack_ps_x4_epi8(k2), - _mm_pack_ps_x4_epi8(k3), - ), - ); - cx += 64; - } - - while cx + 48 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - - let source = _mm_load_pack_x3(v_src.as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); - - for i in 0..half_len { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let rollback = length - i - 1; - let v_source0 = - _mm_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - let v_source1 = _mm_load_pack_x3( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr(), + while cx + 48 <= image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + + let source = _mm_load_pack_x3(v_src.as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); + let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); + + for i in 0..half_len { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let rollback = length - i - 1; + let v_source0 = + _mm_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + let v_source1 = _mm_load_pack_x3( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr(), + ); + k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi8_by_ps_x4(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm_mul_add_symm_epi8_by_ps_x4(k2, v_source0.2, v_source1.2, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x3( + dst_ptr0, + ( + _mm_pack_ps_x4_epi8(k0), + _mm_pack_ps_x4_epi8(k1), + _mm_pack_ps_x4_epi8(k2), + ), ); - k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi8_by_ps_x4(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm_mul_add_symm_epi8_by_ps_x4(k2, v_source0.2, v_source1.2, coeff); + cx += 48; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x3( - dst_ptr0, - ( - _mm_pack_ps_x4_epi8(k0), - _mm_pack_ps_x4_epi8(k1), - _mm_pack_ps_x4_epi8(k2), - ), - ); - cx += 48; - } - - while cx + 32 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - - let source = _mm_load_pack_x2(v_src.as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - - for i in 0..half_len { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let rollback = length - i - 1; - let v_source0 = - _mm_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - let v_source1 = _mm_load_pack_x2( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr(), - ); - k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi8_by_ps_x4(k1, v_source0.1, v_source1.1, coeff); + while cx + 32 <= image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + + let source = _mm_load_pack_x2(v_src.as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); + + for i in 0..half_len { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let rollback = length - i - 1; + let v_source0 = + _mm_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + let v_source1 = _mm_load_pack_x2( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr(), + ); + k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi8_by_ps_x4(k1, v_source0.1, v_source1.1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2(dst_ptr0, (_mm_pack_ps_x4_epi8(k0), _mm_pack_ps_x4_epi8(k1))); + + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2(dst_ptr0, (_mm_pack_ps_x4_epi8(k0), _mm_pack_ps_x4_epi8(k1))); - - cx += 32; - } - - while cx + 16 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - - let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi8_by_ps_x4(source_0, coeff); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source_0 = _mm_loadu_si128( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - let v_source_1 = _mm_loadu_si128( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const __m128i, - ); - k0 = _mm_mul_add_symm_epi8_by_ps_x4( - k0, - v_source_0, - v_source_1, - _mm_set1_ps(coeff.weight), - ); + while cx + 16 <= image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + + let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi8_by_ps_x4(source_0, coeff); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source_0 = + _mm_loadu_si128(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() + as *const __m128i); + let v_source_1 = _mm_loadu_si128( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const __m128i, + ); + k0 = _mm_mul_add_symm_epi8_by_ps_x4( + k0, + v_source_0, + v_source_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); + cx += 16; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); - cx += 16; - } - - let coeff = scanned_kernel.get_unchecked(half_len).weight; - - while cx + 4 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - - let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff); - let mut k1 = (*v_src.get_unchecked(1) as f32).mul(coeff); - let mut k2 = (*v_src.get_unchecked(2) as f32).mul(coeff); - let mut k3 = (*v_src.get_unchecked(3) as f32).mul(coeff); + let coeff = scanned_kernel.get_unchecked(half_len).weight; - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - k0 = mlaf( - k0, - ((*arena_src.get_unchecked(i).get_unchecked(cx)) as f32) - .add(*arena_src.get_unchecked(rollback).get_unchecked(cx) as f32), - coeff.weight, - ); - k1 = mlaf( - k1, - ((*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as f32) - .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 1) as f32), - coeff.weight, - ); - k2 = mlaf( - k2, - ((*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as f32) - .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 2) as f32), - coeff.weight, - ); - k3 = mlaf( - k3, - ((*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as f32) - .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 3) as f32), - coeff.weight, - ); + while cx + 4 <= image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + + let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff); + let mut k1 = (*v_src.get_unchecked(1) as f32).mul(coeff); + let mut k2 = (*v_src.get_unchecked(2) as f32).mul(coeff); + let mut k3 = (*v_src.get_unchecked(3) as f32).mul(coeff); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + k0 = mlaf( + k0, + ((*arena_src.get_unchecked(i).get_unchecked(cx)) as f32) + .add(*arena_src.get_unchecked(rollback).get_unchecked(cx) as f32), + coeff.weight, + ); + k1 = mlaf( + k1, + ((*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as f32) + .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 1) as f32), + coeff.weight, + ); + k2 = mlaf( + k2, + ((*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as f32) + .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 2) as f32), + coeff.weight, + ); + k3 = mlaf( + k3, + ((*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as f32) + .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 3) as f32), + coeff.weight, + ); + } + + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } + for x in cx..image_width { + let coeff = scanned_kernel.get_unchecked(half_len).weight; - for x in cx..image_width { - let coeff = scanned_kernel.get_unchecked(half_len).weight; + let v_src = arena_src.get_unchecked(half_len).get_unchecked(x..); - let v_src = arena_src.get_unchecked(half_len).get_unchecked(x..); + let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff); - let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff); + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + k0 = mlaf( + k0, + ((*arena_src.get_unchecked(i).get_unchecked(x)) as f32) + .add(*arena_src.get_unchecked(rollback).get_unchecked(x) as f32), + coeff.weight, + ); + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - k0 = mlaf( - k0, - ((*arena_src.get_unchecked(i).get_unchecked(x)) as f32) - .add(*arena_src.get_unchecked(rollback).get_unchecked(x) as f32), - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/sse/filter_column_symm_approx.rs b/src/filter1d/sse/filter_column_symm_approx.rs index b4573ee7..a3323a3c 100644 --- a/src/filter1d/sse/filter_column_symm_approx.rs +++ b/src/filter1d/sse/filter_column_symm_approx.rs @@ -27,9 +27,7 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; -use crate::filter1d::sse::filter_column_symm_approx_uq0_7::filter_column_sse_symm_u8_uq0_7; use crate::filter1d::sse::utils::{ _mm_mul_add_symm_epi8_by_epi16_x4, _mm_mul_epi8_by_epi16_x4, _mm_pack_epi32_x2_epi8, }; @@ -51,22 +49,9 @@ pub(crate) fn filter_column_symm_u8_i32_app( dst: &mut [u8], image_size: ImageSize, filter_region: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { - if scanned_kernel.len() <= 7 { - let is_all_positive = scanned_kernel.iter().all(|&x| x.weight > 0); - if is_all_positive { - return filter_column_sse_symm_u8_uq0_7( - arena, - arena_src, - dst, - image_size, - filter_region, - scanned_kernel, - ); - } - } filter_column_symm_u8_i32_impl( arena, arena_src, @@ -85,7 +70,7 @@ unsafe fn filter_column_symm_u8_i32_impl( dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -95,15 +80,7 @@ unsafe fn filter_column_symm_u8_i32_impl( let mut cx = 0usize; - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.weight.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) - }) - .collect::>(); - - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(half_len)); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(half_len)); while cx + 64 < image_width { let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); @@ -116,7 +93,7 @@ unsafe fn filter_column_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); let v_source0 = _mm_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = _mm_load_pack_x4( @@ -154,7 +131,7 @@ unsafe fn filter_column_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); let v_source0 = _mm_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = _mm_load_pack_x3( @@ -189,7 +166,7 @@ unsafe fn filter_column_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); let v_source0 = _mm_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); let v_source1 = _mm_load_pack_x2( @@ -218,7 +195,7 @@ unsafe fn filter_column_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si128( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -237,33 +214,37 @@ unsafe fn filter_column_symm_u8_i32_impl( } let coeff = *scanned_kernel.get_unchecked(half_len); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]) as i32; while cx + 4 < image_width { let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - let mut k0 = (*v_src.get_unchecked(0) as i32).mul(coeff.weight); - let mut k1 = (*v_src.get_unchecked(1) as i32).mul(coeff.weight); - let mut k2 = (*v_src.get_unchecked(2) as i32).mul(coeff.weight); - let mut k3 = (*v_src.get_unchecked(3) as i32).mul(coeff.weight); + let mut k0 = (*v_src.get_unchecked(0) as i32).mul(wb); + let mut k1 = (*v_src.get_unchecked(1) as i32).mul(wb); + let mut k2 = (*v_src.get_unchecked(2) as i32).mul(wb); + let mut k3 = (*v_src.get_unchecked(3) as i32).mul(wb); for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]) as i32; let rollback = length - i - 1; k0 = ((*arena_src.get_unchecked(i).get_unchecked(cx)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(cx)) as i32) - .mul(coeff.weight) + .mul(wb) .add(k0); k1 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(cx + 1)) as i32) - .mul(coeff.weight) + .mul(wb) .add(k1); k2 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(cx + 2)) as i32) - .mul(coeff.weight) + .mul(wb) .add(k2); k3 = ((*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(cx + 3)) as i32) - .mul(coeff.weight) + .mul(wb) .add(k3); } @@ -277,14 +258,16 @@ unsafe fn filter_column_symm_u8_i32_impl( for x in cx..image_width { let v_src = arena_src.get_unchecked(half_len).get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as i32).mul(wb); for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; + let cb = coeff.to_ne_bytes(); + let wb = i16::from_ne_bytes([cb[0], cb[1]]); k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as i32) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as i32) - .mul(coeff.weight) + .mul(wb as i32) .add(k0); } diff --git a/src/filter1d/sse/filter_column_symm_approx_uq0_7.rs b/src/filter1d/sse/filter_column_symm_approx_uq0_7.rs index f76ef1e7..9edb7171 100644 --- a/src/filter1d/sse/filter_column_symm_approx_uq0_7.rs +++ b/src/filter1d/sse/filter_column_symm_approx_uq0_7.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::img_size::ImageSize; #[cfg(target_arch = "x86")] @@ -42,34 +41,16 @@ pub(crate) fn filter_column_sse_symm_u8_uq0_7( dst: &mut [u8], image_size: ImageSize, filter_region: FilterRegion, - scanned_kernel: &[ScanPoint1d], + kernel: &[i32], ) { unsafe { - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8).min(i8::MAX as i32) as i8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } filter_column_sse_symm_u8_i32_impl( arena, arena_src, dst, image_size, filter_region, - &shifted, + kernel, ); } } @@ -81,27 +62,19 @@ unsafe fn filter_column_sse_symm_u8_i32_impl( dst: &mut [u8], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[i8], + kernel: &[i32], ) { unsafe { let image_width = image_size.width * arena.components; - let length = scanned_kernel.len(); + let length = kernel.len(); let half_len = length / 2; let ref0 = arena_src.get_unchecked(half_len); - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[0], z[0], z[0]]) - }) - .collect::>(); - let mut cx = 0usize; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(half_len)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(half_len)); let rnd = _mm_set1_epi16(1 << 6); @@ -120,7 +93,7 @@ unsafe fn filter_column_sse_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si128( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -159,7 +132,7 @@ unsafe fn filter_column_sse_symm_u8_i32_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); + let coeff = _mm_set1_epi32(*kernel.get_unchecked(i)); let v_source0 = _mm_loadu_si64( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -185,19 +158,23 @@ unsafe fn filter_column_sse_symm_u8_i32_impl( cx += 8; } - let coeff = *scanned_kernel.get_unchecked(half_len); + let coeff = *kernel.get_unchecked(half_len); + let cb = coeff.to_ne_bytes(); + let wb = i8::from_ne_bytes([cb[0]]); for x in cx..image_width { let v_src = ref0.get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as u16).mul(coeff as u16); + let mut k0 = ((*v_src.get_unchecked(0)) as u16).mul(wb as u16); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = *kernel.get_unchecked(i); let rollback = length - i - 1; + let cb = coeff.to_ne_bytes(); + let wb = i8::from_ne_bytes([cb[0]]); k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as u16) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as u16) - .mul(coeff as u16) + .mul(wb as u16) .add(k0); } diff --git a/src/filter1d/sse/filter_column_symm_u16.rs b/src/filter1d/sse/filter_column_symm_u16.rs index 78ab7f0e..d8ffb9b9 100644 --- a/src/filter1d/sse/filter_column_symm_u16.rs +++ b/src/filter1d/sse/filter_column_symm_u16.rs @@ -79,7 +79,7 @@ pub(crate) fn filter_column_symm_sse_u16_f32( } #[target_feature(enable = "sse4.1", enable = "fma")] -unsafe fn filter_column_symm_sse_u16_f32_fma( +fn filter_column_symm_sse_u16_f32_fma( arena: Arena, arena_src: &[&[u16]], dst: &mut [u16], @@ -98,7 +98,7 @@ unsafe fn filter_column_symm_sse_u16_f32_fma( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_column_symm_sse_u16_f32_def( +fn filter_column_symm_sse_u16_f32_def( arena: Arena, arena_src: &[&[u16]], dst: &mut [u16], @@ -117,7 +117,7 @@ unsafe fn filter_column_symm_sse_u16_f32_def( } #[inline(always)] -unsafe fn filter_column_symm_sse_u16_f32_impl( +fn filter_column_symm_sse_u16_f32_impl( arena: Arena, arena_src: &[&[u16]], dst: &mut [u16], @@ -125,203 +125,206 @@ unsafe fn filter_column_symm_sse_u16_f32_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let image_width = image_size.width * arena.components; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let mut cx = 0usize; - - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - - while cx + 32 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - - let source = _mm_load_pack_x4(v_src.as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps_x2::(source.0, coeff); - let mut k1 = _mm_mul_epi16_by_ps_x2::(source.1, coeff); - let mut k2 = _mm_mul_epi16_by_ps_x2::(source.2, coeff); - let mut k3 = _mm_mul_epi16_by_ps_x2::(source.3, coeff); - - for i in 0..half_len { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let rollback = length - i - 1; - let v_source0 = _mm_load_pack_x4( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source1 = _mm_load_pack_x4( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const _, + unsafe { + let image_width = image_size.width * arena.components; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let mut cx = 0usize; + + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + + while cx + 32 <= image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + + let source = _mm_load_pack_x4(v_src.as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps_x2::(source.0, coeff); + let mut k1 = _mm_mul_epi16_by_ps_x2::(source.1, coeff); + let mut k2 = _mm_mul_epi16_by_ps_x2::(source.2, coeff); + let mut k3 = _mm_mul_epi16_by_ps_x2::(source.3, coeff); + + for i in 0..half_len { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let rollback = length - i - 1; + let v_source0 = _mm_load_pack_x4( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source1 = _mm_load_pack_x4( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm_mul_add_symm_epi16_by_ps_x2::(k2, v_source0.2, v_source1.2, coeff); + k3 = _mm_mul_add_symm_epi16_by_ps_x2::(k3, v_source0.3, v_source1.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0 as *mut _, + ( + _mm_pack_ps_x2_epi16(k0), + _mm_pack_ps_x2_epi16(k1), + _mm_pack_ps_x2_epi16(k2), + _mm_pack_ps_x2_epi16(k3), + ), ); - k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm_mul_add_symm_epi16_by_ps_x2::(k2, v_source0.2, v_source1.2, coeff); - k3 = _mm_mul_add_symm_epi16_by_ps_x2::(k3, v_source0.3, v_source1.3, coeff); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0 as *mut _, - ( - _mm_pack_ps_x2_epi16(k0), - _mm_pack_ps_x2_epi16(k1), - _mm_pack_ps_x2_epi16(k2), - _mm_pack_ps_x2_epi16(k3), - ), - ); - cx += 32; - } - - while cx + 24 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - - let source = _mm_load_pack_x3(v_src.as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps_x2::(source.0, coeff); - let mut k1 = _mm_mul_epi16_by_ps_x2::(source.1, coeff); - let mut k2 = _mm_mul_epi16_by_ps_x2::(source.2, coeff); - - for i in 0..half_len { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let rollback = length - i - 1; - let v_source0 = _mm_load_pack_x3( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, - ); - let v_source1 = _mm_load_pack_x3( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const _, + while cx + 24 <= image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + + let source = _mm_load_pack_x3(v_src.as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps_x2::(source.0, coeff); + let mut k1 = _mm_mul_epi16_by_ps_x2::(source.1, coeff); + let mut k2 = _mm_mul_epi16_by_ps_x2::(source.2, coeff); + + for i in 0..half_len { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let rollback = length - i - 1; + let v_source0 = _mm_load_pack_x3( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source1 = _mm_load_pack_x3( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm_mul_add_symm_epi16_by_ps_x2::(k2, v_source0.2, v_source1.2, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x3( + dst_ptr0 as *mut _, + ( + _mm_pack_ps_x2_epi16(k0), + _mm_pack_ps_x2_epi16(k1), + _mm_pack_ps_x2_epi16(k2), + ), ); - k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm_mul_add_symm_epi16_by_ps_x2::(k2, v_source0.2, v_source1.2, coeff); + cx += 24; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x3( - dst_ptr0 as *mut _, - ( - _mm_pack_ps_x2_epi16(k0), - _mm_pack_ps_x2_epi16(k1), - _mm_pack_ps_x2_epi16(k2), - ), - ); - cx += 24; - } - - while cx + 16 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - - let source = _mm_load_pack_x2(v_src.as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps_x2::(source.0, coeff); - let mut k1 = _mm_mul_epi16_by_ps_x2::(source.1, coeff); - - for i in 0..half_len { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let rollback = length - i - 1; - let v_source0 = _mm_load_pack_x2( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + while cx + 16 <= image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + + let source = _mm_load_pack_x2(v_src.as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps_x2::(source.0, coeff); + let mut k1 = _mm_mul_epi16_by_ps_x2::(source.1, coeff); + + for i in 0..half_len { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let rollback = length - i - 1; + let v_source0 = _mm_load_pack_x2( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source1 = _mm_load_pack_x2( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2( + dst_ptr0 as *mut _, + (_mm_pack_ps_x2_epi16(k0), _mm_pack_ps_x2_epi16(k1)), ); - let v_source1 = _mm_load_pack_x2( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); - } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2( - dst_ptr0 as *mut _, - (_mm_pack_ps_x2_epi16(k0), _mm_pack_ps_x2_epi16(k1)), - ); - - cx += 16; - } - - while cx + 8 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - - let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi16_by_ps_x2::(source_0, coeff); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source_0 = _mm_loadu_si128( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - let v_source_1 = _mm_loadu_si128( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const __m128i, - ); - k0 = _mm_mul_add_symm_epi16_by_ps_x2::( - k0, - v_source_0, - v_source_1, - _mm_set1_ps(coeff.weight), - ); + cx += 16; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x2_epi16(k0)); - cx += 8; - } - - let coeff = *scanned_kernel.get_unchecked(half_len); - - while cx + 4 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - - let source_0 = _mm_loadu_si64(v_src.as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps::(source_0, _mm_set1_ps(coeff.weight)); + while cx + 8 <= image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + + let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi16_by_ps_x2::(source_0, coeff); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source_0 = _mm_loadu_si128( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, + ); + let v_source_1 = _mm_loadu_si128( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const __m128i, + ); + k0 = _mm_mul_add_symm_epi16_by_ps_x2::( + k0, + v_source_0, + v_source_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_pack_ps_x2_epi16(k0)); + cx += 8; + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source_0 = - _mm_loadu_si64(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _); - let v_source_1 = _mm_loadu_si64( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi16_by_ps::( - k0, - v_source_0, - v_source_1, - _mm_set1_ps(coeff.weight), - ); + let coeff = *scanned_kernel.get_unchecked(half_len); + + while cx + 4 <= image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + + let source_0 = _mm_loadu_si64(v_src.as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps::(source_0, _mm_set1_ps(coeff.weight)); + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source_0 = _mm_loadu_si64( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, + ); + let v_source_1 = _mm_loadu_si64( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps::( + k0, + v_source_0, + v_source_1, + _mm_set1_ps(coeff.weight), + ); + } + + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr as *mut _, _mm_pack_ps_epi16(k0)); + cx += 4; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr as *mut _, _mm_pack_ps_epi16(k0)); - cx += 4; - } + for x in cx..image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(x..); - for x in cx..image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(x..); + let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff.weight); - let mut k0 = (*v_src.get_unchecked(0) as f32).mul(coeff.weight); + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + k0 = mlaf( + k0, + ((*arena_src.get_unchecked(i).get_unchecked(x)) as f32) + .add(*arena_src.get_unchecked(rollback).get_unchecked(x) as f32), + coeff.weight, + ); + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - k0 = mlaf( - k0, - ((*arena_src.get_unchecked(i).get_unchecked(x)) as f32) - .add(*arena_src.get_unchecked(rollback).get_unchecked(x) as f32), - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } diff --git a/src/filter1d/sse/filter_column_symm_u8_i16.rs b/src/filter1d/sse/filter_column_symm_u8_i16.rs index 80066885..1bcc1a6e 100644 --- a/src/filter1d/sse/filter_column_symm_u8_i16.rs +++ b/src/filter1d/sse/filter_column_symm_u8_i16.rs @@ -64,7 +64,7 @@ pub(crate) fn filter_column_symm_sse_u8_i16( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_column_symm_sse_u8_i16_impl( +fn filter_column_symm_sse_u8_i16_impl( arena: Arena, arena_src: &[&[u8]], dst: &mut [u8], @@ -72,214 +72,216 @@ unsafe fn filter_column_symm_sse_u8_i16_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let image_width = image_size.width * arena.components; + unsafe { + let image_width = image_size.width * arena.components; + + let length = scanned_kernel.len(); + let half_len = length / 2; - let length = scanned_kernel.len(); - let half_len = length / 2; + let mut cx = 0usize; - let mut cx = 0usize; + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(half_len).weight); - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(half_len).weight); + while cx + 64 < image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - while cx + 64 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + let source = _mm_load_pack_x4(v_src.as_ptr()); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); + let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); + let mut k3 = _mm_madd_epi8_by_epi16_x4(source.3, coeff); - let source = _mm_load_pack_x4(v_src.as_ptr()); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); - let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); - let mut k3 = _mm_madd_epi8_by_epi16_x4(source.3, coeff); + for i in 0..half_len { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); + let rollback = length - i - 1; + let v_source0 = + _mm_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + let v_source1 = _mm_load_pack_x4( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr(), + ); + k0 = _mm_madd_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_madd_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm_madd_symm_epi8_by_epi16_x4(k2, v_source0.2, v_source1.2, coeff); + k3 = _mm_madd_symm_epi8_by_epi16_x4(k3, v_source0.3, v_source1.3, coeff); + } - for i in 0..half_len { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); - let rollback = length - i - 1; - let v_source0 = - _mm_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - let v_source1 = _mm_load_pack_x4( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr(), + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0, + ( + _mm_packus_epi16(k0.0, k0.1), + _mm_packus_epi16(k1.0, k1.1), + _mm_packus_epi16(k2.0, k2.1), + _mm_packus_epi16(k3.0, k3.1), + ), ); - k0 = _mm_madd_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_madd_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm_madd_symm_epi8_by_epi16_x4(k2, v_source0.2, v_source1.2, coeff); - k3 = _mm_madd_symm_epi8_by_epi16_x4(k3, v_source0.3, v_source1.3, coeff); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0, - ( - _mm_packus_epi16(k0.0, k0.1), - _mm_packus_epi16(k1.0, k1.1), - _mm_packus_epi16(k2.0, k2.1), - _mm_packus_epi16(k3.0, k3.1), - ), - ); - cx += 64; - } + while cx + 48 < image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - while cx + 48 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + let source = _mm_load_pack_x3(v_src.as_ptr()); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); + let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); - let source = _mm_load_pack_x3(v_src.as_ptr()); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); - let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); + for i in 0..half_len { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); + let rollback = length - i - 1; + let v_source0 = + _mm_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + let v_source1 = _mm_load_pack_x3( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr(), + ); + k0 = _mm_madd_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_madd_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm_madd_symm_epi8_by_epi16_x4(k2, v_source0.2, v_source1.2, coeff); + } - for i in 0..half_len { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); - let rollback = length - i - 1; - let v_source0 = - _mm_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - let v_source1 = _mm_load_pack_x3( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr(), + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x3( + dst_ptr0, + ( + _mm_packus_epi16(k0.0, k0.1), + _mm_packus_epi16(k1.0, k1.1), + _mm_packus_epi16(k2.0, k2.1), + ), ); - k0 = _mm_madd_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_madd_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm_madd_symm_epi8_by_epi16_x4(k2, v_source0.2, v_source1.2, coeff); + cx += 48; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x3( - dst_ptr0, - ( - _mm_packus_epi16(k0.0, k0.1), - _mm_packus_epi16(k1.0, k1.1), - _mm_packus_epi16(k2.0, k2.1), - ), - ); - cx += 48; - } + while cx + 32 < image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - while cx + 32 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + let source = _mm_load_pack_x2(v_src.as_ptr()); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); - let source = _mm_load_pack_x2(v_src.as_ptr()); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); + for i in 0..half_len { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); + let rollback = length - i - 1; + let v_source0 = + _mm_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + let v_source1 = _mm_load_pack_x2( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr(), + ); + k0 = _mm_madd_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_madd_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); + } - for i in 0..half_len { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); - let rollback = length - i - 1; - let v_source0 = - _mm_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - let v_source1 = _mm_load_pack_x2( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr(), + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2( + dst_ptr0, + (_mm_packus_epi16(k0.0, k0.1), _mm_packus_epi16(k1.0, k1.1)), ); - k0 = _mm_madd_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_madd_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); + + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2( - dst_ptr0, - (_mm_packus_epi16(k0.0, k0.1), _mm_packus_epi16(k1.0, k1.1)), - ); + while cx + 16 < image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - cx += 32; - } + let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source_0, coeff); - while cx + 16 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + let v_source_0 = _mm_loadu_si128( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, + ); + let v_source_1 = _mm_loadu_si128( + arena_src + .get_unchecked(rollback) + .get_unchecked(cx..) + .as_ptr() as *const __m128i, + ); + k0 = _mm_madd_symm_epi8_by_epi16_x4( + k0, + v_source_0, + v_source_1, + _mm_set1_epi16(coeff.weight), + ); + } - let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source_0, coeff); - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - let v_source_0 = _mm_loadu_si128( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - let v_source_1 = _mm_loadu_si128( - arena_src - .get_unchecked(rollback) - .get_unchecked(cx..) - .as_ptr() as *const __m128i, - ); - k0 = _mm_madd_symm_epi8_by_epi16_x4( - k0, - v_source_0, - v_source_1, - _mm_set1_epi16(coeff.weight), - ); + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_packus_epi16(k0.0, k0.1)); + cx += 16; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_packus_epi16(k0.0, k0.1)); - cx += 16; - } + let coeff = scanned_kernel.get_unchecked(half_len).weight; - let coeff = scanned_kernel.get_unchecked(half_len).weight; + while cx + 4 < image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); - while cx + 4 < image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); + let mut k0 = (*v_src.get_unchecked(0) as i16).mul(coeff); + let mut k1 = (*v_src.get_unchecked(1) as i16).mul(coeff); + let mut k2 = (*v_src.get_unchecked(2) as i16).mul(coeff); + let mut k3 = (*v_src.get_unchecked(3) as i16).mul(coeff); - let mut k0 = (*v_src.get_unchecked(0) as i16).mul(coeff); - let mut k1 = (*v_src.get_unchecked(1) as i16).mul(coeff); - let mut k2 = (*v_src.get_unchecked(2) as i16).mul(coeff); - let mut k3 = (*v_src.get_unchecked(3) as i16).mul(coeff); + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + k0 = mlaf( + k0, + ((*arena_src.get_unchecked(i).get_unchecked(cx)) as i16) + .add(*arena_src.get_unchecked(rollback).get_unchecked(cx) as i16), + coeff.weight, + ); + k1 = mlaf( + k1, + ((*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as i16) + .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 1) as i16), + coeff.weight, + ); + k2 = mlaf( + k2, + ((*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as i16) + .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 2) as i16), + coeff.weight, + ); + k3 = mlaf( + k3, + ((*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as i16) + .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 3) as i16), + coeff.weight, + ); + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - k0 = mlaf( - k0, - ((*arena_src.get_unchecked(i).get_unchecked(cx)) as i16) - .add(*arena_src.get_unchecked(rollback).get_unchecked(cx) as i16), - coeff.weight, - ); - k1 = mlaf( - k1, - ((*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as i16) - .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 1) as i16), - coeff.weight, - ); - k2 = mlaf( - k2, - ((*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as i16) - .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 2) as i16), - coeff.weight, - ); - k3 = mlaf( - k3, - ((*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as i16) - .add(*arena_src.get_unchecked(rollback).get_unchecked(cx + 3) as i16), - coeff.weight, - ); + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } + for x in cx..image_width { + let v_src = arena_src.get_unchecked(half_len).get_unchecked(x..); - for x in cx..image_width { - let v_src = arena_src.get_unchecked(half_len).get_unchecked(x..); + let mut k0 = (*v_src.get_unchecked(0) as i16).mul(coeff); - let mut k0 = (*v_src.get_unchecked(0) as i16).mul(coeff); + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + k0 = mlaf( + k0, + ((*arena_src.get_unchecked(i).get_unchecked(x)) as i16) + .add(*arena_src.get_unchecked(rollback).get_unchecked(x) as i16), + coeff.weight, + ); + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - k0 = mlaf( - k0, - ((*arena_src.get_unchecked(i).get_unchecked(x)) as i16) - .add(*arena_src.get_unchecked(rollback).get_unchecked(x) as i16), - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } diff --git a/src/filter1d/sse/filter_column_symm_uq15_u16.rs b/src/filter1d/sse/filter_column_symm_uq15_u16.rs index 208e43b2..0ef2e9f1 100644 --- a/src/filter1d/sse/filter_column_symm_uq15_u16.rs +++ b/src/filter1d/sse/filter_column_symm_uq15_u16.rs @@ -27,7 +27,6 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter1d::arena::Arena; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::region::FilterRegion; use crate::filter1d::sse::utils::{ _mm_mul_add_symm_epu16_by_epu16_x4, _mm_mul_epu16_widen, _mm_pack_epi32_x2_epu16, @@ -46,7 +45,7 @@ pub(crate) fn filter_column_sse_symm_uq15_u16( dst: &mut [u16], image_size: ImageSize, filter_region: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[u32], ) { unsafe { filter_column_sse_symm_uq15_u16_impl( @@ -67,7 +66,7 @@ unsafe fn filter_column_sse_symm_uq15_u16_impl( dst: &mut [u16], image_size: ImageSize, _: FilterRegion, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[u32], ) { unsafe { let image_width = image_size.width * arena.components; @@ -77,7 +76,7 @@ unsafe fn filter_column_sse_symm_uq15_u16_impl( let mut cx = 0usize; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(half_len).weight as i32); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(half_len) as i32); while cx + 16 < image_width { let v_src = arena_src.get_unchecked(half_len).get_unchecked(cx..); @@ -90,7 +89,7 @@ unsafe fn filter_column_sse_symm_uq15_u16_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let src0 = arena_src.get_unchecked(i).get_unchecked(cx..); let r_src0 = arena_src.get_unchecked(rollback).get_unchecked(cx..); @@ -121,7 +120,7 @@ unsafe fn filter_column_sse_symm_uq15_u16_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0 = _mm_loadu_si128( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, ); @@ -147,7 +146,7 @@ unsafe fn filter_column_sse_symm_uq15_u16_impl( for i in 0..half_len { let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i) as i32); let v_source0 = _mm_loadu_si64( arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const _, ); @@ -186,14 +185,14 @@ unsafe fn filter_column_sse_symm_uq15_u16_impl( for x in cx..image_width { let v_src = arena_src.get_unchecked(half_len).get_unchecked(x..); - let mut k0 = ((*v_src.get_unchecked(0)) as u32).mul(coeff.weight); + let mut k0 = ((*v_src.get_unchecked(0)) as u32).mul(coeff); for i in 0..half_len { let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; k0 = ((*arena_src.get_unchecked(i).get_unchecked(x)) as u32) .add((*arena_src.get_unchecked(rollback).get_unchecked(x)) as u32) - .mul(coeff.weight) + .mul(coeff) .add(k0); } diff --git a/src/filter1d/sse/filter_column_u8_i16.rs b/src/filter1d/sse/filter_column_u8_i16.rs index 935a3b45..da374ec8 100644 --- a/src/filter1d/sse/filter_column_u8_i16.rs +++ b/src/filter1d/sse/filter_column_u8_i16.rs @@ -64,7 +64,7 @@ pub(crate) fn filter_column_sse_u8_i16( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_column_sse_u8_i16_impl( +fn filter_column_sse_u8_i16_impl( arena: Arena, arena_src: &[&[u8]], dst: &mut [u8], @@ -72,173 +72,175 @@ unsafe fn filter_column_sse_u8_i16_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let image_width = image_size.width * arena.components; - - let length = scanned_kernel.len(); - - let mut cx = 0usize; - - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(0).weight); - - while cx + 64 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let source = _mm_load_pack_x4(v_src.as_ptr()); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); - let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); - let mut k3 = _mm_madd_epi8_by_epi16_x4(source.3, coeff); - - for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); - let v_source = - _mm_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source.0, coeff); - k1 = _mm_madd_s_epi8_by_epi16_x4(k1, v_source.1, coeff); - k2 = _mm_madd_s_epi8_by_epi16_x4(k2, v_source.2, coeff); - k3 = _mm_madd_s_epi8_by_epi16_x4(k3, v_source.3, coeff); + unsafe { + let image_width = image_size.width * arena.components; + + let length = scanned_kernel.len(); + + let mut cx = 0usize; + + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(0).weight); + + while cx + 64 < image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let source = _mm_load_pack_x4(v_src.as_ptr()); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); + let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); + let mut k3 = _mm_madd_epi8_by_epi16_x4(source.3, coeff); + + for i in 1..length { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); + let v_source = + _mm_load_pack_x4(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source.0, coeff); + k1 = _mm_madd_s_epi8_by_epi16_x4(k1, v_source.1, coeff); + k2 = _mm_madd_s_epi8_by_epi16_x4(k2, v_source.2, coeff); + k3 = _mm_madd_s_epi8_by_epi16_x4(k3, v_source.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0, + ( + _mm_packus_epi16(k0.0, k0.1), + _mm_packus_epi16(k1.0, k1.1), + _mm_packus_epi16(k2.0, k2.1), + _mm_packus_epi16(k3.0, k3.1), + ), + ); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0, - ( - _mm_packus_epi16(k0.0, k0.1), - _mm_packus_epi16(k1.0, k1.1), - _mm_packus_epi16(k2.0, k2.1), - _mm_packus_epi16(k3.0, k3.1), - ), - ); - cx += 64; - } - - while cx + 48 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let source = _mm_load_pack_x3(v_src.as_ptr()); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); - let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); - - for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); - let v_source = - _mm_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source.0, coeff); - k1 = _mm_madd_s_epi8_by_epi16_x4(k1, v_source.1, coeff); - k2 = _mm_madd_s_epi8_by_epi16_x4(k2, v_source.2, coeff); + while cx + 48 < image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let source = _mm_load_pack_x3(v_src.as_ptr()); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); + let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); + + for i in 1..length { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); + let v_source = + _mm_load_pack_x3(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source.0, coeff); + k1 = _mm_madd_s_epi8_by_epi16_x4(k1, v_source.1, coeff); + k2 = _mm_madd_s_epi8_by_epi16_x4(k2, v_source.2, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x3( + dst_ptr0, + ( + _mm_packus_epi16(k0.0, k0.1), + _mm_packus_epi16(k1.0, k1.1), + _mm_packus_epi16(k2.0, k2.1), + ), + ); + cx += 48; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x3( - dst_ptr0, - ( - _mm_packus_epi16(k0.0, k0.1), - _mm_packus_epi16(k1.0, k1.1), - _mm_packus_epi16(k2.0, k2.1), - ), - ); - cx += 48; - } - - while cx + 32 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let source = _mm_load_pack_x2(v_src.as_ptr()); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); + while cx + 32 < image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let source = _mm_load_pack_x2(v_src.as_ptr()); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); + + for i in 1..length { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); + let v_source = + _mm_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); + k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source.0, coeff); + k1 = _mm_madd_s_epi8_by_epi16_x4(k1, v_source.1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2( + dst_ptr0, + (_mm_packus_epi16(k0.0, k0.1), _mm_packus_epi16(k1.0, k1.1)), + ); - for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); - let v_source = - _mm_load_pack_x2(arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr()); - k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source.0, coeff); - k1 = _mm_madd_s_epi8_by_epi16_x4(k1, v_source.1, coeff); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2( - dst_ptr0, - (_mm_packus_epi16(k0.0, k0.1), _mm_packus_epi16(k1.0, k1.1)), - ); - - cx += 32; - } + while cx + 16 < image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - while cx + 16 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source_0, coeff); - let source_0 = _mm_loadu_si128(v_src.as_ptr() as *const __m128i); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source_0, coeff); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + let v_source_0 = _mm_loadu_si128( + arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, + ); + k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source_0, _mm_set1_epi16(coeff.weight)); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - let v_source_0 = _mm_loadu_si128( - arena_src.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, - ); - k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source_0, _mm_set1_epi16(coeff.weight)); + let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_packus_epi16(k0.0, k0.1)); + cx += 16; } - let dst_ptr = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr as *mut __m128i, _mm_packus_epi16(k0.0, k0.1)); - cx += 16; - } - - let coeff = *scanned_kernel.get_unchecked(0); - - while cx + 4 < image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); - - let mut k0 = (*v_src.get_unchecked(0) as i16).mul(coeff.weight); - let mut k1 = (*v_src.get_unchecked(1) as i16).mul(coeff.weight); - let mut k2 = (*v_src.get_unchecked(2) as i16).mul(coeff.weight); - let mut k3 = (*v_src.get_unchecked(3) as i16).mul(coeff.weight); - - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = mlaf( - k0, - (*arena_src.get_unchecked(i).get_unchecked(cx)) as i16, - coeff.weight, - ); - k1 = mlaf( - k1, - (*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as i16, - coeff.weight, - ); - k2 = mlaf( - k2, - (*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as i16, - coeff.weight, - ); - k3 = mlaf( - k3, - (*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as i16, - coeff.weight, - ); + let coeff = *scanned_kernel.get_unchecked(0); + + while cx + 4 < image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(cx..); + + let mut k0 = (*v_src.get_unchecked(0) as i16).mul(coeff.weight); + let mut k1 = (*v_src.get_unchecked(1) as i16).mul(coeff.weight); + let mut k2 = (*v_src.get_unchecked(2) as i16).mul(coeff.weight); + let mut k3 = (*v_src.get_unchecked(3) as i16).mul(coeff.weight); + + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 = mlaf( + k0, + (*arena_src.get_unchecked(i).get_unchecked(cx)) as i16, + coeff.weight, + ); + k1 = mlaf( + k1, + (*arena_src.get_unchecked(i).get_unchecked(cx + 1)) as i16, + coeff.weight, + ); + k2 = mlaf( + k2, + (*arena_src.get_unchecked(i).get_unchecked(cx + 2)) as i16, + coeff.weight, + ); + k3 = mlaf( + k3, + (*arena_src.get_unchecked(i).get_unchecked(cx + 3)) as i16, + coeff.weight, + ); + } + + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } + for x in cx..image_width { + let v_src = arena_src.get_unchecked(0).get_unchecked(x..); - for x in cx..image_width { - let v_src = arena_src.get_unchecked(0).get_unchecked(x..); + let mut k0 = ((*v_src.get_unchecked(0)) as i16).mul(coeff.weight); - let mut k0 = ((*v_src.get_unchecked(0)) as i16).mul(coeff.weight); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 = mlaf( + k0, + (*arena_src.get_unchecked(i).get_unchecked(cx)) as i16, + coeff.weight, + ); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = mlaf( - k0, - (*arena_src.get_unchecked(i).get_unchecked(cx)) as i16, - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } diff --git a/src/filter1d/sse/filter_row.rs b/src/filter1d/sse/filter_row.rs index 89a97a35..fd2cbf57 100644 --- a/src/filter1d/sse/filter_row.rs +++ b/src/filter1d/sse/filter_row.rs @@ -68,7 +68,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "sse4.1")] - unsafe fn pass( + fn pass( &self, _: Arena, arena_src: &[u8], @@ -77,132 +77,135 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let src = arena_src; - - let local_src = src; - - let length = scanned_kernel.len(); - - let max_width = image_size.width * N; - - let mut cx = 0usize; - - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(0).weight); - - while cx + 64 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_load_pack_x4(shifted_src.as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); - let mut k3 = _mm_mul_epi8_by_ps_x4(source.3, coeff); - - for i in 1..length { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm_load_pack_x4(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source.0, coeff); - k1 = _mm_mul_add_epi8_by_ps_x4(k1, v_source.1, coeff); - k2 = _mm_mul_add_epi8_by_ps_x4(k2, v_source.2, coeff); - k3 = _mm_mul_add_epi8_by_ps_x4(k3, v_source.3, coeff); + unsafe { + let src = arena_src; + + let local_src = src; + + let length = scanned_kernel.len(); + + let max_width = image_size.width * N; + + let mut cx = 0usize; + + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(0).weight); + + while cx + 64 < max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = _mm_load_pack_x4(shifted_src.as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); + let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); + let mut k3 = _mm_mul_epi8_by_ps_x4(source.3, coeff); + + for i in 1..length { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm_load_pack_x4(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source.0, coeff); + k1 = _mm_mul_add_epi8_by_ps_x4(k1, v_source.1, coeff); + k2 = _mm_mul_add_epi8_by_ps_x4(k2, v_source.2, coeff); + k3 = _mm_mul_add_epi8_by_ps_x4(k3, v_source.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0, + ( + _mm_pack_ps_x4_epi8(k0), + _mm_pack_ps_x4_epi8(k1), + _mm_pack_ps_x4_epi8(k2), + _mm_pack_ps_x4_epi8(k3), + ), + ); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0, - ( - _mm_pack_ps_x4_epi8(k0), - _mm_pack_ps_x4_epi8(k1), - _mm_pack_ps_x4_epi8(k2), - _mm_pack_ps_x4_epi8(k3), - ), - ); - cx += 64; - } + while cx + 32 < max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm_load_pack_x2(shifted_src.as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - let source = _mm_load_pack_x2(shifted_src.as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); + for i in 1..length { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm_load_pack_x2(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source.0, coeff); + k1 = _mm_mul_add_epi8_by_ps_x4(k1, v_source.1, coeff); + } - for i in 1..length { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm_load_pack_x2(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source.0, coeff); - k1 = _mm_mul_add_epi8_by_ps_x4(k1, v_source.1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2(dst_ptr0, (_mm_pack_ps_x4_epi8(k0), _mm_pack_ps_x4_epi8(k1))); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2(dst_ptr0, (_mm_pack_ps_x4_epi8(k0), _mm_pack_ps_x4_epi8(k1))); - cx += 32; - } + while cx + 16 < max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source_0 = _mm_loadu_si128(shifted_src.as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi8_by_ps_x4(source_0, coeff); - let source_0 = _mm_loadu_si128(shifted_src.as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi8_by_ps_x4(source_0, coeff); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + let v_source_0 = _mm_loadu_si128( + shifted_src.get_unchecked(i * N..).as_ptr() as *const __m128i + ); + k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source_0, _mm_set1_ps(coeff.weight)); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - let v_source_0 = - _mm_loadu_si128(shifted_src.get_unchecked(i * N..).as_ptr() as *const __m128i); - k0 = _mm_mul_add_epi8_by_ps_x4(k0, v_source_0, _mm_set1_ps(coeff.weight)); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); - cx += 16; - } - - let coeff = *scanned_kernel.get_unchecked(0); - - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let mut k0 = ((*shifted_src.get_unchecked(0)) as f32).mul(coeff.weight); - let mut k1 = ((*shifted_src.get_unchecked(1)) as f32).mul(coeff.weight); - let mut k2 = ((*shifted_src.get_unchecked(2)) as f32).mul(coeff.weight); - let mut k3 = ((*shifted_src.get_unchecked(3)) as f32).mul(coeff.weight); - - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = mlaf(k0, (*shifted_src.get_unchecked(i * N)) as f32, coeff.weight); - k1 = mlaf( - k1, - (*shifted_src.get_unchecked(i * N + 1)) as f32, - coeff.weight, - ); - k2 = mlaf( - k2, - (*shifted_src.get_unchecked(i * N + 2)) as f32, - coeff.weight, - ); - k3 = mlaf( - k3, - (*shifted_src.get_unchecked(i * N + 3)) as f32, - coeff.weight, - ); + let coeff = *scanned_kernel.get_unchecked(0); + + while cx + 4 < max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let mut k0 = ((*shifted_src.get_unchecked(0)) as f32).mul(coeff.weight); + let mut k1 = ((*shifted_src.get_unchecked(1)) as f32).mul(coeff.weight); + let mut k2 = ((*shifted_src.get_unchecked(2)) as f32).mul(coeff.weight); + let mut k3 = ((*shifted_src.get_unchecked(3)) as f32).mul(coeff.weight); + + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 = mlaf(k0, (*shifted_src.get_unchecked(i * N)) as f32, coeff.weight); + k1 = mlaf( + k1, + (*shifted_src.get_unchecked(i * N + 1)) as f32, + coeff.weight, + ); + k2 = mlaf( + k2, + (*shifted_src.get_unchecked(i * N + 2)) as f32, + coeff.weight, + ); + k3 = mlaf( + k3, + (*shifted_src.get_unchecked(i * N + 3)) as f32, + coeff.weight, + ); + } + + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } - - for x in cx..max_width { - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = ((*shifted_src.get_unchecked(0)) as f32).mul(coeff.weight); + for x in cx..max_width { + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = ((*shifted_src.get_unchecked(0)) as f32).mul(coeff.weight); - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 = mlaf(k0, (*shifted_src.get_unchecked(i * N)) as f32, coeff.weight); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 = mlaf(k0, (*shifted_src.get_unchecked(i * N)) as f32, coeff.weight); + } + *dst.get_unchecked_mut(x) = k0.to_(); } - *dst.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/sse/filter_row_approx.rs b/src/filter1d/sse/filter_row_approx.rs index ba5a45a2..c2d213f9 100644 --- a/src/filter1d/sse/filter_row_approx.rs +++ b/src/filter1d/sse/filter_row_approx.rs @@ -61,7 +61,7 @@ pub(crate) fn filter_row_sse_u8_i32( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_row_sse_u8_i32_impl( +fn filter_row_sse_u8_i32_impl( _: Arena, arena_src: &[u8], dst: &mut [u8], @@ -69,125 +69,127 @@ unsafe fn filter_row_sse_u8_i32_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let src = arena_src; - - let local_src = src; - - let length = scanned_kernel.iter().len(); - - let mut cx = 0usize; - - let max_width = image_size.width * N; - - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(0).weight as i16); - - while cx + 64 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_load_pack_x4(shifted_src.as_ptr()); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_epi16_x4(source.1, coeff); - let mut k2 = _mm_mul_epi8_by_epi16_x4(source.2, coeff); - let mut k3 = _mm_mul_epi8_by_epi16_x4(source.3, coeff); - - for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); - let v_source = _mm_load_pack_x4(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); - k1 = _mm_mul_add_epi8_by_epi16_x4(k1, v_source.1, coeff); - k2 = _mm_mul_add_epi8_by_epi16_x4(k2, v_source.2, coeff); - k3 = _mm_mul_add_epi8_by_epi16_x4(k3, v_source.3, coeff); + unsafe { + let src = arena_src; + + let local_src = src; + + let length = scanned_kernel.iter().len(); + + let mut cx = 0usize; + + let max_width = image_size.width * N; + + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(0).weight as i16); + + while cx + 64 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = _mm_load_pack_x4(shifted_src.as_ptr()); + let mut k0 = _mm_mul_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_epi16_x4(source.1, coeff); + let mut k2 = _mm_mul_epi8_by_epi16_x4(source.2, coeff); + let mut k3 = _mm_mul_epi8_by_epi16_x4(source.3, coeff); + + for i in 1..length { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let v_source = _mm_load_pack_x4(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); + k1 = _mm_mul_add_epi8_by_epi16_x4(k1, v_source.1, coeff); + k2 = _mm_mul_add_epi8_by_epi16_x4(k2, v_source.2, coeff); + k3 = _mm_mul_add_epi8_by_epi16_x4(k3, v_source.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0, + ( + _mm_pack_epi32_x2_epi8(k0), + _mm_pack_epi32_x2_epi8(k1), + _mm_pack_epi32_x2_epi8(k2), + _mm_pack_epi32_x2_epi8(k3), + ), + ); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0, - ( - _mm_pack_epi32_x2_epi8(k0), - _mm_pack_epi32_x2_epi8(k1), - _mm_pack_epi32_x2_epi8(k2), - _mm_pack_epi32_x2_epi8(k3), - ), - ); - cx += 64; - } - - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_load_pack_x2(shifted_src.as_ptr()); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_epi16_x4(source.1, coeff); - - for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); - let v_source = _mm_load_pack_x2(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); - k1 = _mm_mul_add_epi8_by_epi16_x4(k1, v_source.1, coeff); + while cx + 32 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = _mm_load_pack_x2(shifted_src.as_ptr()); + let mut k0 = _mm_mul_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_epi16_x4(source.1, coeff); + + for i in 1..length { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let v_source = _mm_load_pack_x2(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source.0, coeff); + k1 = _mm_mul_add_epi8_by_epi16_x4(k1, v_source.1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2( + dst_ptr0, + (_mm_pack_epi32_x2_epi8(k0), _mm_pack_epi32_x2_epi8(k1)), + ); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2( - dst_ptr0, - (_mm_pack_epi32_x2_epi8(k0), _mm_pack_epi32_x2_epi8(k1)), - ); - cx += 32; - } + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm_loadu_si128(shifted_src.as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi8_by_epi16_x4(source, coeff); - let source = _mm_loadu_si128(shifted_src.as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source, coeff); + for i in 1..length { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let v_source = + _mm_loadu_si128(shifted_src.get_unchecked(i * N..).as_ptr() as *const __m128i); + k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source, coeff); + } - for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); - let v_source = - _mm_loadu_si128(shifted_src.get_unchecked(i * N..).as_ptr() as *const __m128i); - k0 = _mm_mul_add_epi8_by_epi16_x4(k0, v_source, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_epi32_x2_epi8(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_epi32_x2_epi8(k0)); - cx += 16; - } + let coeff = *scanned_kernel.get_unchecked(0); - let coeff = *scanned_kernel.get_unchecked(0); + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let mut k0 = *shifted_src.get_unchecked(0) as i32 * coeff.weight; + let mut k1 = *shifted_src.get_unchecked(1) as i32 * coeff.weight; + let mut k2 = *shifted_src.get_unchecked(2) as i32 * coeff.weight; + let mut k3 = *shifted_src.get_unchecked(3) as i32 * coeff.weight; - let mut k0 = *shifted_src.get_unchecked(0) as i32 * coeff.weight; - let mut k1 = *shifted_src.get_unchecked(1) as i32 * coeff.weight; - let mut k2 = *shifted_src.get_unchecked(2) as i32 * coeff.weight; - let mut k3 = *shifted_src.get_unchecked(3) as i32 * coeff.weight; + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); + k0 += *shifted_src.get_unchecked(i * N) as i32 * coeff.weight; + k1 += *shifted_src.get_unchecked(i * N + 1) as i32 * coeff.weight; + k2 += *shifted_src.get_unchecked(i * N + 2) as i32 * coeff.weight; + k3 += *shifted_src.get_unchecked(i * N + 3) as i32 * coeff.weight; + } - k0 += *shifted_src.get_unchecked(i * N) as i32 * coeff.weight; - k1 += *shifted_src.get_unchecked(i * N + 1) as i32 * coeff.weight; - k2 += *shifted_src.get_unchecked(i * N + 2) as i32 * coeff.weight; - k3 += *shifted_src.get_unchecked(i * N + 3) as i32 * coeff.weight; + *dst.get_unchecked_mut(cx) = k0.to_approx_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_approx_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_approx_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_approx_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_approx_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_approx_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_approx_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_approx_(); - cx += 4; - } + for x in cx..max_width { + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(0) as i32 * coeff.weight; - for x in cx..max_width { - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(0) as i32 * coeff.weight; + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); + k0 += (*shifted_src.get_unchecked(i * N) as i32) * coeff.weight; + } - k0 += (*shifted_src.get_unchecked(i * N) as i32) * coeff.weight; + *dst.get_unchecked_mut(x) = k0.to_approx_(); } - - *dst.get_unchecked_mut(x) = k0.to_approx_(); } } diff --git a/src/filter1d/sse/filter_row_complex_u8_i32.rs b/src/filter1d/sse/filter_row_complex_u8_i32.rs index 9cbfd43c..1c954bcc 100644 --- a/src/filter1d/sse/filter_row_complex_u8_i32.rs +++ b/src/filter1d/sse/filter_row_complex_u8_i32.rs @@ -47,21 +47,25 @@ pub(crate) fn filter_sse_row_complex_u8_i32( } #[inline(always)] -pub(crate) unsafe fn widen_mul(a: __m128i, b: __m128i) -> (__m128i, __m128i) { - let lo = _mm_mullo_epi16(a, b); - let hi = _mm_mulhi_epi16(a, b); - (_mm_unpacklo_epi16(lo, hi), _mm_unpackhi_epi16(lo, hi)) +pub(crate) fn widen_mul(a: __m128i, b: __m128i) -> (__m128i, __m128i) { + unsafe { + let lo = _mm_mullo_epi16(a, b); + let hi = _mm_mulhi_epi16(a, b); + (_mm_unpacklo_epi16(lo, hi), _mm_unpackhi_epi16(lo, hi)) + } } #[inline(always)] -pub(crate) unsafe fn widen_mul_lo(a: __m128i, b: __m128i) -> __m128i { - let lo = _mm_mullo_epi16(a, b); - let hi = _mm_mulhi_epi16(a, b); - _mm_unpacklo_epi16(lo, hi) +pub(crate) fn widen_mul_lo(a: __m128i, b: __m128i) -> __m128i { + unsafe { + let lo = _mm_mullo_epi16(a, b); + let hi = _mm_mulhi_epi16(a, b); + _mm_unpacklo_epi16(lo, hi) + } } #[target_feature(enable = "sse4.1")] -unsafe fn filter_row_complex_u8_i32_impl( +fn filter_row_complex_u8_i32_impl( arena: Arena, arena_src: &[u8], dst: &mut [Complex], diff --git a/src/filter1d/sse/filter_row_f32.rs b/src/filter1d/sse/filter_row_f32.rs index 38fad64c..f0b385dc 100644 --- a/src/filter1d/sse/filter_row_f32.rs +++ b/src/filter1d/sse/filter_row_f32.rs @@ -65,7 +65,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "sse4.1")] - unsafe fn pass( + fn pass( &self, _: Arena, arena_src: &[f32], @@ -74,94 +74,96 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let src = arena_src; + unsafe { + let src = arena_src; - let local_src = src; + let local_src = src; - let length = scanned_kernel.len(); + let length = scanned_kernel.len(); - let max_width = image_size.width * N; + let max_width = image_size.width * N; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(0).weight); + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(0).weight); - let mut cx = 0usize; + let mut cx = 0usize; - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - let source = _mm_load_pack_ps_x4(shifted_src.as_ptr()); - let mut k0 = _mm_mul_ps(source.0, coeff); - let mut k1 = _mm_mul_ps(source.1, coeff); - let mut k2 = _mm_mul_ps(source.2, coeff); - let mut k3 = _mm_mul_ps(source.3, coeff); + let source = _mm_load_pack_ps_x4(shifted_src.as_ptr()); + let mut k0 = _mm_mul_ps(source.0, coeff); + let mut k1 = _mm_mul_ps(source.1, coeff); + let mut k2 = _mm_mul_ps(source.2, coeff); + let mut k3 = _mm_mul_ps(source.3, coeff); - for i in 1..length { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm_load_pack_ps_x4(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_opt_fmlaf_ps(k0, v_source.0, coeff); - k1 = _mm_opt_fmlaf_ps(k1, v_source.1, coeff); - k2 = _mm_opt_fmlaf_ps(k2, v_source.2, coeff); - k3 = _mm_opt_fmlaf_ps(k3, v_source.3, coeff); + for i in 1..length { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm_load_pack_ps_x4(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_opt_fmlaf_ps(k0, v_source.0, coeff); + k1 = _mm_opt_fmlaf_ps(k1, v_source.1, coeff); + k2 = _mm_opt_fmlaf_ps(k2, v_source.2, coeff); + k3 = _mm_opt_fmlaf_ps(k3, v_source.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_ps_x4(dst_ptr0, (k0, k1, k2, k3)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_ps_x4(dst_ptr0, (k0, k1, k2, k3)); - cx += 16; - } + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm_load_pack_ps_x2(shifted_src.as_ptr()); + let mut k0 = _mm_mul_ps(source.0, coeff); + let mut k1 = _mm_mul_ps(source.1, coeff); - let source = _mm_load_pack_ps_x2(shifted_src.as_ptr()); - let mut k0 = _mm_mul_ps(source.0, coeff); - let mut k1 = _mm_mul_ps(source.1, coeff); + for i in 1..length { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm_load_pack_ps_x2(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_opt_fmlaf_ps(k0, v_source.0, coeff); + k1 = _mm_opt_fmlaf_ps(k1, v_source.1, coeff); + } - for i in 1..length { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm_load_pack_ps_x2(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_opt_fmlaf_ps(k0, v_source.0, coeff); - k1 = _mm_opt_fmlaf_ps(k1, v_source.1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_ps_x2(dst_ptr0, (k0, k1)); + cx += 8; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_ps_x2(dst_ptr0, (k0, k1)); - cx += 8; - } + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source_0 = _mm_loadu_ps(shifted_src.as_ptr()); + let mut k0 = _mm_mul_ps(source_0, coeff); - let source_0 = _mm_loadu_ps(shifted_src.as_ptr()); - let mut k0 = _mm_mul_ps(source_0, coeff); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + let v_source_0 = _mm_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_opt_fmlaf_ps(k0, v_source_0, _mm_set1_ps(coeff.weight)); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - let v_source_0 = _mm_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_opt_fmlaf_ps(k0, v_source_0, _mm_set1_ps(coeff.weight)); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_ps(dst_ptr0, k0); + cx += 4; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_ps(dst_ptr0, k0); - cx += 4; - } + while cx < max_width { + let coeff = *scanned_kernel.get_unchecked(0); - while cx < max_width { - let coeff = *scanned_kernel.get_unchecked(0); + let shifted_src = local_src.get_unchecked(cx..); - let shifted_src = local_src.get_unchecked(cx..); + let source_0 = _mm_load_ss(shifted_src.as_ptr()); + let mut k0 = _mm_mul_ps(source_0, _mm_set1_ps(coeff.weight)); - let source_0 = _mm_load_ss(shifted_src.as_ptr()); - let mut k0 = _mm_mul_ps(source_0, _mm_set1_ps(coeff.weight)); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + let v_source_0 = _mm_load_ss(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_opt_fmlaf_ps(k0, v_source_0, _mm_set1_ps(coeff.weight)); + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - let v_source_0 = _mm_load_ss(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_opt_fmlaf_ps(k0, v_source_0, _mm_set1_ps(coeff.weight)); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_ss(dst_ptr0, k0); + cx += 1; } - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_ss(dst_ptr0, k0); - cx += 1; } } } diff --git a/src/filter1d/sse/filter_row_f32_f64.rs b/src/filter1d/sse/filter_row_f32_f64.rs index a4abfeb2..c464099d 100644 --- a/src/filter1d/sse/filter_row_f32_f64.rs +++ b/src/filter1d/sse/filter_row_f32_f64.rs @@ -63,7 +63,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "sse4.1")] - unsafe fn pass( + fn pass( &self, _: Arena, arena_src: &[f32], @@ -72,97 +72,99 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let src = arena_src; - - let local_src = src; - - let length = scanned_kernel.len(); - - let max_width = image_size.width * N; - - let coeff = _mm_set1_pd(scanned_kernel.get_unchecked(0).weight); - - let mut cx = 0usize; - - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_load_pack_ps_x2(shifted_src.as_ptr()); - let mut k0 = _mm_mul_pd(_mm_cvtps_pd(source.0), coeff); - let mut k1 = _mm_mul_pd(_mm_cvtps_pd(_mm_movehl_ps(source.0, source.0)), coeff); - let mut k2 = _mm_mul_pd(_mm_cvtps_pd(source.1), coeff); - let mut k3 = _mm_mul_pd(_mm_cvtps_pd(_mm_movehl_ps(source.1, source.1)), coeff); - - for i in 1..length { - let coeff = _mm_set1_pd(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm_load_pack_ps_x2(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_fmla_pd(k0, _mm_cvtps_pd(v_source.0), coeff); - k1 = _mm_fmla_pd( - k1, - _mm_cvtps_pd(_mm_movehl_ps(v_source.0, v_source.0)), - coeff, - ); - k2 = _mm_fmla_pd(k2, _mm_cvtps_pd(v_source.1), coeff); - k3 = _mm_fmla_pd( - k3, - _mm_cvtps_pd(_mm_movehl_ps(v_source.1, v_source.1)), - coeff, - ); + unsafe { + let src = arena_src; + + let local_src = src; + + let length = scanned_kernel.len(); + + let max_width = image_size.width * N; + + let coeff = _mm_set1_pd(scanned_kernel.get_unchecked(0).weight); + + let mut cx = 0usize; + + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = _mm_load_pack_ps_x2(shifted_src.as_ptr()); + let mut k0 = _mm_mul_pd(_mm_cvtps_pd(source.0), coeff); + let mut k1 = _mm_mul_pd(_mm_cvtps_pd(_mm_movehl_ps(source.0, source.0)), coeff); + let mut k2 = _mm_mul_pd(_mm_cvtps_pd(source.1), coeff); + let mut k3 = _mm_mul_pd(_mm_cvtps_pd(_mm_movehl_ps(source.1, source.1)), coeff); + + for i in 1..length { + let coeff = _mm_set1_pd(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm_load_pack_ps_x2(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_fmla_pd(k0, _mm_cvtps_pd(v_source.0), coeff); + k1 = _mm_fmla_pd( + k1, + _mm_cvtps_pd(_mm_movehl_ps(v_source.0, v_source.0)), + coeff, + ); + k2 = _mm_fmla_pd(k2, _mm_cvtps_pd(v_source.1), coeff); + k3 = _mm_fmla_pd( + k3, + _mm_cvtps_pd(_mm_movehl_ps(v_source.1, v_source.1)), + coeff, + ); + } + + let z0 = _mm_cvtpd_ps(k0); + let z1 = _mm_cvtpd_ps(k1); + let z2 = _mm_cvtpd_ps(k2); + let z3 = _mm_cvtpd_ps(k3); + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_ps_x2(dst_ptr0, (_mm_movelh_ps(z0, z1), _mm_movelh_ps(z2, z3))); + cx += 8; } - let z0 = _mm_cvtpd_ps(k0); - let z1 = _mm_cvtpd_ps(k1); - let z2 = _mm_cvtpd_ps(k2); - let z3 = _mm_cvtpd_ps(k3); - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_ps_x2(dst_ptr0, (_mm_movelh_ps(z0, z1), _mm_movelh_ps(z2, z3))); - cx += 8; - } - - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source_0 = _mm_loadu_ps(shifted_src.as_ptr()); - let mut k0 = _mm_mul_pd(_mm_cvtps_pd(source_0), coeff); - let mut k1 = _mm_mul_pd(_mm_cvtps_pd(_mm_movehl_ps(source_0, source_0)), coeff); - - for i in 1..length { - let coeff = _mm_set1_pd(scanned_kernel.get_unchecked(i).weight); - let v_source_0 = _mm_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_fmla_pd(k0, _mm_cvtps_pd(v_source_0), coeff); - k1 = _mm_fmla_pd( - k1, - _mm_cvtps_pd(_mm_movehl_ps(v_source_0, v_source_0)), - coeff, - ); + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source_0 = _mm_loadu_ps(shifted_src.as_ptr()); + let mut k0 = _mm_mul_pd(_mm_cvtps_pd(source_0), coeff); + let mut k1 = _mm_mul_pd(_mm_cvtps_pd(_mm_movehl_ps(source_0, source_0)), coeff); + + for i in 1..length { + let coeff = _mm_set1_pd(scanned_kernel.get_unchecked(i).weight); + let v_source_0 = _mm_loadu_ps(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_fmla_pd(k0, _mm_cvtps_pd(v_source_0), coeff); + k1 = _mm_fmla_pd( + k1, + _mm_cvtps_pd(_mm_movehl_ps(v_source_0, v_source_0)), + coeff, + ); + } + + let z0 = _mm_cvtpd_ps(k0); + let z1 = _mm_cvtpd_ps(k1); + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_ps(dst_ptr0, _mm_movelh_ps(z0, z1)); + cx += 4; } - let z0 = _mm_cvtpd_ps(k0); - let z1 = _mm_cvtpd_ps(k1); + while cx < max_width { + let shifted_src = local_src.get_unchecked(cx..); - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_ps(dst_ptr0, _mm_movelh_ps(z0, z1)); - cx += 4; - } + let source_0 = _mm_load_ss(shifted_src.as_ptr()); + let mut k0 = _mm_mul_pd(_mm_cvtps_pd(source_0), coeff); - while cx < max_width { - let shifted_src = local_src.get_unchecked(cx..); + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + let v_source_0 = _mm_load_ss(shifted_src.get_unchecked(i * N..).as_ptr()); + k0 = _mm_fmla_pd(k0, _mm_cvtps_pd(v_source_0), _mm_set1_pd(coeff.weight)); + } - let source_0 = _mm_load_ss(shifted_src.as_ptr()); - let mut k0 = _mm_mul_pd(_mm_cvtps_pd(source_0), coeff); + let z0 = _mm_cvtpd_ps(k0); - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - let v_source_0 = _mm_load_ss(shifted_src.get_unchecked(i * N..).as_ptr()); - k0 = _mm_fmla_pd(k0, _mm_cvtps_pd(v_source_0), _mm_set1_pd(coeff.weight)); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_ss(dst_ptr0, z0); + cx += 1; } - - let z0 = _mm_cvtpd_ps(k0); - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_ss(dst_ptr0, z0); - cx += 1; } } } diff --git a/src/filter1d/sse/filter_row_symm.rs b/src/filter1d/sse/filter_row_symm.rs index fdb860ac..1d5b1b0d 100644 --- a/src/filter1d/sse/filter_row_symm.rs +++ b/src/filter1d/sse/filter_row_symm.rs @@ -68,7 +68,7 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "sse4.1")] - unsafe fn pass( + fn pass( &self, _: Arena, arena_src: &[u8], @@ -77,193 +77,195 @@ impl ExecutionUnit { _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let width = image_size.width; - - let src = arena_src; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let local_src = src; - let mut cx = 0usize; - - let max_width = width * N; - - while cx + 64 < max_width { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_load_pack_x4(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); - let mut k3 = _mm_mul_epi8_by_ps_x4(source.3, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm_load_pack_x4(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = - _mm_load_pack_x4(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi8_by_ps_x4(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm_mul_add_symm_epi8_by_ps_x4(k2, v_source0.2, v_source1.2, coeff); - k3 = _mm_mul_add_symm_epi8_by_ps_x4(k3, v_source0.3, v_source1.3, coeff); + unsafe { + let width = image_size.width; + + let src = arena_src; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let local_src = src; + let mut cx = 0usize; + + let max_width = width * N; + + while cx + 64 <= max_width { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + + let shifted_src = local_src.get_unchecked(cx..); + + let source = _mm_load_pack_x4(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); + let mut k2 = _mm_mul_epi8_by_ps_x4(source.2, coeff); + let mut k3 = _mm_mul_epi8_by_ps_x4(source.3, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm_load_pack_x4(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm_load_pack_x4(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi8_by_ps_x4(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm_mul_add_symm_epi8_by_ps_x4(k2, v_source0.2, v_source1.2, coeff); + k3 = _mm_mul_add_symm_epi8_by_ps_x4(k3, v_source0.3, v_source1.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0, + ( + _mm_pack_ps_x4_epi8(k0), + _mm_pack_ps_x4_epi8(k1), + _mm_pack_ps_x4_epi8(k2), + _mm_pack_ps_x4_epi8(k3), + ), + ); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0, - ( - _mm_pack_ps_x4_epi8(k0), - _mm_pack_ps_x4_epi8(k1), - _mm_pack_ps_x4_epi8(k2), - _mm_pack_ps_x4_epi8(k3), - ), - ); - cx += 64; - } + while cx + 32 <= max_width { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - while cx + 32 < max_width { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + let shifted_src = local_src.get_unchecked(cx..); - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); - let source = _mm_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_ps_x4(source.1, coeff); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm_load_pack_x2(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi8_by_ps_x4(k1, v_source0.1, v_source1.1, coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = - _mm_load_pack_x2(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi8_by_ps_x4(k1, v_source0.1, v_source1.1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2(dst_ptr0, (_mm_pack_ps_x4_epi8(k0), _mm_pack_ps_x4_epi8(k1))); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2(dst_ptr0, (_mm_pack_ps_x4_epi8(k0), _mm_pack_ps_x4_epi8(k1))); - cx += 32; - } - - while cx + 16 < max_width { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_ps_x4(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si128( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0, v_source1, coeff); + while cx + 16 <= max_width { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_ps_x4(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_ps_x4(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_ps_x4_epi8(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_ps_x4_epi8(k0)); - cx += 16; - } - - while cx + 8 < max_width { - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_ps_x2(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si64( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_mul_add_symm_epi8_by_ps_x2(k0, v_source0, v_source1, coeff); + while cx + 8 <= max_width { + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_ps_x2(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_ps_x2(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_ps_x2_epi8(k0)); + cx += 8; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_ps_x2_epi8(k0)); - cx += 8; - } - - while cx + 4 < max_width { - let coeff = *scanned_kernel.get_unchecked(half_len); - let shifted_src = local_src.get_unchecked(cx..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as f32 * coeff.weight; - let mut k1 = *shifted_src.get_unchecked(half_len * N + 1) as f32 * coeff.weight; - let mut k2 = *shifted_src.get_unchecked(half_len * N + 2) as f32 * coeff.weight; - let mut k3 = *shifted_src.get_unchecked(half_len * N + 3) as f32 * coeff.weight; - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - - k0 = mlaf( - k0, - *shifted_src.get_unchecked(i * N) as f32 - + *shifted_src.get_unchecked(rollback * N) as f32, - coeff.weight, - ); - - k1 = mlaf( - k1, - *shifted_src.get_unchecked(i * N + 1) as f32 - + *shifted_src.get_unchecked(rollback * N + 1) as f32, - coeff.weight, - ); - - k2 = mlaf( - k2, - *shifted_src.get_unchecked(i * N + 2) as f32 - + *shifted_src.get_unchecked(rollback * N + 2) as f32, - coeff.weight, - ); - k3 = mlaf( - k3, - *shifted_src.get_unchecked(i * N + 3) as f32 - + *shifted_src.get_unchecked(rollback * N + 3) as f32, - coeff.weight, - ); + while cx + 4 <= max_width { + let coeff = *scanned_kernel.get_unchecked(half_len); + let shifted_src = local_src.get_unchecked(cx..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as f32 * coeff.weight; + let mut k1 = *shifted_src.get_unchecked(half_len * N + 1) as f32 * coeff.weight; + let mut k2 = *shifted_src.get_unchecked(half_len * N + 2) as f32 * coeff.weight; + let mut k3 = *shifted_src.get_unchecked(half_len * N + 3) as f32 * coeff.weight; + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + + k0 = mlaf( + k0, + *shifted_src.get_unchecked(i * N) as f32 + + *shifted_src.get_unchecked(rollback * N) as f32, + coeff.weight, + ); + + k1 = mlaf( + k1, + *shifted_src.get_unchecked(i * N + 1) as f32 + + *shifted_src.get_unchecked(rollback * N + 1) as f32, + coeff.weight, + ); + + k2 = mlaf( + k2, + *shifted_src.get_unchecked(i * N + 2) as f32 + + *shifted_src.get_unchecked(rollback * N + 2) as f32, + coeff.weight, + ); + k3 = mlaf( + k3, + *shifted_src.get_unchecked(i * N + 3) as f32 + + *shifted_src.get_unchecked(rollback * N + 3) as f32, + coeff.weight, + ); + } + + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } + for x in cx..max_width { + let coeff = *scanned_kernel.get_unchecked(half_len); + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as f32 * coeff.weight; - for x in cx..max_width { - let coeff = *scanned_kernel.get_unchecked(half_len); - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as f32 * coeff.weight; + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + k0 = mlaf( + k0, + *shifted_src.get_unchecked(i * N) as f32 + + *shifted_src.get_unchecked(rollback * N) as f32, + coeff.weight, + ); + } - k0 = mlaf( - k0, - *shifted_src.get_unchecked(i * N) as f32 - + *shifted_src.get_unchecked(rollback * N) as f32, - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } } diff --git a/src/filter1d/sse/filter_row_symm_approx.rs b/src/filter1d/sse/filter_row_symm_approx.rs index 2bc8617a..3f18f6f1 100644 --- a/src/filter1d/sse/filter_row_symm_approx.rs +++ b/src/filter1d/sse/filter_row_symm_approx.rs @@ -62,7 +62,7 @@ pub(crate) fn filter_row_symm_sse_u8_i32_app( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_row_symm_sse_u8_i32_impl( +fn filter_row_symm_sse_u8_i32_impl( _: Arena, arena_src: &[u8], dst: &mut [u8], @@ -70,171 +70,181 @@ unsafe fn filter_row_symm_sse_u8_i32_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let width = image_size.width; - - let src = arena_src; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let local_src = src; - - let max_width = width * N; - - let mut cx = 0usize; - - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(half_len).weight as i16); - - while cx + 64 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_load_pack_x4(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_epi16_x4(source.1, coeff); - let mut k2 = _mm_mul_epi8_by_epi16_x4(source.2, coeff); - let mut k3 = _mm_mul_epi8_by_epi16_x4(source.3, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); - let v_source0 = _mm_load_pack_x4(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = _mm_load_pack_x4(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm_mul_add_symm_epi8_by_epi16_x4(k2, v_source0.2, v_source1.2, coeff); - k3 = _mm_mul_add_symm_epi8_by_epi16_x4(k3, v_source0.3, v_source1.3, coeff); + unsafe { + let width = image_size.width; + + let src = arena_src; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let local_src = src; + + let max_width = width * N; + + let mut cx = 0usize; + + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(half_len).weight as i16); + + while cx + 64 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = _mm_load_pack_x4(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm_mul_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_epi16_x4(source.1, coeff); + let mut k2 = _mm_mul_epi8_by_epi16_x4(source.2, coeff); + let mut k3 = _mm_mul_epi8_by_epi16_x4(source.3, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let v_source0 = _mm_load_pack_x4(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm_load_pack_x4(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm_mul_add_symm_epi8_by_epi16_x4(k2, v_source0.2, v_source1.2, coeff); + k3 = _mm_mul_add_symm_epi8_by_epi16_x4(k3, v_source0.3, v_source1.3, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0, + ( + _mm_pack_epi32_x2_epi8(k0), + _mm_pack_epi32_x2_epi8(k1), + _mm_pack_epi32_x2_epi8(k2), + _mm_pack_epi32_x2_epi8(k3), + ), + ); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0, - ( - _mm_pack_epi32_x2_epi8(k0), - _mm_pack_epi32_x2_epi8(k1), - _mm_pack_epi32_x2_epi8(k2), - _mm_pack_epi32_x2_epi8(k3), - ), - ); - cx += 64; - } - - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_mul_epi8_by_epi16_x4(source.1, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); - let v_source0 = _mm_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = _mm_load_pack_x2(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); + while cx + 32 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = _mm_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm_mul_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_mul_epi8_by_epi16_x4(source.1, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let v_source0 = _mm_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm_load_pack_x2(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2( + dst_ptr0, + (_mm_pack_epi32_x2_epi8(k0), _mm_pack_epi32_x2_epi8(k1)), + ); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2( - dst_ptr0, - (_mm_pack_epi32_x2_epi8(k0), _mm_pack_epi32_x2_epi8(k1)), - ); - cx += 32; - } - - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x4(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si128(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_epi16_x4(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epi8(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epi8(k0)); - cx += 16; - } - - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi8_by_epi16_x2(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si64(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi8_by_epi16_x2(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight as i16); + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi8_by_epi16_x2(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); + cx += 8; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_epi32_epi8(k0)); - cx += 8; - } - - let coeff = *scanned_kernel.get_unchecked(half_len); - - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; - let mut k1 = *shifted_src.get_unchecked(half_len * N + 1) as i32 * coeff.weight; - let mut k2 = *shifted_src.get_unchecked(half_len * N + 2) as i32 * coeff.weight; - let mut k3 = *shifted_src.get_unchecked(half_len * N + 3) as i32 * coeff.weight; - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - - k0 += (*shifted_src.get_unchecked(i * N) as i16 - + *shifted_src.get_unchecked(rollback * N) as i16) as i32 - * coeff.weight; - - k1 += (*shifted_src.get_unchecked(i * N + 1) as i16 - + *shifted_src.get_unchecked(rollback * N + 1) as i16) as i32 - * coeff.weight; - - k2 += (*shifted_src.get_unchecked(i * N + 2) as i16 - + *shifted_src.get_unchecked(rollback * N + 2) as i16) as i32 - * coeff.weight; - - k3 += (*shifted_src.get_unchecked(i * N + 3) as i16 - + *shifted_src.get_unchecked(rollback * N + 3) as i16) as i32 - * coeff.weight; + let coeff = *scanned_kernel.get_unchecked(half_len); + + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; + let mut k1 = *shifted_src.get_unchecked(half_len * N + 1) as i32 * coeff.weight; + let mut k2 = *shifted_src.get_unchecked(half_len * N + 2) as i32 * coeff.weight; + let mut k3 = *shifted_src.get_unchecked(half_len * N + 3) as i32 * coeff.weight; + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + + k0 += (*shifted_src.get_unchecked(i * N) as i16 + + *shifted_src.get_unchecked(rollback * N) as i16) as i32 + * coeff.weight; + + k1 += (*shifted_src.get_unchecked(i * N + 1) as i16 + + *shifted_src.get_unchecked(rollback * N + 1) as i16) + as i32 + * coeff.weight; + + k2 += (*shifted_src.get_unchecked(i * N + 2) as i16 + + *shifted_src.get_unchecked(rollback * N + 2) as i16) + as i32 + * coeff.weight; + + k3 += (*shifted_src.get_unchecked(i * N + 3) as i16 + + *shifted_src.get_unchecked(rollback * N + 3) as i16) + as i32 + * coeff.weight; + } + + *dst.get_unchecked_mut(cx) = k0.to_approx_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_approx_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_approx_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_approx_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_approx_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_approx_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_approx_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_approx_(); - cx += 4; - } + for x in cx..max_width { + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; - for x in cx..max_width { - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as i32 * coeff.weight; + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + k0 += (*shifted_src.get_unchecked(i * N) as i16 + + *shifted_src.get_unchecked(rollback * N) as i16) as i32 + * coeff.weight; + } - k0 += (*shifted_src.get_unchecked(i * N) as i16 - + *shifted_src.get_unchecked(rollback * N) as i16) as i32 - * coeff.weight; + *dst.get_unchecked_mut(x) = k0.to_approx_(); } - - *dst.get_unchecked_mut(x) = k0.to_approx_(); } } diff --git a/src/filter1d/sse/filter_row_symm_u16.rs b/src/filter1d/sse/filter_row_symm_u16.rs index 4461487c..81699abd 100644 --- a/src/filter1d/sse/filter_row_symm_u16.rs +++ b/src/filter1d/sse/filter_row_symm_u16.rs @@ -75,7 +75,7 @@ pub(crate) fn filter_row_sse_symm_u16_f32( } #[target_feature(enable = "sse4.1", enable = "fma")] -unsafe fn filter_row_sse_symm_u16_f32_fma( +fn filter_row_sse_symm_u16_f32_fma( arena: Arena, arena_src: &[u16], dst: &mut [u16], @@ -94,7 +94,7 @@ unsafe fn filter_row_sse_symm_u16_f32_fma( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_row_sse_symm_u16_f32_def( +fn filter_row_sse_symm_u16_f32_def( arena: Arena, arena_src: &[u16], dst: &mut [u16], @@ -113,7 +113,7 @@ unsafe fn filter_row_sse_symm_u16_f32_def( } #[inline(always)] -unsafe fn filter_row_sse_symm_u16_f32_impl( +fn filter_row_sse_symm_u16_f32_impl( _: Arena, arena_src: &[u16], dst: &mut [u16], @@ -121,144 +121,151 @@ unsafe fn filter_row_sse_symm_u16_f32_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let width = image_size.width; + unsafe { + let width = image_size.width; + + let src = arena_src; - let src = arena_src; + let length = scanned_kernel.len(); + let half_len = length / 2; - let length = scanned_kernel.len(); - let half_len = length / 2; + let local_src = src; + let mut cx = 0usize; - let local_src = src; - let mut cx = 0usize; + let max_width = width * N; - let max_width = width * N; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(half_len).weight); + while cx + 32 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 32 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = + _mm_load_pack_x4(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps_x2::(source.0, coeff); + let mut k1 = _mm_mul_epi16_by_ps_x2::(source.1, coeff); + let mut k2 = _mm_mul_epi16_by_ps_x2::(source.2, coeff); + let mut k3 = _mm_mul_epi16_by_ps_x2::(source.3, coeff); - let source = - _mm_load_pack_x4(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps_x2::(source.0, coeff); - let mut k1 = _mm_mul_epi16_by_ps_x2::(source.1, coeff); - let mut k2 = _mm_mul_epi16_by_ps_x2::(source.2, coeff); - let mut k3 = _mm_mul_epi16_by_ps_x2::(source.3, coeff); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_load_pack_x4(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_load_pack_x4( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm_mul_add_symm_epi16_by_ps_x2::(k2, v_source0.2, v_source1.2, coeff); + k3 = _mm_mul_add_symm_epi16_by_ps_x2::(k3, v_source0.3, v_source1.3, coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_load_pack_x4(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_load_pack_x4(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm_mul_add_symm_epi16_by_ps_x2::(k2, v_source0.2, v_source1.2, coeff); - k3 = _mm_mul_add_symm_epi16_by_ps_x2::(k3, v_source0.3, v_source1.3, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0 as *mut _, + ( + _mm_pack_ps_x2_epi16(k0), + _mm_pack_ps_x2_epi16(k1), + _mm_pack_ps_x2_epi16(k2), + _mm_pack_ps_x2_epi16(k3), + ), + ); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0 as *mut _, - ( - _mm_pack_ps_x2_epi16(k0), - _mm_pack_ps_x2_epi16(k1), - _mm_pack_ps_x2_epi16(k2), - _mm_pack_ps_x2_epi16(k3), - ), - ); - cx += 32; - } + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = + _mm_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps_x2::(source.0, coeff); + let mut k1 = _mm_mul_epi16_by_ps_x2::(source.1, coeff); - let source = - _mm_load_pack_x2(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps_x2::(source.0, coeff); - let mut k1 = _mm_mul_epi16_by_ps_x2::(source.1, coeff); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_load_pack_x2( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_load_pack_x2(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_load_pack_x2(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_mul_add_symm_epi16_by_ps_x2::(k1, v_source0.1, v_source1.1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2( + dst_ptr0 as *mut _, + (_mm_pack_ps_x2_epi16(k0), _mm_pack_ps_x2_epi16(k1)), + ); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2( - dst_ptr0 as *mut _, - (_mm_pack_ps_x2_epi16(k0), _mm_pack_ps_x2_epi16(k1)), - ); - cx += 16; - } + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = + _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps_x2::(source, coeff); - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps_x2::(source, coeff); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0, v_source1, coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si128(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epi16_by_ps_x2::(k0, v_source0, v_source1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_ps_x2_epi16(k0)); + cx += 8; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_ps_x2_epi16(k0)); - cx += 8; - } + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = + _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epi16_by_ps::(source, coeff); - let source = _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epi16_by_ps::(source, coeff); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epi16_by_ps::(k0, v_source0, v_source1, coeff); + } - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_ps(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si64(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epi16_by_ps::(k0, v_source0, v_source1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_ps_epi16(k0)); + cx += 4; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si64(dst_ptr0 as *mut _, _mm_pack_ps_epi16(k0)); - cx += 4; - } + let coeff = *scanned_kernel.get_unchecked(half_len); - let coeff = *scanned_kernel.get_unchecked(half_len); + for x in cx..max_width { + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as f32 * coeff.weight; - for x in cx..max_width { - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as f32 * coeff.weight; + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + k0 = mlaf( + k0, + *shifted_src.get_unchecked(i * N) as f32 + + *shifted_src.get_unchecked(rollback * N) as f32, + coeff.weight, + ); + } - k0 = mlaf( - k0, - *shifted_src.get_unchecked(i * N) as f32 - + *shifted_src.get_unchecked(rollback * N) as f32, - coeff.weight, - ); + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } diff --git a/src/filter1d/sse/filter_row_symm_u8_i16.rs b/src/filter1d/sse/filter_row_symm_u8_i16.rs index e03fff98..edb4203b 100644 --- a/src/filter1d/sse/filter_row_symm_u8_i16.rs +++ b/src/filter1d/sse/filter_row_symm_u8_i16.rs @@ -59,7 +59,7 @@ pub(crate) fn filter_row_sse_symm_u8_i16( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_row_sse_symm_u8_i16_impl( +fn filter_row_sse_symm_u8_i16_impl( _: Arena, arena_src: &[u8], dst: &mut [u8], @@ -67,123 +67,127 @@ unsafe fn filter_row_sse_symm_u8_i16_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let width = image_size.width; - - let src = arena_src; - - let length = scanned_kernel.len(); - let half_len = length / 2; - - let local_src = src; - - let mut cx = 0usize; - - let max_width = width * N; - - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(half_len).weight); - - while cx + 48 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_load_pack_x3(shifted_src.get_unchecked(half_len * N..).as_ptr()); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); - let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); - let v_source0 = _mm_load_pack_x3(shifted_src.get_unchecked((i * N)..).as_ptr()); - let v_source1 = _mm_load_pack_x3(shifted_src.get_unchecked((rollback * N)..).as_ptr()); - k0 = _mm_madd_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); - k1 = _mm_madd_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); - k2 = _mm_madd_symm_epi8_by_epi16_x4(k2, v_source0.2, v_source1.2, coeff); + unsafe { + let width = image_size.width; + + let src = arena_src; + + let length = scanned_kernel.len(); + let half_len = length / 2; + + let local_src = src; + + let mut cx = 0usize; + + let max_width = width * N; + + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(half_len).weight); + + while cx + 48 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = _mm_load_pack_x3(shifted_src.get_unchecked(half_len * N..).as_ptr()); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); + let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); + let v_source0 = _mm_load_pack_x3(shifted_src.get_unchecked((i * N)..).as_ptr()); + let v_source1 = + _mm_load_pack_x3(shifted_src.get_unchecked((rollback * N)..).as_ptr()); + k0 = _mm_madd_symm_epi8_by_epi16_x4(k0, v_source0.0, v_source1.0, coeff); + k1 = _mm_madd_symm_epi8_by_epi16_x4(k1, v_source0.1, v_source1.1, coeff); + k2 = _mm_madd_symm_epi8_by_epi16_x4(k2, v_source0.2, v_source1.2, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x3( + dst_ptr0, + ( + _mm_packus_epi16(k0.0, k0.1), + _mm_packus_epi16(k1.0, k1.1), + _mm_packus_epi16(k2.0, k2.1), + ), + ); + cx += 48; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x3( - dst_ptr0, - ( - _mm_packus_epi16(k0.0, k0.1), - _mm_packus_epi16(k1.0, k1.1), - _mm_packus_epi16(k2.0, k2.1), - ), - ); - cx += 48; - } - - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si128(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_madd_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_madd_symm_epi8_by_epi16_x4(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_packus_epi16(k0.0, k0.1)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_packus_epi16(k0.0, k0.1)); - cx += 16; - } - - let coeff = *scanned_kernel.get_unchecked(half_len); - - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as i16 * coeff.weight; - let mut k1 = *shifted_src.get_unchecked(half_len * N + 1) as i16 * coeff.weight; - let mut k2 = *shifted_src.get_unchecked(half_len * N + 2) as i16 * coeff.weight; - let mut k3 = *shifted_src.get_unchecked(half_len * N + 3) as i16 * coeff.weight; - - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; - - k0 += (*shifted_src.get_unchecked(i * N) as i16 - + *shifted_src.get_unchecked(rollback * N) as i16) - * coeff.weight; - - k1 += (*shifted_src.get_unchecked(i * N + 1) as i16 - + *shifted_src.get_unchecked(rollback * N + 1) as i16) - * coeff.weight; - - k2 += (*shifted_src.get_unchecked(i * N + 2) as i16 - + *shifted_src.get_unchecked(rollback * N + 2) as i16) - * coeff.weight; - - k3 += (*shifted_src.get_unchecked(i * N + 3) as i16 - + *shifted_src.get_unchecked(rollback * N + 3) as i16) - * coeff.weight; + let coeff = *scanned_kernel.get_unchecked(half_len); + + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as i16 * coeff.weight; + let mut k1 = *shifted_src.get_unchecked(half_len * N + 1) as i16 * coeff.weight; + let mut k2 = *shifted_src.get_unchecked(half_len * N + 2) as i16 * coeff.weight; + let mut k3 = *shifted_src.get_unchecked(half_len * N + 3) as i16 * coeff.weight; + + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; + + k0 += (*shifted_src.get_unchecked(i * N) as i16 + + *shifted_src.get_unchecked(rollback * N) as i16) + * coeff.weight; + + k1 += (*shifted_src.get_unchecked(i * N + 1) as i16 + + *shifted_src.get_unchecked(rollback * N + 1) as i16) + * coeff.weight; + + k2 += (*shifted_src.get_unchecked(i * N + 2) as i16 + + *shifted_src.get_unchecked(rollback * N + 2) as i16) + * coeff.weight; + + k3 += (*shifted_src.get_unchecked(i * N + 3) as i16 + + *shifted_src.get_unchecked(rollback * N + 3) as i16) + * coeff.weight; + } + + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } + for x in cx..max_width { + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as i16 * coeff.weight; - for x in cx..max_width { - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as i16 * coeff.weight; + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + k0 += (*shifted_src.get_unchecked(i * N) as i16 + + *shifted_src.get_unchecked(rollback * N) as i16) + * coeff.weight; + } - k0 += (*shifted_src.get_unchecked(i * N) as i16 - + *shifted_src.get_unchecked(rollback * N) as i16) - * coeff.weight; + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } diff --git a/src/filter1d/sse/filter_row_symm_uq15_u16.rs b/src/filter1d/sse/filter_row_symm_uq15_u16.rs index 091a2ba6..d8e1d66a 100644 --- a/src/filter1d/sse/filter_row_symm_uq15_u16.rs +++ b/src/filter1d/sse/filter_row_symm_uq15_u16.rs @@ -61,7 +61,7 @@ pub(crate) fn filter_row_sse_symm_uq15_u16( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_row_sse_symm_uq15_u16_impl( +fn filter_row_sse_symm_uq15_u16_impl( arena: Arena, arena_src: &[u16], dst: &mut [u16], @@ -69,129 +69,134 @@ unsafe fn filter_row_sse_symm_uq15_u16_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - let width = image_size.width; - - let src = arena_src; + unsafe { + let width = image_size.width; - let length = scanned_kernel.len(); - let half_len = length / 2; + let src = arena_src; - let local_src = src; + let length = scanned_kernel.len(); + let half_len = length / 2; - let mut cx = 0usize; + let local_src = src; - let max_width = width * arena.components; + let mut cx = 0usize; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(half_len).weight as i32); + let max_width = width * arena.components; - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(half_len).weight as i32); - let src0 = shifted_src.get_unchecked(half_len * N..); + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - let source0 = _mm_loadu_si128(src0.as_ptr() as *const _); - let source1 = _mm_loadu_si128(src0.get_unchecked(8..).as_ptr() as *const _); + let src0 = shifted_src.get_unchecked(half_len * N..); - let mut k0 = _mm_mul_epu16_widen(source0, coeff); - let mut k1 = _mm_mul_epu16_widen(source1, coeff); + let source0 = _mm_loadu_si128(src0.as_ptr() as *const _); + let source1 = _mm_loadu_si128(src0.get_unchecked(8..).as_ptr() as *const _); - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let mut k0 = _mm_mul_epu16_widen(source0, coeff); + let mut k1 = _mm_mul_epu16_widen(source1, coeff); - let src0 = shifted_src.get_unchecked((i * N)..); - let r_src0 = shifted_src.get_unchecked((rollback * N)..); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0_0 = _mm_loadu_si128(src0.as_ptr() as *const _); - let v_source0_1 = _mm_loadu_si128(r_src0.as_ptr() as *const _); + let src0 = shifted_src.get_unchecked((i * N)..); + let r_src0 = shifted_src.get_unchecked((rollback * N)..); - let v_source1_0 = _mm_loadu_si128(src0.get_unchecked(8..).as_ptr() as *const _); - let v_source1_1 = _mm_loadu_si128(r_src0.get_unchecked(8..).as_ptr() as *const _); + let v_source0_0 = _mm_loadu_si128(src0.as_ptr() as *const _); + let v_source0_1 = _mm_loadu_si128(r_src0.as_ptr() as *const _); - k0 = _mm_mul_add_symm_epu16_by_epu16_x4(k0, v_source0_0, v_source0_1, coeff); - k1 = _mm_mul_add_symm_epu16_by_epu16_x4(k1, v_source1_0, v_source1_1, coeff); - } + let v_source1_0 = _mm_loadu_si128(src0.get_unchecked(8..).as_ptr() as *const _); + let v_source1_1 = _mm_loadu_si128(r_src0.get_unchecked(8..).as_ptr() as *const _); - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epu16(k0)); - _mm_storeu_si128(dst_ptr0.add(8) as *mut _, _mm_pack_epi32_x2_epu16(k1)); - cx += 16; - } + k0 = _mm_mul_add_symm_epu16_by_epu16_x4(k0, v_source0_0, v_source0_1, coeff); + k1 = _mm_mul_add_symm_epu16_by_epu16_x4(k1, v_source1_0, v_source1_1, coeff); + } - while cx + 8 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mul_epu16_widen(source, coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si128(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_mul_add_symm_epu16_by_epu16_x4(k0, v_source0, v_source1, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epu16(k0)); + _mm_storeu_si128(dst_ptr0.add(8) as *mut _, _mm_pack_epi32_x2_epu16(k1)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epu16(k0)); - cx += 8; - } + while cx + 8 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mul_epu16_widen(source, coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_mul_add_symm_epu16_by_epu16_x4(k0, v_source0, v_source1, coeff); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_pack_epi32_x2_epu16(k0)); + cx += 8; + } - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - - let source = _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_mullo_epi32(_mm_unpacklo_epi16(source, _mm_setzero_si128()), coeff); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = - _mm_loadu_si64(shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _); - k0 = _mm_add_epi32( - k0, - _mm_mullo_epi32( - _mm_add_epi32( - _mm_unpacklo_epi16(v_source0, _mm_setzero_si128()), - _mm_unpacklo_epi16(v_source1, _mm_setzero_si128()), + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + + let source = + _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); + let mut k0 = _mm_mullo_epi32(_mm_unpacklo_epi16(source, _mm_setzero_si128()), coeff); + + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(scanned_kernel.get_unchecked(i).weight as i32); + let v_source0 = + _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_add_epi32( + k0, + _mm_mullo_epi32( + _mm_add_epi32( + _mm_unpacklo_epi16(v_source0, _mm_setzero_si128()), + _mm_unpacklo_epi16(v_source1, _mm_setzero_si128()), + ), + coeff, ), - coeff, + ); + } + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + + let rnd = _mm_set1_epi32((1 << 14) - 1); + _mm_storeu_si64( + dst_ptr0 as *mut _, + _mm_packus_epi32( + _mm_srli_epi32::<15>(_mm_add_epi32(k0, rnd)), + _mm_setzero_si128(), ), ); + cx += 4; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - - let rnd = _mm_set1_epi32((1 << 14) - 1); - _mm_storeu_si64( - dst_ptr0 as *mut _, - _mm_packus_epi32( - _mm_srli_epi32::<15>(_mm_add_epi32(k0, rnd)), - _mm_setzero_si128(), - ), - ); - cx += 4; - } + for x in cx..max_width { + let coeff = *scanned_kernel.get_unchecked(half_len); + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as u32 * coeff.weight; - for x in cx..max_width { - let coeff = *scanned_kernel.get_unchecked(half_len); - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as u32 * coeff.weight; + for i in 0..half_len { + let coeff = *scanned_kernel.get_unchecked(i); + let rollback = length - i - 1; - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + k0 += (*shifted_src.get_unchecked(i * N) as u32 + + *shifted_src.get_unchecked(rollback * N) as u32) + * coeff.weight; + } - k0 += (*shifted_src.get_unchecked(i * N) as u32 - + *shifted_src.get_unchecked(rollback * N) as u32) - * coeff.weight; + *dst.get_unchecked_mut(x) = k0.to_approx_(); } - - *dst.get_unchecked_mut(x) = k0.to_approx_(); } } diff --git a/src/filter1d/sse/filter_row_u8_i16.rs b/src/filter1d/sse/filter_row_u8_i16.rs index 53fc4611..b9d511a0 100644 --- a/src/filter1d/sse/filter_row_u8_i16.rs +++ b/src/filter1d/sse/filter_row_u8_i16.rs @@ -59,7 +59,7 @@ pub(crate) fn filter_rgb_row_sse_u8_i16( } #[target_feature(enable = "sse4.1")] -unsafe fn filter_rgb_row_sse_u8_i16_impl( +fn filter_rgb_row_sse_u8_i16_impl( _: Arena, arena_src: &[u8], dst: &mut [u8], @@ -67,99 +67,101 @@ unsafe fn filter_rgb_row_sse_u8_i16_impl( _: FilterRegion, scanned_kernel: &[ScanPoint1d], ) { - const N: usize = 3; + unsafe { + const N: usize = 3; + + let src = arena_src; - let src = arena_src; + let length = scanned_kernel.len(); - let length = scanned_kernel.len(); + let local_src = src; - let local_src = src; + let mut cx = 0usize; - let mut cx = 0usize; + let max_width = image_size.width * N; - let max_width = image_size.width * N; + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(0).weight); - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(0).weight); + while cx + 48 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 48 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm_load_pack_x3(shifted_src.as_ptr()); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); + let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); + let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); - let source = _mm_load_pack_x3(shifted_src.as_ptr()); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source.0, coeff); - let mut k1 = _mm_madd_epi8_by_epi16_x4(source.1, coeff); - let mut k2 = _mm_madd_epi8_by_epi16_x4(source.2, coeff); + for i in 1..length { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); + let v_source = _mm_load_pack_x3(shifted_src.get_unchecked((i * N)..).as_ptr()); + k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source.0, coeff); + k1 = _mm_madd_s_epi8_by_epi16_x4(k1, v_source.1, coeff); + k2 = _mm_madd_s_epi8_by_epi16_x4(k2, v_source.2, coeff); + } - for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); - let v_source = _mm_load_pack_x3(shifted_src.get_unchecked((i * N)..).as_ptr()); - k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source.0, coeff); - k1 = _mm_madd_s_epi8_by_epi16_x4(k1, v_source.1, coeff); - k2 = _mm_madd_s_epi8_by_epi16_x4(k2, v_source.2, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x3( + dst_ptr0, + ( + _mm_packus_epi16(k0.0, k0.1), + _mm_packus_epi16(k1.0, k1.1), + _mm_packus_epi16(k2.0, k2.1), + ), + ); + cx += 48; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x3( - dst_ptr0, - ( - _mm_packus_epi16(k0.0, k0.1), - _mm_packus_epi16(k1.0, k1.1), - _mm_packus_epi16(k2.0, k2.1), - ), - ); - cx += 48; - } + while cx + 16 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); - while cx + 16 < max_width { - let shifted_src = local_src.get_unchecked(cx..); + let source = _mm_loadu_si128(shifted_src.as_ptr() as *const _); + let mut k0 = _mm_madd_epi8_by_epi16_x4(source, coeff); - let source = _mm_loadu_si128(shifted_src.as_ptr() as *const _); - let mut k0 = _mm_madd_epi8_by_epi16_x4(source, coeff); + for i in 1..length { + let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); + let v_source = + _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); + k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source, coeff); + } - for i in 1..length { - let coeff = _mm_set1_epi16(scanned_kernel.get_unchecked(i).weight); - let v_source = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - k0 = _mm_madd_s_epi8_by_epi16_x4(k0, v_source, coeff); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut _, _mm_packus_epi16(k0.0, k0.1)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut _, _mm_packus_epi16(k0.0, k0.1)); - cx += 16; - } - - let coeff = *scanned_kernel.get_unchecked(0); - - while cx + 4 < max_width { - let shifted_src = local_src.get_unchecked(cx..); - let mut k0 = *shifted_src.get_unchecked(0) as i16 * coeff.weight; - let mut k1 = *shifted_src.get_unchecked(1) as i16 * coeff.weight; - let mut k2 = *shifted_src.get_unchecked(2) as i16 * coeff.weight; - let mut k3 = *shifted_src.get_unchecked(3) as i16 * coeff.weight; - - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 += *shifted_src.get_unchecked(i * N) as i16 * coeff.weight; - k1 += *shifted_src.get_unchecked(i * N + 1) as i16 * coeff.weight; - k2 += *shifted_src.get_unchecked(i * N + 2) as i16 * coeff.weight; - k3 += *shifted_src.get_unchecked(i * N + 3) as i16 * coeff.weight; + let coeff = *scanned_kernel.get_unchecked(0); + + while cx + 4 <= max_width { + let shifted_src = local_src.get_unchecked(cx..); + let mut k0 = *shifted_src.get_unchecked(0) as i16 * coeff.weight; + let mut k1 = *shifted_src.get_unchecked(1) as i16 * coeff.weight; + let mut k2 = *shifted_src.get_unchecked(2) as i16 * coeff.weight; + let mut k3 = *shifted_src.get_unchecked(3) as i16 * coeff.weight; + + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 += *shifted_src.get_unchecked(i * N) as i16 * coeff.weight; + k1 += *shifted_src.get_unchecked(i * N + 1) as i16 * coeff.weight; + k2 += *shifted_src.get_unchecked(i * N + 2) as i16 * coeff.weight; + k3 += *shifted_src.get_unchecked(i * N + 3) as i16 * coeff.weight; + } + + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } + for x in cx..max_width { + let shifted_src = local_src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(0) as i16 * coeff.weight; - for x in cx..max_width { - let shifted_src = local_src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(0) as i16 * coeff.weight; + for i in 1..length { + let coeff = *scanned_kernel.get_unchecked(i); + k0 += *shifted_src.get_unchecked(i * N) as i16 * coeff.weight; + } - for i in 1..length { - let coeff = *scanned_kernel.get_unchecked(i); - k0 += *shifted_src.get_unchecked(i * N) as i16 * coeff.weight; + *dst.get_unchecked_mut(x) = k0.to_(); } - - *dst.get_unchecked_mut(x) = k0.to_(); } } diff --git a/src/filter1d/sse/mod.rs b/src/filter1d/sse/mod.rs index dbc84e1d..8dbb6f66 100644 --- a/src/filter1d/sse/mod.rs +++ b/src/filter1d/sse/mod.rs @@ -65,31 +65,32 @@ pub(crate) mod utils; pub(crate) use filter_column::filter_column_sse_u8_f32; pub(crate) use filter_column_approx::filter_column_sse_u8_i32_app; pub(crate) use filter_column_complex_f32_f32::filter_sse_column_complex_f32_f32; -pub(crate) use filter_column_complex_u16_f32::filter_sse_column_complex_u16_f32; -pub(crate) use filter_column_complex_u16_i32::filter_sse_column_complex_u16_i32; pub(crate) use filter_column_complex_u8_f32::filter_sse_column_complex_u8_f32; pub(crate) use filter_column_complex_u8_i32::filter_sse_column_complex_u8_i32; +pub(crate) use filter_column_complex_u16_f32::filter_sse_column_complex_u16_f32; +pub(crate) use filter_column_complex_u16_i32::filter_sse_column_complex_u16_i32; pub(crate) use filter_column_f32::filter_column_sse_f32_f32; pub(crate) use filter_column_f32_f64::filter_column_sse_f32_f64; pub(crate) use filter_column_symm::filter_column_symm_sse_u8_f32; pub(crate) use filter_column_symm_approx::filter_column_symm_u8_i32_app; -pub(crate) use filter_column_symm_u16::filter_column_symm_sse_u16_f32; +pub(crate) use filter_column_symm_approx_uq0_7::filter_column_sse_symm_u8_uq0_7; pub(crate) use filter_column_symm_u8_i16::filter_column_symm_sse_u8_i16; +pub(crate) use filter_column_symm_u16::filter_column_symm_sse_u16_f32; pub(crate) use filter_column_symm_uq15_u16::filter_column_sse_symm_uq15_u16; pub(crate) use filter_column_u8_i16::filter_column_sse_u8_i16; pub(crate) use filter_row::filter_row_sse_u8_f32; pub(crate) use filter_row_approx::filter_row_sse_u8_i32; pub(crate) use filter_row_complex_f32_f32::filter_sse_row_complex_f32_f32; -pub(crate) use filter_row_complex_u16_f32::filter_sse_row_complex_u16_f32; -pub(crate) use filter_row_complex_u16_i32::filter_sse_row_complex_u16_i32; pub(crate) use filter_row_complex_u8_f32::filter_sse_row_complex_u8_f32; pub(crate) use filter_row_complex_u8_i32::filter_sse_row_complex_u8_i32; +pub(crate) use filter_row_complex_u16_f32::filter_sse_row_complex_u16_f32; +pub(crate) use filter_row_complex_u16_i32::filter_sse_row_complex_u16_i32; pub(crate) use filter_row_f32::filter_row_sse_f32_f32; pub(crate) use filter_row_f32_f64::filter_row_sse_f32_f64; pub(crate) use filter_row_symm::filter_row_sse_symm_u8_f32; pub(crate) use filter_row_symm_approx::filter_row_symm_sse_u8_i32_app; -pub(crate) use filter_row_symm_u16::filter_row_sse_symm_u16_f32; pub(crate) use filter_row_symm_u8_i16::filter_row_sse_symm_u8_i16; +pub(crate) use filter_row_symm_u16::filter_row_sse_symm_u16_f32; pub(crate) use filter_row_symm_uq15_u16::filter_row_sse_symm_uq15_u16; pub(crate) use filter_row_u8_i16::filter_rgb_row_sse_u8_i16; pub(crate) use row_symm_approx_binter_uq0_7::filter_row_sse_symm_u8_uq0_7_any; diff --git a/src/filter1d/sse/row_symm_approx_binter_uq0_7.rs b/src/filter1d/sse/row_symm_approx_binter_uq0_7.rs index 65969058..486cf2c1 100644 --- a/src/filter1d/sse/row_symm_approx_binter_uq0_7.rs +++ b/src/filter1d/sse/row_symm_approx_binter_uq0_7.rs @@ -26,11 +26,10 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::BorderHandle; use crate::edge_mode::border_interpolate; -use crate::filter1d::filter_scan::ScanPoint1d; use crate::filter1d::row_handler_small_approx::{RowsHolder, RowsHolderMut}; use crate::img_size::ImageSize; -use crate::BorderHandle; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] @@ -41,29 +40,11 @@ pub(crate) fn filter_row_sse_symm_u8_uq0_7_any( m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[ScanPoint1d], + scanned_kernel: &[i32], ) { unsafe { - let mut shifted = scanned_kernel - .iter() - .map(|&x| ((x.weight) >> 8).min(i8::MAX as i32) as i8) - .collect::>(); - let mut sum: u32 = shifted.iter().map(|&x| x as u32).sum(); - if sum > 128 { - let half = shifted.len() / 2; - while sum > 128 { - shifted[half] = shifted[half].saturating_sub(1); - sum -= 1; - } - } else if sum < 128 { - let half = shifted.len() / 2; - while sum < 128 { - shifted[half] = shifted[half].saturating_add(1); - sum += 1; - } - } let unit = ExecutionUnit::::default(); - unit.pass(edge_mode, m_src, m_dst, image_size, &shifted); + unit.pass(edge_mode, m_src, m_dst, image_size, scanned_kernel); } } @@ -72,248 +53,249 @@ struct ExecutionUnit {} impl ExecutionUnit { #[target_feature(enable = "sse4.1")] - unsafe fn pass( + fn pass( &self, edge_mode: BorderHandle, m_src: &RowsHolder, m_dst: &mut RowsHolderMut, image_size: ImageSize, - scanned_kernel: &[i8], + scanned_kernel: &[i32], ) { - let width = image_size.width; - - let v_prepared = scanned_kernel - .iter() - .map(|&x| { - let z = x.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[0], z[0], z[0]]) - }) - .collect::>(); + unsafe { + let width = image_size.width; + let length = scanned_kernel.len(); + let half_len = length / 2; + + let min_left = half_len.min(width); + let s_kernel = half_len as i64; + + for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { + let dst = &mut **dst; + let mut f_cx = 0usize; + + let coeff = scanned_kernel.get_unchecked(half_len).to_ne_bytes()[0]; + + while f_cx < min_left { + for c in 0..N { + let mx = f_cx as i64 - s_kernel; + let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * coeff as u16; + + for i in 0..half_len { + let coeff = scanned_kernel.get_unchecked(i).to_ne_bytes(); + let rollback = length - i - 1; + + let src0 = border_interpolate!( + src, + edge_mode, + i as i64 + mx, + 0, + width as i64, + N, + c + ); + let src1 = border_interpolate!( + src, + edge_mode, + rollback as i64 + mx, + 0, + width as i64, + N, + c + ); + + k0 += (src0 as u16 + src1 as u16) * coeff[0] as u16; + } + + *dst.get_unchecked_mut(f_cx * N + c) = + ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + f_cx += 1; + } - let length = scanned_kernel.len(); - let half_len = length / 2; + let mut m_cx = f_cx * N; - let min_left = half_len.min(width); - let s_kernel = half_len as i64; + let s_half = half_len * N; + let m_right = width.saturating_sub(half_len); + let max_width = m_right * N; - for (src, dst) in m_src.holder.iter().zip(m_dst.holder.iter_mut()) { - let dst = &mut **dst; - let mut f_cx = 0usize; + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(half_len)); + let rnd = _mm_set1_epi16(1 << 6); - let coeff = *scanned_kernel.get_unchecked(half_len); + while m_cx + 16 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - while f_cx < min_left { - for c in 0..N { - let mx = f_cx as i64 - s_kernel; - let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * coeff as u16; + let source = _mm_loadu_si128( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _, + ); + let mut k0 = _mm_add_epi16( + rnd, + _mm_maddubs_epi16(_mm_unpacklo_epi8(source, _mm_setzero_si128()), coeff), + ); + let mut k1 = _mm_add_epi16( + rnd, + _mm_maddubs_epi16(_mm_unpackhi_epi8(source, _mm_setzero_si128()), coeff), + ); for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); let rollback = length - i - 1; - - let src0 = border_interpolate!( - src, - edge_mode, - i as i64 + mx, - 0, - width as i64, - N, - c + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm_loadu_si128( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, ); - let src1 = border_interpolate!( - src, - edge_mode, - rollback as i64 + mx, - 0, - width as i64, - N, - c + let v_source1 = _mm_loadu_si128( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), + ); + k1 = _mm_add_epi16( + k1, + _mm_maddubs_epi16(_mm_unpackhi_epi8(v_source0, v_source1), coeff), ); - - k0 += (src0 as u16 + src1 as u16) * coeff as u16; } - *dst.get_unchecked_mut(f_cx * N + c) = ((k0 + (1 << 6)) >> 7).min(255) as u8; - } - f_cx += 1; - } + k0 = _mm_srai_epi16::<7>(k0); + k1 = _mm_srai_epi16::<7>(k1); - let mut m_cx = f_cx * N; - - let s_half = half_len * N; - let m_right = width.saturating_sub(half_len); - let max_width = m_right * N; - - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(half_len)); - let rnd = _mm_set1_epi16(1 << 6); - - while m_cx + 16 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si128(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - rnd, - _mm_maddubs_epi16(_mm_unpacklo_epi8(source, _mm_setzero_si128()), coeff), - ); - let mut k1 = _mm_add_epi16( - rnd, - _mm_maddubs_epi16(_mm_unpackhi_epi8(source, _mm_setzero_si128()), coeff), - ); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si128(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si128( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), - ); - k1 = _mm_add_epi16( - k1, - _mm_maddubs_epi16(_mm_unpackhi_epi8(v_source0, v_source1), coeff), - ); + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_packus_epi16(k0, k1)); + m_cx += 16; } - k0 = _mm_srai_epi16::<7>(k0); - k1 = _mm_srai_epi16::<7>(k1); - - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_packus_epi16(k0, k1)); - m_cx += 16; - } + while m_cx + 8 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - while m_cx + 8 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si64(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - rnd, - _mm_maddubs_epi16(_mm_unpacklo_epi8(source, _mm_setzero_si128()), coeff), - ); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si64(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si64( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + let source = _mm_loadu_si64( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ ); - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), + let mut k0 = _mm_add_epi16( + rnd, + _mm_maddubs_epi16(_mm_unpacklo_epi8(source, _mm_setzero_si128()), coeff), ); - } - k0 = _mm_srai_epi16::<7>(k0); + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm_loadu_si64( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si64( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), + ); + } - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si64( - dst_ptr0 as *mut _, - _mm_packus_epi16(k0, _mm_setzero_si128()), - ); - m_cx += 8; - } + k0 = _mm_srai_epi16::<7>(k0); - while m_cx + 4 < max_width { - let cx = m_cx - s_half; - let shifted_src = src.get_unchecked(cx..); - - let source = - _mm_loadu_si32(shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _); - let mut k0 = _mm_add_epi16( - rnd, - _mm_maddubs_epi16(_mm_unpacklo_epi8(source, _mm_setzero_si128()), coeff), - ); - - for i in 0..half_len { - let rollback = length - i - 1; - let coeff = _mm_set1_epi32(*v_prepared.get_unchecked(i)); - let v_source0 = - _mm_loadu_si32(shifted_src.get_unchecked((i * N)..).as_ptr() as *const _); - let v_source1 = _mm_loadu_si32( - shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, - ); - k0 = _mm_add_epi16( - k0, - _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si64( + dst_ptr0 as *mut _, + _mm_packus_epi16(k0, _mm_setzero_si128()), ); + m_cx += 8; } - k0 = _mm_srai_epi16::<7>(k0); + while m_cx + 4 < max_width { + let cx = m_cx - s_half; + let shifted_src = src.get_unchecked(cx..); - let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); - _mm_storeu_si32( - dst_ptr0 as *mut _, - _mm_packus_epi16(k0, _mm_setzero_si128()), - ); - m_cx += 4; - } - - let coeff = *scanned_kernel.get_unchecked(half_len); - - for zx in m_cx..max_width { - let x = zx - s_half; + let source = _mm_loadu_si32( + shifted_src.get_unchecked(half_len * N..).as_ptr() as *const _ + ); + let mut k0 = _mm_add_epi16( + rnd, + _mm_maddubs_epi16(_mm_unpacklo_epi8(source, _mm_setzero_si128()), coeff), + ); - let shifted_src = src.get_unchecked(x..); - let mut k0 = *shifted_src.get_unchecked(half_len * N) as u16 * coeff as u16; + for i in 0..half_len { + let rollback = length - i - 1; + let coeff = _mm_set1_epi32(*scanned_kernel.get_unchecked(i)); + let v_source0 = _mm_loadu_si32( + shifted_src.get_unchecked((i * N)..).as_ptr() as *const _, + ); + let v_source1 = _mm_loadu_si32( + shifted_src.get_unchecked((rollback * N)..).as_ptr() as *const _, + ); + k0 = _mm_add_epi16( + k0, + _mm_maddubs_epi16(_mm_unpacklo_epi8(v_source0, v_source1), coeff), + ); + } - for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); - let rollback = length - i - 1; + k0 = _mm_srai_epi16::<7>(k0); - k0 += (*shifted_src.get_unchecked(i * N) as u16 - + *shifted_src.get_unchecked(rollback * N) as u16) - * coeff as u16; + let dst_ptr0 = dst.get_unchecked_mut(m_cx..).as_mut_ptr(); + _mm_storeu_si32( + dst_ptr0 as *mut _, + _mm_packus_epi16(k0, _mm_setzero_si128()), + ); + m_cx += 4; } - *dst.get_unchecked_mut(zx) = ((k0 + (1 << 6)) >> 7).min(255) as u8; - } + let coeff = scanned_kernel.get_unchecked(half_len).to_ne_bytes()[0]; - f_cx = m_right; + for zx in m_cx..max_width { + let x = zx - s_half; - while f_cx < width { - for c in 0..N { - let mx = f_cx as i64 - s_kernel; - let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * coeff as u16; + let shifted_src = src.get_unchecked(x..); + let mut k0 = *shifted_src.get_unchecked(half_len * N) as u16 * coeff as u16; for i in 0..half_len { - let coeff = *scanned_kernel.get_unchecked(i); + let coeff = scanned_kernel.get_unchecked(i).to_ne_bytes()[0]; let rollback = length - i - 1; - let src0 = border_interpolate!( - src, - edge_mode, - i as i64 + mx, - 0, - width as i64, - N, - c - ); - let src1 = border_interpolate!( - src, - edge_mode, - rollback as i64 + mx, - 0, - width as i64, - N, - c - ); - - k0 += (src0 as u16 + src1 as u16) * coeff as u16; + k0 += (*shifted_src.get_unchecked(i * N) as u16 + + *shifted_src.get_unchecked(rollback * N) as u16) + * coeff as u16; } - *dst.get_unchecked_mut(f_cx * N + c) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + *dst.get_unchecked_mut(zx) = ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + + f_cx = m_right; + + while f_cx < width { + for c in 0..N { + let mx = f_cx as i64 - s_kernel; + let mut k0: u16 = *src.get_unchecked(f_cx * N + c) as u16 * coeff as u16; + + for i in 0..half_len { + let coeff = scanned_kernel.get_unchecked(i).to_ne_bytes()[0]; + let rollback = length - i - 1; + + let src0 = border_interpolate!( + src, + edge_mode, + i as i64 + mx, + 0, + width as i64, + N, + c + ); + let src1 = border_interpolate!( + src, + edge_mode, + rollback as i64 + mx, + 0, + width as i64, + N, + c + ); + + k0 += (src0 as u16 + src1 as u16) * coeff as u16; + } + + *dst.get_unchecked_mut(f_cx * N + c) = + ((k0 + (1 << 6)) >> 7).min(255) as u8; + } + f_cx += 1; } - f_cx += 1; } } } diff --git a/src/filter1d/sse/utils.rs b/src/filter1d/sse/utils.rs index 133a2076..42288249 100644 --- a/src/filter1d/sse/utils.rs +++ b/src/filter1d/sse/utils.rs @@ -33,102 +33,110 @@ use std::arch::x86::*; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_epi16_x4( - input: __m128i, - weight: __m128i, -) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); +pub(crate) fn _mm_mul_epi8_by_epi16_x4(input: __m128i, weight: __m128i) -> (__m128i, __m128i) { + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); - ( - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), - ) + ( + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epu16_widen(input: __m128i, weight: __m128i) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi16(input, _mm_setzero_si128()); - let k1 = _mm_unpackhi_epi16(input, _mm_setzero_si128()); +pub(crate) fn _mm_mul_epu16_widen(input: __m128i, weight: __m128i) -> (__m128i, __m128i) { + unsafe { + let k0 = _mm_unpacklo_epi16(input, _mm_setzero_si128()); + let k1 = _mm_unpackhi_epi16(input, _mm_setzero_si128()); - (_mm_mullo_epi32(k0, weight), _mm_mullo_epi32(k1, weight)) + (_mm_mullo_epi32(k0, weight), _mm_mullo_epi32(k1, weight)) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_ps_x4( +pub(crate) fn _mm_mul_epi8_by_ps_x4( input: __m128i, weight: __m128, ) -> (__m128, __m128, __m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); - let hi_16 = _mm_unpackhi_epi8(input, zeros); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); + let hi_16 = _mm_unpackhi_epi8(input, zeros); - let a0 = _mm_unpacklo_epi16(lo_16, zeros); - let a1 = _mm_unpackhi_epi16(lo_16, zeros); - let a2 = _mm_unpacklo_epi16(hi_16, zeros); - let a3 = _mm_unpackhi_epi16(hi_16, zeros); + let a0 = _mm_unpacklo_epi16(lo_16, zeros); + let a1 = _mm_unpackhi_epi16(lo_16, zeros); + let a2 = _mm_unpacklo_epi16(hi_16, zeros); + let a3 = _mm_unpackhi_epi16(hi_16, zeros); - let v0 = _mm_cvtepi32_ps(a0); - let v1 = _mm_cvtepi32_ps(a1); - let v2 = _mm_cvtepi32_ps(a2); - let v3 = _mm_cvtepi32_ps(a3); + let v0 = _mm_cvtepi32_ps(a0); + let v1 = _mm_cvtepi32_ps(a1); + let v2 = _mm_cvtepi32_ps(a2); + let v3 = _mm_cvtepi32_ps(a3); - ( - _mm_mul_ps(v0, weight), - _mm_mul_ps(v1, weight), - _mm_mul_ps(v2, weight), - _mm_mul_ps(v3, weight), - ) + ( + _mm_mul_ps(v0, weight), + _mm_mul_ps(v1, weight), + _mm_mul_ps(v2, weight), + _mm_mul_ps(v3, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi16_by_ps_x2( +pub(crate) fn _mm_mul_epi16_by_ps_x2( input: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let a0 = _mm_unpacklo_epi16(input, zeros); - let a1 = _mm_unpackhi_epi16(input, zeros); + unsafe { + let zeros = _mm_setzero_si128(); + let a0 = _mm_unpacklo_epi16(input, zeros); + let a1 = _mm_unpackhi_epi16(input, zeros); - let v0 = _mm_cvtepi32_ps(a0); - let v1 = _mm_cvtepi32_ps(a1); + let v0 = _mm_cvtepi32_ps(a0); + let v1 = _mm_cvtepi32_ps(a1); - (_mm_mul_ps(v0, weight), _mm_mul_ps(v1, weight)) + (_mm_mul_ps(v0, weight), _mm_mul_ps(v1, weight)) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_epi8_by_epi16_x4( - input: __m128i, - weight: __m128i, -) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); +pub(crate) fn _mm_mull_epi8_by_epi16_x4(input: __m128i, weight: __m128i) -> (__m128i, __m128i) { + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); - ( - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), - ) + ( + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight), + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); +pub(crate) fn _mm_mull_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_ps_x2(input: __m128i, weight: __m128) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); +pub(crate) fn _mm_mul_epi8_by_ps_x2(input: __m128i, weight: __m128) -> (__m128, __m128) { + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); - (_mm_mul_ps(a0, weight), _mm_mul_ps(a1, weight)) + (_mm_mul_ps(a0, weight), _mm_mul_ps(a1, weight)) + } } #[inline(always)] @@ -136,17 +144,21 @@ pub(crate) unsafe fn _mm_mul_epi16_by_ps( input: __m128i, weight: __m128, ) -> __m128 { - let zeros = _mm_setzero_si128(); - let k0 = _mm_unpacklo_epi16(input, zeros); - let a0 = _mm_cvtepi32_ps(k0); + unsafe { + let zeros = _mm_setzero_si128(); + let k0 = _mm_unpacklo_epi16(input, zeros); + let a0 = _mm_cvtepi32_ps(k0); - _mm_mul_ps(a0, weight) + _mm_mul_ps(a0, weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) +pub(crate) fn _mm_mul_epi8_by_epi16_x2(input: __m128i, weight: __m128i) -> __m128i { + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight) + } } #[inline(always)] @@ -155,442 +167,487 @@ pub(crate) fn _mm_opt_fmlaf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { } #[inline(always)] -pub(crate) unsafe fn _mm_fmla_pd(a: __m128d, b: __m128d, c: __m128d) -> __m128d { - _mm_add_pd(_mm_mul_pd(b, c), a) +pub(crate) fn _mm_fmla_pd(a: __m128d, b: __m128d, c: __m128d) -> __m128d { + unsafe { _mm_add_pd(_mm_mul_pd(b, c), a) } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_ps_x4( +pub(crate) fn _mm_mul_add_epi8_by_ps_x4( accumulator: (__m128, __m128, __m128, __m128), input: __m128i, weight: __m128, ) -> (__m128, __m128, __m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); - let hi_16 = _mm_unpackhi_epi8(input, zeros); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); + let hi_16 = _mm_unpackhi_epi8(input, zeros); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let k2 = _mm_unpacklo_epi16(hi_16, zeros); - let k3 = _mm_unpackhi_epi16(hi_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k2 = _mm_unpacklo_epi16(hi_16, zeros); + let k3 = _mm_unpackhi_epi16(hi_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); - let a2 = _mm_cvtepi32_ps(k2); - let a3 = _mm_cvtepi32_ps(k3); + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); + let a2 = _mm_cvtepi32_ps(k2); + let a3 = _mm_cvtepi32_ps(k3); - ( - _mm_opt_fmlaf_ps(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps(accumulator.1, a1, weight), - _mm_opt_fmlaf_ps(accumulator.2, a2, weight), - _mm_opt_fmlaf_ps(accumulator.3, a3, weight), - ) + ( + _mm_opt_fmlaf_ps(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps(accumulator.1, a1, weight), + _mm_opt_fmlaf_ps(accumulator.2, a2, weight), + _mm_opt_fmlaf_ps(accumulator.3, a3, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_epi8_by_epi16_x4( +pub(crate) fn _mm_mull_add_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); - let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); - let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); - ( - _mm_add_epi16(accumulator.0, j0), - _mm_add_epi16(accumulator.1, j1), - ) + let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); + let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); + ( + _mm_add_epi16(accumulator.0, j0), + _mm_add_epi16(accumulator.1, j1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_epi8_by_epi16_x2( +pub(crate) fn _mm_mull_add_epi8_by_epi16_x2( accumulator: __m128i, input: __m128i, weight: __m128i, ) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_add_epi16( - accumulator, - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), - ) + _mm_add_epi16( + accumulator, + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_symm_epi8_by_epi16_x4( +pub(crate) fn _mm_mull_add_symm_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); - let hi_16 = _mm_add_epi16( - _mm_unpackhi_epi8(input0, zeros), - _mm_unpackhi_epi8(input1, zeros), - ); - let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight); - let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(hi_16), weight); - - ( - _mm_add_epi16(accumulator.0, j0), - _mm_add_epi16(accumulator.1, j1), - ) -} - -#[inline(always)] -pub(crate) unsafe fn _mm_madd_symm_epi8_by_epi16_x4( + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); + let hi_16 = _mm_add_epi16( + _mm_unpackhi_epi8(input0, zeros), + _mm_unpackhi_epi8(input1, zeros), + ); + let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight); + let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(hi_16), weight); + + ( + _mm_add_epi16(accumulator.0, j0), + _mm_add_epi16(accumulator.1, j1), + ) + } +} + +#[inline(always)] +pub(crate) fn _mm_madd_symm_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); - let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); - let v2 = _mm_unpacklo_epi8(input1, _mm_setzero_si128()); - let v3 = _mm_unpackhi_epi8(input1, _mm_setzero_si128()); + unsafe { + let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); + let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); + let v2 = _mm_unpacklo_epi8(input1, _mm_setzero_si128()); + let v3 = _mm_unpackhi_epi8(input1, _mm_setzero_si128()); - let m0 = _mm_add_epi16(v0, v2); - let m1 = _mm_add_epi16(v1, v3); + let m0 = _mm_add_epi16(v0, v2); + let m1 = _mm_add_epi16(v1, v3); - let a0 = _mm_mullo_epi16(m0, weight); - let a1 = _mm_mullo_epi16(m1, weight); + let a0 = _mm_mullo_epi16(m0, weight); + let a1 = _mm_mullo_epi16(m1, weight); - ( - _mm_add_epi16(accumulator.0, a0), - _mm_add_epi16(accumulator.1, a1), - ) + ( + _mm_add_epi16(accumulator.0, a0), + _mm_add_epi16(accumulator.1, a1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_madd_s_epi8_by_epi16_x4( +pub(crate) fn _mm_madd_s_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); - let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); + unsafe { + let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); + let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); - let a0 = _mm_mullo_epi16(v0, weight); - let a1 = _mm_mullo_epi16(v1, weight); + let a0 = _mm_mullo_epi16(v0, weight); + let a1 = _mm_mullo_epi16(v1, weight); - ( - _mm_add_epi16(accumulator.0, a0), - _mm_add_epi16(accumulator.1, a1), - ) + ( + _mm_add_epi16(accumulator.0, a0), + _mm_add_epi16(accumulator.1, a1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_madd_epi8_by_epi16_x4( - input0: __m128i, - weight: __m128i, -) -> (__m128i, __m128i) { - let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); - let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); +pub(crate) fn _mm_madd_epi8_by_epi16_x4(input0: __m128i, weight: __m128i) -> (__m128i, __m128i) { + unsafe { + let v0 = _mm_unpacklo_epi8(input0, _mm_setzero_si128()); + let v1 = _mm_unpackhi_epi8(input0, _mm_setzero_si128()); - let a0 = _mm_mullo_epi16(v0, weight); - let a1 = _mm_mullo_epi16(v1, weight); + let a0 = _mm_mullo_epi16(v0, weight); + let a1 = _mm_mullo_epi16(v1, weight); - (a0, a1) + (a0, a1) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_ps_x4( +pub(crate) fn _mm_mul_add_symm_epi8_by_ps_x4( accumulator: (__m128, __m128, __m128, __m128), input0: __m128i, input1: __m128i, weight: __m128, ) -> (__m128, __m128, __m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); - let hi_16 = _mm_add_epi16( - _mm_unpackhi_epi8(input0, zeros), - _mm_unpackhi_epi8(input1, zeros), - ); - - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let k2 = _mm_unpacklo_epi16(hi_16, zeros); - let k3 = _mm_unpackhi_epi16(hi_16, zeros); - - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); - let a2 = _mm_cvtepi32_ps(k2); - let a3 = _mm_cvtepi32_ps(k3); - - ( - _mm_opt_fmlaf_ps(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps(accumulator.1, a1, weight), - _mm_opt_fmlaf_ps(accumulator.2, a2, weight), - _mm_opt_fmlaf_ps(accumulator.3, a3, weight), - ) -} - -#[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi16_by_ps_x2( + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); + let hi_16 = _mm_add_epi16( + _mm_unpackhi_epi8(input0, zeros), + _mm_unpackhi_epi8(input1, zeros), + ); + + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k2 = _mm_unpacklo_epi16(hi_16, zeros); + let k3 = _mm_unpackhi_epi16(hi_16, zeros); + + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); + let a2 = _mm_cvtepi32_ps(k2); + let a3 = _mm_cvtepi32_ps(k3); + + ( + _mm_opt_fmlaf_ps(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps(accumulator.1, a1, weight), + _mm_opt_fmlaf_ps(accumulator.2, a2, weight), + _mm_opt_fmlaf_ps(accumulator.3, a3, weight), + ) + } +} + +#[inline(always)] +pub(crate) fn _mm_mul_add_symm_epi16_by_ps_x2( accumulator: (__m128, __m128), input0: __m128i, input1: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let l32 = _mm_add_epi32( - _mm_unpacklo_epi16(input0, zeros), - _mm_unpacklo_epi16(input1, zeros), - ); - let h32 = _mm_add_epi32( - _mm_unpackhi_epi16(input0, zeros), - _mm_unpackhi_epi16(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let l32 = _mm_add_epi32( + _mm_unpacklo_epi16(input0, zeros), + _mm_unpacklo_epi16(input1, zeros), + ); + let h32 = _mm_add_epi32( + _mm_unpackhi_epi16(input0, zeros), + _mm_unpackhi_epi16(input1, zeros), + ); - let a0 = _mm_cvtepi32_ps(l32); - let a1 = _mm_cvtepi32_ps(h32); + let a0 = _mm_cvtepi32_ps(l32); + let a1 = _mm_cvtepi32_ps(h32); - ( - _mm_opt_fmlaf_ps(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps(accumulator.1, a1, weight), - ) + ( + _mm_opt_fmlaf_ps(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps(accumulator.1, a1, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_ps_x2( +pub(crate) fn _mm_mul_add_epi8_by_ps_x2( accumulator: (__m128, __m128), input: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_unpacklo_epi8(input, zeros); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_unpacklo_epi8(input, zeros); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); - ( - _mm_opt_fmlaf_ps(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps(accumulator.1, a1, weight), - ) + ( + _mm_opt_fmlaf_ps(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps(accumulator.1, a1, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_ps_x2( +pub(crate) fn _mm_mul_add_symm_epi8_by_ps_x2( accumulator: (__m128, __m128), input0: __m128i, input1: __m128i, weight: __m128, ) -> (__m128, __m128) { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); - let k0 = _mm_unpacklo_epi16(lo_16, zeros); - let k1 = _mm_unpackhi_epi16(lo_16, zeros); + let k0 = _mm_unpacklo_epi16(lo_16, zeros); + let k1 = _mm_unpackhi_epi16(lo_16, zeros); - let a0 = _mm_cvtepi32_ps(k0); - let a1 = _mm_cvtepi32_ps(k1); + let a0 = _mm_cvtepi32_ps(k0); + let a1 = _mm_cvtepi32_ps(k1); - ( - _mm_opt_fmlaf_ps(accumulator.0, a0, weight), - _mm_opt_fmlaf_ps(accumulator.1, a1, weight), - ) + ( + _mm_opt_fmlaf_ps(accumulator.0, a0, weight), + _mm_opt_fmlaf_ps(accumulator.1, a1, weight), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi16_by_ps( +pub(crate) fn _mm_mul_add_symm_epi16_by_ps( accumulator: __m128, input0: __m128i, input1: __m128i, weight: __m128, ) -> __m128 { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi32( - _mm_unpacklo_epi16(input0, zeros), - _mm_unpacklo_epi16(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi32( + _mm_unpacklo_epi16(input0, zeros), + _mm_unpacklo_epi16(input1, zeros), + ); - let a0 = _mm_cvtepi32_ps(lo_16); + let a0 = _mm_cvtepi32_ps(lo_16); - _mm_opt_fmlaf_ps(accumulator, a0, weight) + _mm_opt_fmlaf_ps(accumulator, a0, weight) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mull_add_symm_epi8_by_epi16_x2( +pub(crate) fn _mm_mull_add_symm_epi8_by_epi16_x2( accumulator: __m128i, input0: __m128i, input1: __m128i, weight: __m128i, ) -> __m128i { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); - _mm_add_epi16(accumulator, _mm_mulhrs_epi16(lo_16, weight)) + _mm_add_epi16(accumulator, _mm_mulhrs_epi16(lo_16, weight)) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_epi16_x4( +pub(crate) fn _mm_mul_add_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let k0 = _mm_unpacklo_epi8(input, input); - let k1 = _mm_unpackhi_epi8(input, input); + unsafe { + let k0 = _mm_unpacklo_epi8(input, input); + let k1 = _mm_unpackhi_epi8(input, input); - let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); - let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); + let j0 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k0), weight); + let j1 = _mm_mulhrs_epi16(_mm_srli_epi16::<2>(k1), weight); - ( - _mm_add_epi16(j0, accumulator.0), - _mm_add_epi16(j1, accumulator.1), - ) + ( + _mm_add_epi16(j0, accumulator.0), + _mm_add_epi16(j1, accumulator.1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_epi16_x4( +pub(crate) fn _mm_mul_add_symm_epi8_by_epi16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let zeros = _mm_setzero_si128(); - let k0 = _mm_unpacklo_epi8(input0, zeros); - let k1 = _mm_unpacklo_epi8(input1, zeros); - let k2 = _mm_unpackhi_epi8(input0, zeros); - let k3 = _mm_unpackhi_epi8(input1, zeros); - let v0 = _mm_add_epi16(k0, k1); - let v1 = _mm_add_epi16(k2, k3); + unsafe { + let zeros = _mm_setzero_si128(); + let k0 = _mm_unpacklo_epi8(input0, zeros); + let k1 = _mm_unpacklo_epi8(input1, zeros); + let k2 = _mm_unpackhi_epi8(input0, zeros); + let k3 = _mm_unpackhi_epi8(input1, zeros); + let v0 = _mm_add_epi16(k0, k1); + let v1 = _mm_add_epi16(k2, k3); - let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v0), weight); - let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v1), weight); + let j0 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v0), weight); + let j1 = _mm_mulhrs_epi16(_mm_slli_epi16::<6>(v1), weight); - ( - _mm_add_epi16(j0, accumulator.0), - _mm_add_epi16(j1, accumulator.1), - ) + ( + _mm_add_epi16(j0, accumulator.0), + _mm_add_epi16(j1, accumulator.1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epu16_by_epu16_x4( +pub(crate) fn _mm_mul_add_symm_epu16_by_epu16_x4( accumulator: (__m128i, __m128i), input0: __m128i, input1: __m128i, weight: __m128i, ) -> (__m128i, __m128i) { - let zeros = _mm_setzero_si128(); - let k0 = _mm_unpacklo_epi16(input0, zeros); - let k1 = _mm_unpacklo_epi16(input1, zeros); - let k2 = _mm_unpackhi_epi16(input0, zeros); - let k3 = _mm_unpackhi_epi16(input1, zeros); - let v0 = _mm_add_epi32(k0, k1); - let v1 = _mm_add_epi32(k2, k3); + unsafe { + let zeros = _mm_setzero_si128(); + let k0 = _mm_unpacklo_epi16(input0, zeros); + let k1 = _mm_unpacklo_epi16(input1, zeros); + let k2 = _mm_unpackhi_epi16(input0, zeros); + let k3 = _mm_unpackhi_epi16(input1, zeros); + let v0 = _mm_add_epi32(k0, k1); + let v1 = _mm_add_epi32(k2, k3); - let j0 = _mm_mullo_epi32(v0, weight); - let j1 = _mm_mullo_epi32(v1, weight); + let j0 = _mm_mullo_epi32(v0, weight); + let j1 = _mm_mullo_epi32(v1, weight); - ( - _mm_add_epi32(j0, accumulator.0), - _mm_add_epi32(j1, accumulator.1), - ) + ( + _mm_add_epi32(j0, accumulator.0), + _mm_add_epi32(j1, accumulator.1), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_epi8_by_epi16_x2( +pub(crate) fn _mm_mul_add_epi8_by_epi16_x2( accumulator: __m128i, input: __m128i, weight: __m128i, ) -> __m128i { - let lo_16 = _mm_unpacklo_epi8(input, input); + unsafe { + let lo_16 = _mm_unpacklo_epi8(input, input); - _mm_add_epi16( - _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), - accumulator, - ) + _mm_add_epi16( + _mm_mulhrs_epi16(_mm_srli_epi16::<2>(lo_16), weight), + accumulator, + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_add_symm_epi8_by_epi16_x2( +pub(crate) fn _mm_mul_add_symm_epi8_by_epi16_x2( accumulator: __m128i, input0: __m128i, input1: __m128i, weight: __m128i, ) -> __m128i { - let zeros = _mm_setzero_si128(); - let lo_16 = _mm_add_epi16( - _mm_unpacklo_epi8(input0, zeros), - _mm_unpacklo_epi8(input1, zeros), - ); + unsafe { + let zeros = _mm_setzero_si128(); + let lo_16 = _mm_add_epi16( + _mm_unpacklo_epi8(input0, zeros), + _mm_unpacklo_epi8(input1, zeros), + ); - _mm_add_epi16( - _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight), - accumulator, - ) + _mm_add_epi16( + _mm_mulhrs_epi16(_mm_slli_epi16::<6>(lo_16), weight), + accumulator, + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_epi32_x2_epi8(store: (__m128i, __m128i)) -> __m128i { - let rnd = _mm_set1_epi16((1 << 5) - 1); - _mm_packus_epi16( - _mm_srai_epi16::<6>(_mm_add_epi16(store.0, rnd)), - _mm_srai_epi16::<6>(_mm_add_epi16(store.1, rnd)), - ) +pub(crate) fn _mm_pack_epi32_x2_epi8(store: (__m128i, __m128i)) -> __m128i { + unsafe { + let rnd = _mm_set1_epi16((1 << 5) - 1); + _mm_packus_epi16( + _mm_srai_epi16::<6>(_mm_add_epi16(store.0, rnd)), + _mm_srai_epi16::<6>(_mm_add_epi16(store.1, rnd)), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_epi32_x2_epu16(store: (__m128i, __m128i)) -> __m128i { - let rnd = _mm_set1_epi32((1 << 14) - 1); - _mm_packus_epi32( - _mm_srli_epi32::<15>(_mm_add_epi32(store.0, rnd)), - _mm_srli_epi32::<15>(_mm_add_epi32(store.1, rnd)), - ) +pub(crate) fn _mm_pack_epi32_x2_epu16(store: (__m128i, __m128i)) -> __m128i { + unsafe { + let rnd = _mm_set1_epi32((1 << 14) - 1); + _mm_packus_epi32( + _mm_srli_epi32::<15>(_mm_add_epi32(store.0, rnd)), + _mm_srli_epi32::<15>(_mm_add_epi32(store.1, rnd)), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_epi32_epi8(store: __m128i) -> __m128i { - let rnd = _mm_set1_epi16((1 << 5) - 1); - _mm_packus_epi16( - _mm_srai_epi16::<6>(_mm_add_epi16(store, rnd)), - _mm_setzero_si128(), - ) +pub(crate) fn _mm_pack_epi32_epi8(store: __m128i) -> __m128i { + unsafe { + let rnd = _mm_set1_epi16((1 << 5) - 1); + _mm_packus_epi16( + _mm_srai_epi16::<6>(_mm_add_epi16(store, rnd)), + _mm_setzero_si128(), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_x4_epi8(store: (__m128, __m128, __m128, __m128)) -> __m128i { - let o0 = _mm_cvtps_epi32(store.0); - let o1 = _mm_cvtps_epi32(store.1); - let o2 = _mm_cvtps_epi32(store.2); - let o3 = _mm_cvtps_epi32(store.3); - let hi_s = _mm_packs_epi32(o2, o3); - let lo_s = _mm_packs_epi32(o0, o1); - _mm_packus_epi16(lo_s, hi_s) +pub(crate) fn _mm_pack_ps_x4_epi8(store: (__m128, __m128, __m128, __m128)) -> __m128i { + unsafe { + let o0 = _mm_cvtps_epi32(store.0); + let o1 = _mm_cvtps_epi32(store.1); + let o2 = _mm_cvtps_epi32(store.2); + let o3 = _mm_cvtps_epi32(store.3); + let hi_s = _mm_packs_epi32(o2, o3); + let lo_s = _mm_packs_epi32(o0, o1); + _mm_packus_epi16(lo_s, hi_s) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_x2_epi16(store: (__m128, __m128)) -> __m128i { - let o0 = _mm_cvtps_epi32(store.0); - let o1 = _mm_cvtps_epi32(store.1); - _mm_packus_epi32(o0, o1) +pub(crate) fn _mm_pack_ps_x2_epi16(store: (__m128, __m128)) -> __m128i { + unsafe { + let o0 = _mm_cvtps_epi32(store.0); + let o1 = _mm_cvtps_epi32(store.1); + _mm_packus_epi32(o0, o1) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_x2_epi8(store: (__m128, __m128)) -> __m128i { - let lo_s = _mm_packs_epi32(_mm_cvtps_epi32(store.0), _mm_cvtps_epi32(store.1)); - _mm_packus_epi16(lo_s, _mm_setzero_si128()) +pub(crate) fn _mm_pack_ps_x2_epi8(store: (__m128, __m128)) -> __m128i { + unsafe { + let lo_s = _mm_packs_epi32(_mm_cvtps_epi32(store.0), _mm_cvtps_epi32(store.1)); + _mm_packus_epi16(lo_s, _mm_setzero_si128()) + } } #[inline(always)] -pub(crate) unsafe fn _mm_pack_ps_epi16(store: __m128) -> __m128i { - _mm_packus_epi32(_mm_cvtps_epi32(store), _mm_setzero_si128()) +pub(crate) fn _mm_pack_ps_epi16(store: __m128) -> __m128i { + unsafe { _mm_packus_epi32(_mm_cvtps_epi32(store), _mm_setzero_si128()) } } diff --git a/src/filter2d/avx/convolve_op_u8_i16_fp.rs b/src/filter2d/avx/convolve_op_u8_i16_fp.rs index 786703ff..fcf10661 100644 --- a/src/filter2d/avx/convolve_op_u8_i16_fp.rs +++ b/src/filter2d/avx/convolve_op_u8_i16_fp.rs @@ -26,11 +26,11 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; use crate::avx::{_mm256_load_pack_x2, _mm256_store_pack_x2, shuffle}; use crate::filter1d::{Arena, ToApproxStorage}; use crate::filter2d::scan_point_2d::ScanPoint2d; use crate::mlaf::mlaf; -use crate::ImageSize; use std::arch::x86_64::*; use std::ops::Mul; @@ -48,7 +48,7 @@ pub(crate) fn convolve_segment_sse_2d_u8_i16_fp( } #[target_feature(enable = "avx2")] -unsafe fn convolve_segment_2d_u8_i16_impl( +fn convolve_segment_2d_u8_i16_impl( arena: Arena, arena_source: &[u8], dst: &mut [u8], @@ -56,321 +56,323 @@ unsafe fn convolve_segment_2d_u8_i16_impl( prepared_kernel: &[ScanPoint2d], y: usize, ) { - let width = image_size.width; - - let dx = arena.pad_w as i64; - let dy = arena.pad_h as i64; - - let arena_stride = arena.width * arena.components; - - let v_prepared = prepared_kernel - .iter() - .map(|&x| { - let z = x.weight.to_ne_bytes(); - i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) - }) - .collect::>(); - - let offsets = prepared_kernel - .iter() - .map(|&x| { - arena_source.get_unchecked( - ((x.y + dy + y as i64) as usize * arena_stride - + (x.x + dx) as usize * arena.components).., - ) - }) - .collect::>(); - - let length = prepared_kernel.len(); - - let total_width = width * arena.components; - - let mut cx = 0usize; - - let k_weight = _mm256_set1_epi32(*v_prepared.get_unchecked(0)); - - let rnd = _mm256_set1_epi32(1i32 << 14); - - let off0 = offsets.get_unchecked(0); - - while cx + 64 < total_width { - let items0 = _mm256_load_pack_x2(off0.get_unchecked(cx..).as_ptr()); - - let lo0 = _mm256_unpacklo_epi8(items0.0, _mm256_setzero_si256()); - let hi0 = _mm256_unpackhi_epi8(items0.0, _mm256_setzero_si256()); - let hi1 = _mm256_unpackhi_epi8(items0.1, _mm256_setzero_si256()); - let lo1 = _mm256_unpacklo_epi8(items0.1, _mm256_setzero_si256()); - - let mut k0 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()), k_weight), - ); - let mut k1 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()), k_weight), - ); - let mut k2 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpacklo_epi16(hi0, _mm256_setzero_si256()), k_weight), - ); - let mut k3 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpackhi_epi16(hi0, _mm256_setzero_si256()), k_weight), - ); - let mut k4 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpacklo_epi16(lo1, _mm256_setzero_si256()), k_weight), - ); - let mut k5 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpackhi_epi16(lo1, _mm256_setzero_si256()), k_weight), - ); - let mut k6 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpacklo_epi16(hi1, _mm256_setzero_si256()), k_weight), - ); - let mut k7 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpackhi_epi16(hi1, _mm256_setzero_si256()), k_weight), - ); - - for i in 1..length { - let weight = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let s_ptr = offsets.get_unchecked(i); - let items0 = _mm256_load_pack_x2(s_ptr.get_unchecked(cx..).as_ptr()); + unsafe { + let width = image_size.width; + + let dx = arena.pad_w as i64; + let dy = arena.pad_h as i64; + + let arena_stride = arena.width * arena.components; + + let v_prepared = prepared_kernel + .iter() + .map(|&x| { + let z = x.weight.to_ne_bytes(); + i32::from_ne_bytes([z[0], z[1], z[0], z[1]]) + }) + .collect::>(); + + let offsets = prepared_kernel + .iter() + .map(|&x| { + arena_source.get_unchecked( + ((x.y + dy + y as i64) as usize * arena_stride + + (x.x + dx) as usize * arena.components).., + ) + }) + .collect::>(); + + let length = prepared_kernel.len(); + + let total_width = width * arena.components; + + let mut cx = 0usize; + + let k_weight = _mm256_set1_epi32(*v_prepared.get_unchecked(0)); + + let rnd = _mm256_set1_epi32(1i32 << 14); + + let off0 = offsets.get_unchecked(0); + + while cx + 64 <= total_width { + let items0 = _mm256_load_pack_x2(off0.get_unchecked(cx..).as_ptr()); let lo0 = _mm256_unpacklo_epi8(items0.0, _mm256_setzero_si256()); let hi0 = _mm256_unpackhi_epi8(items0.0, _mm256_setzero_si256()); let hi1 = _mm256_unpackhi_epi8(items0.1, _mm256_setzero_si256()); let lo1 = _mm256_unpacklo_epi8(items0.1, _mm256_setzero_si256()); - k0 = _mm256_add_epi32( - k0, - _mm256_madd_epi16(_mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()), weight), + let mut k0 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()), k_weight), ); - k1 = _mm256_add_epi32( - k1, - _mm256_madd_epi16(_mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()), weight), + let mut k1 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()), k_weight), ); - k2 = _mm256_add_epi32( - k2, - _mm256_madd_epi16(_mm256_unpacklo_epi16(hi0, _mm256_setzero_si256()), weight), + let mut k2 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpacklo_epi16(hi0, _mm256_setzero_si256()), k_weight), ); - k3 = _mm256_add_epi32( - k3, - _mm256_madd_epi16(_mm256_unpackhi_epi16(hi0, _mm256_setzero_si256()), weight), + let mut k3 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpackhi_epi16(hi0, _mm256_setzero_si256()), k_weight), ); - k4 = _mm256_add_epi32( - k4, - _mm256_madd_epi16(_mm256_unpacklo_epi16(lo1, _mm256_setzero_si256()), weight), + let mut k4 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpacklo_epi16(lo1, _mm256_setzero_si256()), k_weight), ); - k5 = _mm256_add_epi32( - k5, - _mm256_madd_epi16(_mm256_unpackhi_epi16(lo1, _mm256_setzero_si256()), weight), + let mut k5 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpackhi_epi16(lo1, _mm256_setzero_si256()), k_weight), ); - k6 = _mm256_add_epi32( - k6, - _mm256_madd_epi16(_mm256_unpacklo_epi16(hi1, _mm256_setzero_si256()), weight), + let mut k6 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpacklo_epi16(hi1, _mm256_setzero_si256()), k_weight), ); - k7 = _mm256_add_epi32( - k7, - _mm256_madd_epi16(_mm256_unpackhi_epi16(hi1, _mm256_setzero_si256()), weight), + let mut k7 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpackhi_epi16(hi1, _mm256_setzero_si256()), k_weight), ); - } - - k0 = _mm256_srai_epi32::<15>(k0); - k1 = _mm256_srai_epi32::<15>(k1); - k2 = _mm256_srai_epi32::<15>(k2); - k3 = _mm256_srai_epi32::<15>(k3); - k4 = _mm256_srai_epi32::<15>(k4); - k5 = _mm256_srai_epi32::<15>(k5); - k6 = _mm256_srai_epi32::<15>(k6); - k7 = _mm256_srai_epi32::<15>(k7); - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - let z0 = _mm256_packus_epi32(k0, k1); - let z1 = _mm256_packus_epi32(k2, k3); - let z2 = _mm256_packus_epi32(k4, k5); - let z3 = _mm256_packus_epi32(k6, k7); - - let r0 = _mm256_packus_epi16(z0, z1); - let r1 = _mm256_packus_epi16(z2, z3); - - _mm256_store_pack_x2(dst_ptr0, (r0, r1)); - cx += 64; - } + for i in 1..length { + let weight = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let s_ptr = offsets.get_unchecked(i); + let items0 = _mm256_load_pack_x2(s_ptr.get_unchecked(cx..).as_ptr()); + + let lo0 = _mm256_unpacklo_epi8(items0.0, _mm256_setzero_si256()); + let hi0 = _mm256_unpackhi_epi8(items0.0, _mm256_setzero_si256()); + let hi1 = _mm256_unpackhi_epi8(items0.1, _mm256_setzero_si256()); + let lo1 = _mm256_unpacklo_epi8(items0.1, _mm256_setzero_si256()); + + k0 = _mm256_add_epi32( + k0, + _mm256_madd_epi16(_mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()), weight), + ); + k1 = _mm256_add_epi32( + k1, + _mm256_madd_epi16(_mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()), weight), + ); + k2 = _mm256_add_epi32( + k2, + _mm256_madd_epi16(_mm256_unpacklo_epi16(hi0, _mm256_setzero_si256()), weight), + ); + k3 = _mm256_add_epi32( + k3, + _mm256_madd_epi16(_mm256_unpackhi_epi16(hi0, _mm256_setzero_si256()), weight), + ); + k4 = _mm256_add_epi32( + k4, + _mm256_madd_epi16(_mm256_unpacklo_epi16(lo1, _mm256_setzero_si256()), weight), + ); + k5 = _mm256_add_epi32( + k5, + _mm256_madd_epi16(_mm256_unpackhi_epi16(lo1, _mm256_setzero_si256()), weight), + ); + k6 = _mm256_add_epi32( + k6, + _mm256_madd_epi16(_mm256_unpacklo_epi16(hi1, _mm256_setzero_si256()), weight), + ); + k7 = _mm256_add_epi32( + k7, + _mm256_madd_epi16(_mm256_unpackhi_epi16(hi1, _mm256_setzero_si256()), weight), + ); + } + + k0 = _mm256_srai_epi32::<15>(k0); + k1 = _mm256_srai_epi32::<15>(k1); + k2 = _mm256_srai_epi32::<15>(k2); + k3 = _mm256_srai_epi32::<15>(k3); + k4 = _mm256_srai_epi32::<15>(k4); + k5 = _mm256_srai_epi32::<15>(k5); + k6 = _mm256_srai_epi32::<15>(k6); + k7 = _mm256_srai_epi32::<15>(k7); + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + + let z0 = _mm256_packus_epi32(k0, k1); + let z1 = _mm256_packus_epi32(k2, k3); + let z2 = _mm256_packus_epi32(k4, k5); + let z3 = _mm256_packus_epi32(k6, k7); + + let r0 = _mm256_packus_epi16(z0, z1); + let r1 = _mm256_packus_epi16(z2, z3); + + _mm256_store_pack_x2(dst_ptr0, (r0, r1)); + cx += 64; + } - while cx + 32 < total_width { - let items0 = _mm256_loadu_si256(off0.get_unchecked(cx..).as_ptr() as *const _); - - let lo0 = _mm256_unpacklo_epi8(items0, _mm256_setzero_si256()); - let hi0 = _mm256_unpackhi_epi8(items0, _mm256_setzero_si256()); - - let mut k0 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()), k_weight), - ); - let mut k1 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()), k_weight), - ); - let mut k2 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpacklo_epi16(hi0, _mm256_setzero_si256()), k_weight), - ); - let mut k3 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16(_mm256_unpackhi_epi16(hi0, _mm256_setzero_si256()), k_weight), - ); - - for i in 1..length { - let weight = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); - let s_ptr = offsets.get_unchecked(i); - let items0 = _mm256_loadu_si256(s_ptr.get_unchecked(cx..).as_ptr() as *const _); + while cx + 32 <= total_width { + let items0 = _mm256_loadu_si256(off0.get_unchecked(cx..).as_ptr() as *const _); let lo0 = _mm256_unpacklo_epi8(items0, _mm256_setzero_si256()); let hi0 = _mm256_unpackhi_epi8(items0, _mm256_setzero_si256()); - k0 = _mm256_add_epi32( - k0, - _mm256_madd_epi16(_mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()), weight), + let mut k0 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()), k_weight), ); - k1 = _mm256_add_epi32( - k1, - _mm256_madd_epi16(_mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()), weight), + let mut k1 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()), k_weight), ); - k2 = _mm256_add_epi32( - k2, - _mm256_madd_epi16(_mm256_unpacklo_epi16(hi0, _mm256_setzero_si256()), weight), + let mut k2 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpacklo_epi16(hi0, _mm256_setzero_si256()), k_weight), ); - k3 = _mm256_add_epi32( - k3, - _mm256_madd_epi16(_mm256_unpackhi_epi16(hi0, _mm256_setzero_si256()), weight), + let mut k3 = _mm256_add_epi32( + rnd, + _mm256_madd_epi16(_mm256_unpackhi_epi16(hi0, _mm256_setzero_si256()), k_weight), ); - } - k0 = _mm256_srai_epi32::<15>(k0); - k1 = _mm256_srai_epi32::<15>(k1); - k2 = _mm256_srai_epi32::<15>(k2); - k3 = _mm256_srai_epi32::<15>(k3); - - let z0 = _mm256_packus_epi32(k0, k1); - let z1 = _mm256_packus_epi32(k2, k3); - - let r0 = _mm256_packus_epi16(z0, z1); - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm256_storeu_si256(dst_ptr0 as *mut _, r0); - cx += 32; - } + for i in 1..length { + let weight = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let s_ptr = offsets.get_unchecked(i); + let items0 = _mm256_loadu_si256(s_ptr.get_unchecked(cx..).as_ptr() as *const _); + + let lo0 = _mm256_unpacklo_epi8(items0, _mm256_setzero_si256()); + let hi0 = _mm256_unpackhi_epi8(items0, _mm256_setzero_si256()); + + k0 = _mm256_add_epi32( + k0, + _mm256_madd_epi16(_mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()), weight), + ); + k1 = _mm256_add_epi32( + k1, + _mm256_madd_epi16(_mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()), weight), + ); + k2 = _mm256_add_epi32( + k2, + _mm256_madd_epi16(_mm256_unpacklo_epi16(hi0, _mm256_setzero_si256()), weight), + ); + k3 = _mm256_add_epi32( + k3, + _mm256_madd_epi16(_mm256_unpackhi_epi16(hi0, _mm256_setzero_si256()), weight), + ); + } + + k0 = _mm256_srai_epi32::<15>(k0); + k1 = _mm256_srai_epi32::<15>(k1); + k2 = _mm256_srai_epi32::<15>(k2); + k3 = _mm256_srai_epi32::<15>(k3); + + let z0 = _mm256_packus_epi32(k0, k1); + let z1 = _mm256_packus_epi32(k2, k3); + + let r0 = _mm256_packus_epi16(z0, z1); + + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm256_storeu_si256(dst_ptr0 as *mut _, r0); + cx += 32; + } - while cx + 16 < total_width { - let items0 = _mm256_cvtepu8_epi16(_mm_loadu_si128( - off0.get_unchecked(cx..).as_ptr() as *const __m128i - )); - - let mut k0 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16( - _mm256_unpacklo_epi16(items0, _mm256_setzero_si256()), - k_weight, - ), - ); - let mut k1 = _mm256_add_epi32( - rnd, - _mm256_madd_epi16( - _mm256_unpackhi_epi16(items0, _mm256_setzero_si256()), - k_weight, - ), - ); - - for i in 1..length { - let weight = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + while cx + 16 <= total_width { let items0 = _mm256_cvtepu8_epi16(_mm_loadu_si128( - offsets.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, + off0.get_unchecked(cx..).as_ptr() as *const __m128i )); - k0 = _mm256_add_epi32( - k0, + + let mut k0 = _mm256_add_epi32( + rnd, _mm256_madd_epi16( _mm256_unpacklo_epi16(items0, _mm256_setzero_si256()), - weight, + k_weight, ), ); - k1 = _mm256_add_epi32( - k1, + let mut k1 = _mm256_add_epi32( + rnd, _mm256_madd_epi16( _mm256_unpackhi_epi16(items0, _mm256_setzero_si256()), - weight, + k_weight, ), ); - } - - k0 = _mm256_srai_epi32::<15>(k0); - k1 = _mm256_srai_epi32::<15>(k1); - - let z0 = _mm256_packus_epi32(k0, k1); - - let p0 = _mm256_permute4x64_epi64::<{ shuffle(3, 1, 2, 0) }>(_mm256_packus_epi16( - z0, - _mm256_setzero_si256(), - )); - - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm256_castsi256_si128(p0)); - cx += 16; - } - - let k_weight = prepared_kernel.get_unchecked(0).weight; - while cx + 4 < total_width { - let mut k0 = ((*off0.get_unchecked(cx)) as i32).mul(k_weight as i32); - let mut k1 = ((*off0.get_unchecked(cx + 1)) as i32).mul(k_weight as i32); - let mut k2 = ((*off0.get_unchecked(cx + 2)) as i32).mul(k_weight as i32); - let mut k3 = ((*off0.get_unchecked(cx + 3)) as i32).mul(k_weight as i32); + for i in 1..length { + let weight = _mm256_set1_epi32(*v_prepared.get_unchecked(i)); + let items0 = _mm256_cvtepu8_epi16(_mm_loadu_si128( + offsets.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i, + )); + k0 = _mm256_add_epi32( + k0, + _mm256_madd_epi16( + _mm256_unpacklo_epi16(items0, _mm256_setzero_si256()), + weight, + ), + ); + k1 = _mm256_add_epi32( + k1, + _mm256_madd_epi16( + _mm256_unpackhi_epi16(items0, _mm256_setzero_si256()), + weight, + ), + ); + } + + k0 = _mm256_srai_epi32::<15>(k0); + k1 = _mm256_srai_epi32::<15>(k1); + + let z0 = _mm256_packus_epi32(k0, k1); + + let p0 = _mm256_permute4x64_epi64::<{ shuffle(3, 1, 2, 0) }>(_mm256_packus_epi16( + z0, + _mm256_setzero_si256(), + )); - for i in 1..length { - let weight = prepared_kernel.get_unchecked(i).weight; - k0 = mlaf( - k0, - (*offsets.get_unchecked(i).get_unchecked(cx)) as i32, - weight as i32, - ); - k1 = mlaf( - k1, - (*offsets.get_unchecked(i).get_unchecked(cx + 1)) as i32, - weight as i32, - ); - k2 = mlaf( - k2, - (*offsets.get_unchecked(i).get_unchecked(cx + 2)) as i32, - weight as i32, - ); - k3 = mlaf( - k3, - (*offsets.get_unchecked(i).get_unchecked(cx + 3)) as i32, - weight as i32, - ); + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm256_castsi256_si128(p0)); + cx += 16; } - *dst.get_unchecked_mut(cx) = k0.to_approx_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_approx_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_approx_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_approx_(); - cx += 4; - } - - for x in cx..total_width { - let mut k0 = ((*(*off0).get_unchecked(x)) as i32).mul(k_weight as i32); + let k_weight = prepared_kernel.get_unchecked(0).weight; + + while cx + 4 <= total_width { + let mut k0 = ((*off0.get_unchecked(cx)) as i32).mul(k_weight as i32); + let mut k1 = ((*off0.get_unchecked(cx + 1)) as i32).mul(k_weight as i32); + let mut k2 = ((*off0.get_unchecked(cx + 2)) as i32).mul(k_weight as i32); + let mut k3 = ((*off0.get_unchecked(cx + 3)) as i32).mul(k_weight as i32); + + for i in 1..length { + let weight = prepared_kernel.get_unchecked(i).weight; + k0 = mlaf( + k0, + (*offsets.get_unchecked(i).get_unchecked(cx)) as i32, + weight as i32, + ); + k1 = mlaf( + k1, + (*offsets.get_unchecked(i).get_unchecked(cx + 1)) as i32, + weight as i32, + ); + k2 = mlaf( + k2, + (*offsets.get_unchecked(i).get_unchecked(cx + 2)) as i32, + weight as i32, + ); + k3 = mlaf( + k3, + (*offsets.get_unchecked(i).get_unchecked(cx + 3)) as i32, + weight as i32, + ); + } + + *dst.get_unchecked_mut(cx) = k0.to_approx_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_approx_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_approx_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_approx_(); + cx += 4; + } - for i in 1..length { - let k_weight = prepared_kernel.get_unchecked(i).weight; - k0 = mlaf( - k0, - (*offsets.get_unchecked(i).get_unchecked(x)) as i32, - k_weight as i32, - ); + for x in cx..total_width { + let mut k0 = ((*(*off0).get_unchecked(x)) as i32).mul(k_weight as i32); + + for i in 1..length { + let k_weight = prepared_kernel.get_unchecked(i).weight; + k0 = mlaf( + k0, + (*offsets.get_unchecked(i).get_unchecked(x)) as i32, + k_weight as i32, + ); + } + *dst.get_unchecked_mut(x) = k0.to_approx_(); } - *dst.get_unchecked_mut(x) = k0.to_approx_(); } } diff --git a/src/filter2d/convolve_op.rs b/src/filter2d/convolve_op.rs index b1117d47..51b79f89 100644 --- a/src/filter2d/convolve_op.rs +++ b/src/filter2d/convolve_op.rs @@ -26,11 +26,11 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; use crate::filter1d::Arena; use crate::filter2d::scan_point_2d::ScanPoint2d; use crate::mlaf::mlaf; use crate::to_storage::ToStorage; -use crate::ImageSize; use num_traits::{AsPrimitive, MulAdd}; use std::ops::{Add, Mul}; diff --git a/src/filter2d/convolve_op_fp.rs b/src/filter2d/convolve_op_fp.rs index d75cd74b..f758cfc5 100644 --- a/src/filter2d/convolve_op_fp.rs +++ b/src/filter2d/convolve_op_fp.rs @@ -27,10 +27,10 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #![allow(dead_code)] +use crate::ImageSize; use crate::filter1d::{Arena, ToApproxStorage}; use crate::filter2d::scan_point_2d::ScanPoint2d; use crate::mlaf::mlaf; -use crate::ImageSize; use num_traits::AsPrimitive; use std::ops::Mul; diff --git a/src/filter2d/fft_utils.rs b/src/filter2d/fft_utils.rs index 1270e5f7..278016f8 100644 --- a/src/filter2d/fft_utils.rs +++ b/src/filter2d/fft_utils.rs @@ -36,17 +36,17 @@ pub fn fft_next_good_size(mut n: usize) -> usize { let mut m = n; // Divide by 2 until it's no longer divisible - while m % 2 == 0 { + while m.is_multiple_of(2) { m /= 2; } // Divide by 3 until it's no longer divisible - while m % 3 == 0 { + while m.is_multiple_of(3) { m /= 3; } // Divide by 5 until it's no longer divisible - while m % 5 == 0 { + while m.is_multiple_of(5) { m /= 5; } @@ -69,27 +69,27 @@ pub(crate) fn fft_next_good_size_real(mut n: usize) -> usize { let mut m = n; // Divide by 2 until it's no longer divisible - while m % 2 == 0 { + while m.is_multiple_of(2) { m /= 2; } // Divide by 3 until it's no longer divisible - while m % 3 == 0 { + while m.is_multiple_of(3) { m /= 3; } // Divide by 5 until it's no longer divisible - while m % 5 == 0 { + while m.is_multiple_of(5) { m /= 5; } // Divide by 7 until it's no longer divisible - while m % 7 == 0 { + while m.is_multiple_of(7) { m /= 7; } // If m is reduced to 1, n is a good FFT size - if m <= 1 && n % 2 != 0 { + if m <= 1 && !n.is_multiple_of(2) { return n; } diff --git a/src/filter2d/filter_2d.rs b/src/filter2d/filter_2d.rs index 30b020ce..93479e76 100644 --- a/src/filter2d/filter_2d.rs +++ b/src/filter2d/filter_2d.rs @@ -26,7 +26,7 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -use crate::filter1d::{make_arena, ArenaPads, KernelShape}; +use crate::filter1d::{ArenaPads, KernelShape, make_arena}; use crate::filter2d::filter_2d_handler::Filter2dHandler; use crate::filter2d::scan_point_2d::ScanPoint2d; use crate::filter2d::scan_se_2d::scan_se_2d; diff --git a/src/filter2d/filter_2d_fft.rs b/src/filter2d/filter_2d_fft.rs index 3ea16163..93a385ed 100644 --- a/src/filter2d/filter_2d_fft.rs +++ b/src/filter2d/filter_2d_fft.rs @@ -27,7 +27,7 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::fast_divide::{DividerIsize, RemEuclidFast}; -use crate::filter1d::{make_arena, ArenaPads}; +use crate::filter1d::{ArenaPads, make_arena}; use crate::filter2d::fft_utils::fft_next_good_size; use crate::filter2d::scan_se_2d::scan_se_2d_complex; use crate::to_storage::ToStorage; diff --git a/src/filter2d/filter_2d_fft_real.rs b/src/filter2d/filter_2d_fft_real.rs index a3c89e20..e937d89e 100644 --- a/src/filter2d/filter_2d_fft_real.rs +++ b/src/filter2d/filter_2d_fft_real.rs @@ -28,7 +28,7 @@ */ use crate::fast_divide::{DividerIsize, RemEuclidFast}; -use crate::filter1d::{make_arena, ArenaPads}; +use crate::filter1d::{ArenaPads, make_arena}; use crate::filter2d::fft_utils::fft_next_good_size_real; use crate::filter2d::mul_spectrum::SpectrumMultiplier; use crate::to_storage::ToStorage; diff --git a/src/filter2d/filter_2d_handler.rs b/src/filter2d/filter_2d_handler.rs index 449bb66c..47a5e32c 100644 --- a/src/filter2d/filter_2d_handler.rs +++ b/src/filter2d/filter_2d_handler.rs @@ -26,6 +26,7 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; use crate::filter1d::Arena; use crate::filter2d::convolve_op::convolve_segment_2d; #[cfg(all(target_arch = "aarch64", feature = "neon"))] @@ -33,7 +34,6 @@ use crate::filter2d::neon::{convolve_segment_neon_2d_u8_f32, convolve_segment_ne use crate::filter2d::scan_point_2d::ScanPoint2d; #[cfg(all(any(target_arch = "x86_64", target_arch = "x86"), feature = "sse"))] use crate::filter2d::sse::{convolve_segment_sse_2d_u8_f32, convolve_segment_sse_2d_u8_i16}; -use crate::ImageSize; pub type Executor2DConvolution = fn( arena: Arena, diff --git a/src/filter2d/filter_2d_rgb.rs b/src/filter2d/filter_2d_rgb.rs index ec3a28e5..23a94c99 100644 --- a/src/filter2d/filter_2d_rgb.rs +++ b/src/filter2d/filter_2d_rgb.rs @@ -29,7 +29,7 @@ use crate::filter1d::KernelShape; use crate::filter2d::filter_2d_handler::Filter2dHandler; use crate::to_storage::ToStorage; -use crate::{filter_2d, BlurError, BlurImage, BlurImageMut, EdgeMode2D, Scalar, ThreadingPolicy}; +use crate::{BlurError, BlurImage, BlurImageMut, EdgeMode2D, Scalar, ThreadingPolicy, filter_2d}; use num_traits::{AsPrimitive, MulAdd}; use std::fmt::Debug; use std::ops::Mul; diff --git a/src/filter2d/filter_2d_rgb_fft.rs b/src/filter2d/filter_2d_rgb_fft.rs index 9aba92e4..38403f23 100644 --- a/src/filter2d/filter_2d_rgb_fft.rs +++ b/src/filter2d/filter_2d_rgb_fft.rs @@ -27,7 +27,7 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::filter2d::filter_2d_fft::filter_2d_fft_impl; -use crate::filter2d::filter_2d_fft_real::{filter_2d_fft_real_impl, FftNumber}; +use crate::filter2d::filter_2d_fft_real::{FftNumber, filter_2d_fft_real_impl}; use crate::filter2d::gather_channel::{gather_channel, squash_channel}; use crate::filter2d::scan_se_2d::{scan_se_2d, scan_se_2d_complex}; use crate::to_storage::ToStorage; diff --git a/src/filter2d/filter_2d_rgba.rs b/src/filter2d/filter_2d_rgba.rs index 26ba0fb9..f84c3c2c 100644 --- a/src/filter2d/filter_2d_rgba.rs +++ b/src/filter2d/filter_2d_rgba.rs @@ -29,7 +29,7 @@ use crate::filter1d::KernelShape; use crate::filter2d::filter_2d_handler::Filter2dHandler; use crate::to_storage::ToStorage; -use crate::{filter_2d, BlurError, BlurImage, BlurImageMut, EdgeMode2D, Scalar, ThreadingPolicy}; +use crate::{BlurError, BlurImage, BlurImageMut, EdgeMode2D, Scalar, ThreadingPolicy, filter_2d}; use num_traits::{AsPrimitive, MulAdd}; use std::fmt::Debug; use std::ops::Mul; diff --git a/src/filter2d/neon/convolve_op_u8_f32.rs b/src/filter2d/neon/convolve_op_u8_f32.rs index 3cef8825..4d898e4a 100644 --- a/src/filter2d/neon/convolve_op_u8_f32.rs +++ b/src/filter2d/neon/convolve_op_u8_f32.rs @@ -26,15 +26,15 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; +use crate::filter1d::Arena; use crate::filter1d::neon::utils::{ vfmlaq_u8_f32, vmulq_u8_by_f32, vqmovnq_f32_u8, xvld1q_u8_x2, xvld1q_u8_x4, xvst1q_u8_x2, xvst1q_u8_x4, }; -use crate::filter1d::Arena; use crate::filter2d::scan_point_2d::ScanPoint2d; use crate::mlaf::mlaf; use crate::to_storage::ToStorage; -use crate::ImageSize; use std::arch::aarch64::*; use std::ops::Mul; diff --git a/src/filter2d/neon/convolve_op_u8_i16.rs b/src/filter2d/neon/convolve_op_u8_i16.rs index 512acbc5..1883016e 100644 --- a/src/filter2d/neon/convolve_op_u8_i16.rs +++ b/src/filter2d/neon/convolve_op_u8_i16.rs @@ -26,13 +26,13 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; +use crate::filter1d::Arena; use crate::filter1d::neon::utils::{ vdotq_exact_s16, vmulq_u8_by_i16, xvld1q_u8_x2, xvld1q_u8_x4, xvst1q_u8_x2, xvst1q_u8_x4, }; -use crate::filter1d::Arena; use crate::filter2d::scan_point_2d::ScanPoint2d; use crate::to_storage::ToStorage; -use crate::ImageSize; use num_traits::MulAdd; use std::arch::aarch64::*; use std::ops::Mul; diff --git a/src/filter2d/neon/convolve_op_u8_i16_fp.rs b/src/filter2d/neon/convolve_op_u8_i16_fp.rs index e5706fd9..9137c94a 100644 --- a/src/filter2d/neon/convolve_op_u8_i16_fp.rs +++ b/src/filter2d/neon/convolve_op_u8_i16_fp.rs @@ -26,6 +26,7 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; use crate::filter1d::neon::utils::{ vfmlaq_u8_s16, vmullq_u8_by_i16, vqmovnq_s32_u8, xvld1q_u8_x2, xvld1q_u8_x4, xvst1q_u8_x2, xvst1q_u8_x4, @@ -33,7 +34,6 @@ use crate::filter1d::neon::utils::{ use crate::filter1d::{Arena, ToApproxStorage}; use crate::filter2d::scan_point_2d::ScanPoint2d; use crate::mlaf::mlaf; -use crate::ImageSize; use std::arch::aarch64::*; use std::ops::Mul; diff --git a/src/filter2d/sse/convolve_op_u8_f32.rs b/src/filter2d/sse/convolve_op_u8_f32.rs index 1233248d..32b6f0dd 100644 --- a/src/filter2d/sse/convolve_op_u8_f32.rs +++ b/src/filter2d/sse/convolve_op_u8_f32.rs @@ -26,15 +26,15 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +use crate::ImageSize; +use crate::filter1d::Arena; use crate::filter1d::sse::utils::{ _mm_mul_add_epi8_by_ps_x4, _mm_mul_epi8_by_ps_x4, _mm_pack_ps_x4_epi8, }; -use crate::filter1d::Arena; use crate::filter2d::scan_point_2d::ScanPoint2d; use crate::mlaf::mlaf; use crate::sse::{_mm_load_pack_x2, _mm_load_pack_x4, _mm_store_pack_x2, _mm_store_pack_x4}; use crate::to_storage::ToStorage; -use crate::ImageSize; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] @@ -55,7 +55,7 @@ pub(crate) fn convolve_segment_sse_2d_u8_f32( } #[target_feature(enable = "sse4.1")] -unsafe fn convolve_segment_2d_u8_f32_impl( +fn convolve_segment_2d_u8_f32_impl( arena: Arena, arena_source: &[u8], dst: &mut [u8], @@ -63,142 +63,144 @@ unsafe fn convolve_segment_2d_u8_f32_impl( prepared_kernel: &[ScanPoint2d], y: usize, ) { - let width = image_size.width; - - let dx = arena.pad_w as i64; - let dy = arena.pad_h as i64; - - let arena_stride = arena.width * arena.components; - - let offsets = prepared_kernel - .iter() - .map(|&x| { - arena_source.get_unchecked( - ((x.y + dy + y as i64) as usize * arena_stride - + (x.x + dx) as usize * arena.components).., - ) - }) - .collect::>(); - - let length = prepared_kernel.len(); - - let total_width = width * arena.components; - - let mut cx = 0usize; - - let off0 = offsets.get_unchecked(0); - - let k_weight = _mm_set1_ps(prepared_kernel.get_unchecked(0).weight); - - while cx + 64 < total_width { - let items0 = _mm_load_pack_x4(off0.get_unchecked(cx..).as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(items0.0, k_weight); - let mut k1 = _mm_mul_epi8_by_ps_x4(items0.1, k_weight); - let mut k2 = _mm_mul_epi8_by_ps_x4(items0.2, k_weight); - let mut k3 = _mm_mul_epi8_by_ps_x4(items0.3, k_weight); - for i in 1..length { - let weight = _mm_set1_ps(prepared_kernel.get_unchecked(i).weight); - let s_ptr = offsets.get_unchecked(i); - let items0 = _mm_load_pack_x4(s_ptr.get_unchecked(cx..).as_ptr()); - k0 = _mm_mul_add_epi8_by_ps_x4(k0, items0.0, weight); - k1 = _mm_mul_add_epi8_by_ps_x4(k1, items0.1, weight); - k2 = _mm_mul_add_epi8_by_ps_x4(k2, items0.2, weight); - k3 = _mm_mul_add_epi8_by_ps_x4(k3, items0.3, weight); + unsafe { + let width = image_size.width; + + let dx = arena.pad_w as i64; + let dy = arena.pad_h as i64; + + let arena_stride = arena.width * arena.components; + + let offsets = prepared_kernel + .iter() + .map(|&x| { + arena_source.get_unchecked( + ((x.y + dy + y as i64) as usize * arena_stride + + (x.x + dx) as usize * arena.components).., + ) + }) + .collect::>(); + + let length = prepared_kernel.len(); + + let total_width = width * arena.components; + + let mut cx = 0usize; + + let off0 = offsets.get_unchecked(0); + + let k_weight = _mm_set1_ps(prepared_kernel.get_unchecked(0).weight); + + while cx + 64 <= total_width { + let items0 = _mm_load_pack_x4(off0.get_unchecked(cx..).as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(items0.0, k_weight); + let mut k1 = _mm_mul_epi8_by_ps_x4(items0.1, k_weight); + let mut k2 = _mm_mul_epi8_by_ps_x4(items0.2, k_weight); + let mut k3 = _mm_mul_epi8_by_ps_x4(items0.3, k_weight); + for i in 1..length { + let weight = _mm_set1_ps(prepared_kernel.get_unchecked(i).weight); + let s_ptr = offsets.get_unchecked(i); + let items0 = _mm_load_pack_x4(s_ptr.get_unchecked(cx..).as_ptr()); + k0 = _mm_mul_add_epi8_by_ps_x4(k0, items0.0, weight); + k1 = _mm_mul_add_epi8_by_ps_x4(k1, items0.1, weight); + k2 = _mm_mul_add_epi8_by_ps_x4(k2, items0.2, weight); + k3 = _mm_mul_add_epi8_by_ps_x4(k3, items0.3, weight); + } + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x4( + dst_ptr0, + ( + _mm_pack_ps_x4_epi8(k0), + _mm_pack_ps_x4_epi8(k1), + _mm_pack_ps_x4_epi8(k2), + _mm_pack_ps_x4_epi8(k3), + ), + ); + cx += 64; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x4( - dst_ptr0, - ( - _mm_pack_ps_x4_epi8(k0), - _mm_pack_ps_x4_epi8(k1), - _mm_pack_ps_x4_epi8(k2), - _mm_pack_ps_x4_epi8(k3), - ), - ); - cx += 64; - } - while cx + 32 < total_width { - let items0 = _mm_load_pack_x2(off0.get_unchecked(cx..).as_ptr()); - let mut k0 = _mm_mul_epi8_by_ps_x4(items0.0, k_weight); - let mut k1 = _mm_mul_epi8_by_ps_x4(items0.1, k_weight); - for i in 1..length { - let weight = _mm_set1_ps(prepared_kernel.get_unchecked(i).weight); - let s_ptr = offsets.get_unchecked(i); - let items0 = _mm_load_pack_x2(s_ptr.get_unchecked(cx..).as_ptr()); - k0 = _mm_mul_add_epi8_by_ps_x4(k0, items0.0, weight); - k1 = _mm_mul_add_epi8_by_ps_x4(k1, items0.1, weight); + while cx + 32 <= total_width { + let items0 = _mm_load_pack_x2(off0.get_unchecked(cx..).as_ptr()); + let mut k0 = _mm_mul_epi8_by_ps_x4(items0.0, k_weight); + let mut k1 = _mm_mul_epi8_by_ps_x4(items0.1, k_weight); + for i in 1..length { + let weight = _mm_set1_ps(prepared_kernel.get_unchecked(i).weight); + let s_ptr = offsets.get_unchecked(i); + let items0 = _mm_load_pack_x2(s_ptr.get_unchecked(cx..).as_ptr()); + k0 = _mm_mul_add_epi8_by_ps_x4(k0, items0.0, weight); + k1 = _mm_mul_add_epi8_by_ps_x4(k1, items0.1, weight); + } + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_store_pack_x2(dst_ptr0, (_mm_pack_ps_x4_epi8(k0), _mm_pack_ps_x4_epi8(k1))); + cx += 32; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_store_pack_x2(dst_ptr0, (_mm_pack_ps_x4_epi8(k0), _mm_pack_ps_x4_epi8(k1))); - cx += 32; - } - while cx + 16 < total_width { - let items0 = _mm_loadu_si128(off0.get_unchecked(cx..).as_ptr() as *const __m128i); - let mut k0 = _mm_mul_epi8_by_ps_x4(items0, k_weight); - for i in 1..length { - let weight = _mm_set1_ps(prepared_kernel.get_unchecked(i).weight); - let items0 = _mm_loadu_si128( - offsets.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i - ); - k0 = _mm_mul_add_epi8_by_ps_x4(k0, items0, weight); + while cx + 16 <= total_width { + let items0 = _mm_loadu_si128(off0.get_unchecked(cx..).as_ptr() as *const __m128i); + let mut k0 = _mm_mul_epi8_by_ps_x4(items0, k_weight); + for i in 1..length { + let weight = _mm_set1_ps(prepared_kernel.get_unchecked(i).weight); + let items0 = _mm_loadu_si128( + offsets.get_unchecked(i).get_unchecked(cx..).as_ptr() as *const __m128i + ); + k0 = _mm_mul_add_epi8_by_ps_x4(k0, items0, weight); + } + let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); + _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); + cx += 16; } - let dst_ptr0 = dst.get_unchecked_mut(cx..).as_mut_ptr(); - _mm_storeu_si128(dst_ptr0 as *mut __m128i, _mm_pack_ps_x4_epi8(k0)); - cx += 16; - } - let k_weight = prepared_kernel.get_unchecked(0).weight; - - while cx + 4 < total_width { - let mut k0 = ((*off0.get_unchecked(cx)) as f32).mul(k_weight); - let mut k1 = ((*off0.get_unchecked(cx + 1)) as f32).mul(k_weight); - let mut k2 = ((*off0.get_unchecked(cx + 2)) as f32).mul(k_weight); - let mut k3 = ((*off0.get_unchecked(cx + 3)) as f32).mul(k_weight); - - for i in 1..length { - let weight = prepared_kernel.get_unchecked(i).weight; - k0 = mlaf( - k0, - (*offsets.get_unchecked(i).get_unchecked(cx)) as f32, - weight, - ); - k1 = mlaf( - k1, - (*offsets.get_unchecked(i).get_unchecked(cx + 1)) as f32, - weight, - ); - k2 = mlaf( - k2, - (*offsets.get_unchecked(i).get_unchecked(cx + 2)) as f32, - weight, - ); - k3 = mlaf( - k3, - (*offsets.get_unchecked(i).get_unchecked(cx + 3)) as f32, - weight, - ); + let k_weight = prepared_kernel.get_unchecked(0).weight; + + while cx + 4 <= total_width { + let mut k0 = ((*off0.get_unchecked(cx)) as f32).mul(k_weight); + let mut k1 = ((*off0.get_unchecked(cx + 1)) as f32).mul(k_weight); + let mut k2 = ((*off0.get_unchecked(cx + 2)) as f32).mul(k_weight); + let mut k3 = ((*off0.get_unchecked(cx + 3)) as f32).mul(k_weight); + + for i in 1..length { + let weight = prepared_kernel.get_unchecked(i).weight; + k0 = mlaf( + k0, + (*offsets.get_unchecked(i).get_unchecked(cx)) as f32, + weight, + ); + k1 = mlaf( + k1, + (*offsets.get_unchecked(i).get_unchecked(cx + 1)) as f32, + weight, + ); + k2 = mlaf( + k2, + (*offsets.get_unchecked(i).get_unchecked(cx + 2)) as f32, + weight, + ); + k3 = mlaf( + k3, + (*offsets.get_unchecked(i).get_unchecked(cx + 3)) as f32, + weight, + ); + } + + *dst.get_unchecked_mut(cx) = k0.to_(); + *dst.get_unchecked_mut(cx + 1) = k1.to_(); + *dst.get_unchecked_mut(cx + 2) = k2.to_(); + *dst.get_unchecked_mut(cx + 3) = k3.to_(); + cx += 4; } - *dst.get_unchecked_mut(cx) = k0.to_(); - *dst.get_unchecked_mut(cx + 1) = k1.to_(); - *dst.get_unchecked_mut(cx + 2) = k2.to_(); - *dst.get_unchecked_mut(cx + 3) = k3.to_(); - cx += 4; - } - - for x in cx..total_width { - let mut k0 = ((*(*off0).get_unchecked(x)) as f32).mul(k_weight); - - for i in 1..length { - let k_weight = prepared_kernel.get_unchecked(i).weight; - k0 = mlaf( - k0, - (*offsets.get_unchecked(i).get_unchecked(x)) as f32, - k_weight, - ); + for x in cx..total_width { + let mut k0 = ((*(*off0).get_unchecked(x)) as f32).mul(k_weight); + + for i in 1..length { + let k_weight = prepared_kernel.get_unchecked(i).weight; + k0 = mlaf( + k0, + (*offsets.get_unchecked(i).get_unchecked(x)) as f32, + k_weight, + ); + } + *dst.get_unchecked_mut(x) = k0.to_(); } - *dst.get_unchecked_mut(x) = k0.to_(); } } diff --git a/src/filter2d/sse/convolve_op_u8_i16.rs b/src/filter2d/sse/convolve_op_u8_i16.rs index c08ff2f6..a369b031 100644 --- a/src/filter2d/sse/convolve_op_u8_i16.rs +++ b/src/filter2d/sse/convolve_op_u8_i16.rs @@ -26,12 +26,12 @@ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -use crate::filter1d::sse::utils::{_mm_mull_add_epi8_by_epi16_x4, _mm_mull_epi8_by_epi16_x4}; +use crate::ImageSize; use crate::filter1d::Arena; +use crate::filter1d::sse::utils::{_mm_mull_add_epi8_by_epi16_x4, _mm_mull_epi8_by_epi16_x4}; use crate::filter2d::scan_point_2d::ScanPoint2d; use crate::sse::{_mm_load_pack_x2, _mm_load_pack_x4, _mm_store_pack_x2, _mm_store_pack_x4}; use crate::to_storage::ToStorage; -use crate::ImageSize; use num_traits::MulAdd; #[cfg(target_arch = "x86")] use std::arch::x86::*; diff --git a/src/gaussian/declaration.rs b/src/gaussian/declaration.rs index 0fcb9807..37a5bc8f 100644 --- a/src/gaussian/declaration.rs +++ b/src/gaussian/declaration.rs @@ -30,8 +30,8 @@ use crate::gaussian::gaussian_hint::IeeeBinaryConvolutionMode; use crate::gaussian::gaussian_kernel::gaussian_kernel_1d; use crate::gaussian::gaussian_util::{kernel_size as get_kernel_size, kernel_size_d}; use crate::{ - filter_1d_approx, filter_1d_exact, gaussian_kernel_1d_f64, sigma_size, sigma_size_d, BlurError, - BlurImage, BlurImageMut, ConvolutionMode, EdgeMode2D, Scalar, ThreadingPolicy, + BlurError, BlurImage, BlurImageMut, ConvolutionMode, EdgeMode2D, Scalar, ThreadingPolicy, + filter_1d_approx, filter_1d_exact, gaussian_kernel_1d_f64, sigma_size, sigma_size_d, }; #[cfg(feature = "nightly_f16")] use core::f16; @@ -194,10 +194,10 @@ impl GaussianBlurParams { if self.x_sigma < 0. || self.y_sigma < 0. { return Err(BlurError::NegativeOrZeroSigma); } - if self.x_kernel > 0 && self.x_kernel % 2 == 0 { + if self.x_kernel > 0 && self.x_kernel.is_multiple_of(2) { return Err(BlurError::OddKernel(self.x_kernel as usize)); } - if self.y_kernel > 0 && self.y_kernel % 2 == 0 { + if self.y_kernel > 0 && self.y_kernel.is_multiple_of(2) { return Err(BlurError::OddKernel(self.y_kernel as usize)); } if self.x_sigma == 0. && self.x_kernel == 0 { diff --git a/src/gaussian/gaussian_util.rs b/src/gaussian/gaussian_util.rs index ce14b51b..08acfc05 100644 --- a/src/gaussian/gaussian_util.rs +++ b/src/gaussian/gaussian_util.rs @@ -42,7 +42,7 @@ pub fn sigma_size_d(kernel_size: f64) -> f64 { /// Computes kernel size from sigma pub fn kernel_size(sigma: f32) -> u32 { let possible_size = (((((sigma - 0.8) / 0.3) + 1.) * 2.) + 1.).max(3.) as u32; - if possible_size % 2 == 0 { + if possible_size.is_multiple_of(2) { return possible_size + 1; } possible_size @@ -51,7 +51,7 @@ pub fn kernel_size(sigma: f32) -> u32 { /// Computes kernel size from sigma pub fn kernel_size_d(sigma: f64) -> u32 { let possible_size = (((((sigma - 0.8) / 0.3) + 1.) * 2.) + 1.).max(3.) as u32; - if possible_size % 2 == 0 { + if possible_size.is_multiple_of(2) { return possible_size + 1; } possible_size diff --git a/src/gaussian/mod.rs b/src/gaussian/mod.rs index 55c8fab9..243da872 100644 --- a/src/gaussian/mod.rs +++ b/src/gaussian/mod.rs @@ -32,7 +32,7 @@ mod gaussian_util; #[cfg(feature = "nightly_f16")] pub use declaration::gaussian_blur_f16; -pub use declaration::{gaussian_blur, gaussian_blur_f32, gaussian_blur_u16, GaussianBlurParams}; +pub use declaration::{GaussianBlurParams, gaussian_blur, gaussian_blur_f32, gaussian_blur_u16}; pub use gaussian_hint::{ConvolutionMode, IeeeBinaryConvolutionMode}; pub use gaussian_kernel::{complex_gaussian_kernel, gaussian_kernel_1d, gaussian_kernel_1d_f64}; pub use gaussian_util::{sigma_size, sigma_size_d}; diff --git a/src/gaussian_blur_image.rs b/src/gaussian_blur_image.rs index 1bdea535..b651cf2e 100644 --- a/src/gaussian_blur_image.rs +++ b/src/gaussian_blur_image.rs @@ -27,8 +27,9 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::{ - gaussian_blur, gaussian_blur_f32, gaussian_blur_u16, BlurImage, BlurImageMut, ConvolutionMode, - EdgeMode2D, FastBlurChannels, GaussianBlurParams, IeeeBinaryConvolutionMode, ThreadingPolicy, + BlurImage, BlurImageMut, ConvolutionMode, EdgeMode2D, FastBlurChannels, GaussianBlurParams, + IeeeBinaryConvolutionMode, ThreadingPolicy, gaussian_blur, gaussian_blur_f32, + gaussian_blur_u16, }; use image::{ DynamicImage, GrayAlphaImage, GrayImage, ImageBuffer, Luma, LumaA, Rgb, Rgb32FImage, RgbImage, diff --git a/src/image.rs b/src/image.rs index 008c1a81..a8b21f4a 100644 --- a/src/image.rs +++ b/src/image.rs @@ -314,11 +314,11 @@ impl<'a, T: Clone + Copy + Default + Debug> BlurImageMut<'a, T> { /// Checks if layout matches necessary requirements #[inline] pub fn check_layout(&mut self, other: Option<&BlurImage<'_, T>>) -> Result<(), BlurError> { - if let Some(other) = other { - if matches!(self.data, BufferStore::Owned(_)) { - self.resize(other.width, other.height, other.channels); - return Ok(()); - } + if let Some(other) = other + && matches!(self.data, BufferStore::Owned(_)) + { + self.resize(other.width, other.height, other.channels); + return Ok(()); } if self.width == 0 || self.height == 0 { return Err(BlurError::ZeroBaseSize); @@ -347,11 +347,11 @@ impl<'a, T: Clone + Copy + Default + Debug> BlurImageMut<'a, T> { cn: usize, other: Option<&BlurImage<'_, T>>, ) -> Result<(), BlurError> { - if let Some(other) = other { - if matches!(self.data, BufferStore::Owned(_)) { - self.resize_arbitrary(other.width, other.height, cn); - return Ok(()); - } + if let Some(other) = other + && matches!(self.data, BufferStore::Owned(_)) + { + self.resize_arbitrary(other.width, other.height, cn); + return Ok(()); } if self.width == 0 || self.height == 0 { return Err(BlurError::ZeroBaseSize); diff --git a/src/laplacian.rs b/src/laplacian.rs index 7e298d3a..4f05bbb9 100644 --- a/src/laplacian.rs +++ b/src/laplacian.rs @@ -28,8 +28,8 @@ */ use crate::{ - filter_2d, sigma_size, BlurError, BlurImage, BlurImageMut, EdgeMode2D, KernelShape, Scalar, - ThreadingPolicy, + BlurError, BlurImage, BlurImageMut, EdgeMode2D, KernelShape, Scalar, ThreadingPolicy, + filter_2d, sigma_size, }; pub fn laplacian_kernel(size: usize) -> Vec { diff --git a/src/lens.rs b/src/lens.rs index 122915f7..b3657d21 100644 --- a/src/lens.rs +++ b/src/lens.rs @@ -46,10 +46,10 @@ pub fn lens_kernel( if shape.width == 0 || shape.height == 0 { return Err(BlurError::ZeroBaseSize); } - if shape.width % 2 == 0 { + if shape.width.is_multiple_of(2) { return Err(BlurError::OddKernel(shape.width)); } - if shape.height % 2 == 0 { + if shape.height.is_multiple_of(2) { return Err(BlurError::OddKernel(shape.height)); } assert!(k.abs() <= 1., "Roundness contract is not satisfied"); diff --git a/src/lib.rs b/src/lib.rs index 2b1958a3..e178dd3c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,31 +37,19 @@ feature(stdarch_neon_fcma) )] #![cfg_attr( - all( - feature = "nightly_avx512", - any(target_arch = "x86", target_arch = "x86_64") - ), + all(feature = "avx512", any(target_arch = "x86", target_arch = "x86_64")), feature(cfg_version) )] #![cfg_attr( - all( - feature = "nightly_avx512", - any(target_arch = "x86", target_arch = "x86_64") - ), + all(feature = "avx512", any(target_arch = "x86", target_arch = "x86_64")), feature(avx512_target_feature) )] #![cfg_attr( - all( - feature = "nightly_avx512", - any(target_arch = "x86", target_arch = "x86_64") - ), + all(feature = "avx512", any(target_arch = "x86", target_arch = "x86_64")), feature(stdarch_x86_avx512) )] #![cfg_attr( - all( - feature = "nightly_avx512", - any(target_arch = "x86", target_arch = "x86_64") - ), + all(feature = "avx512", any(target_arch = "x86", target_arch = "x86_64")), feature(x86_amx_intrinsics) )] #![cfg_attr(feature = "nightly_f16", feature(f16))] @@ -129,11 +117,10 @@ mod wasm32; #[cfg(feature = "fft")] #[cfg_attr(docsrs, doc(cfg(feature = "fft")))] pub use adaptive_blur::adaptive_blur; -pub use bilateral::{bilateral_filter, BilateralBlurParams}; +pub use bilateral::{BilateralBlurParams, bilateral_filter}; pub use box_filter::{ - box_blur, box_blur_f32, box_blur_u16, gaussian_box_blur, gaussian_box_blur_f32, - gaussian_box_blur_u16, tent_blur, tent_blur_f32, tent_blur_u16, BoxBlurParameters, - CLTParameters, + BoxBlurParameters, CLTParameters, box_blur, box_blur_f32, box_blur_u16, gaussian_box_blur, + gaussian_box_blur_f32, gaussian_box_blur_u16, tent_blur, tent_blur_f32, tent_blur_u16, }; pub use channels_configuration::FastBlurChannels; pub use edge_mode::{BorderHandle, EdgeMode, EdgeMode2D, Scalar}; @@ -156,23 +143,23 @@ pub use fast_gaussian_image_next::fast_gaussian_next_blur_image; pub use fast_gaussian_next::fast_gaussian_next_f16; pub use fast_gaussian_next::{fast_gaussian_next, fast_gaussian_next_f32, fast_gaussian_next_u16}; pub use filter1d::{ - filter_1d_approx, filter_1d_complex, filter_1d_complex_fixed_point, filter_1d_exact, - KernelShape, + KernelShape, filter_1d_approx, filter_1d_complex, filter_1d_complex_fixed_point, + filter_1d_exact, }; #[cfg(feature = "fft")] #[cfg_attr(docsrs, doc(cfg(feature = "fft")))] pub use filter2d::{ - fft_next_good_size, filter_2d_fft, filter_2d_fft_complex, filter_2d_rgb_fft, - filter_2d_rgb_fft_complex, filter_2d_rgba_fft, filter_2d_rgba_fft_complex, FftNumber, + FftNumber, fft_next_good_size, filter_2d_fft, filter_2d_fft_complex, filter_2d_rgb_fft, + filter_2d_rgb_fft_complex, filter_2d_rgba_fft, filter_2d_rgba_fft_complex, }; pub use filter2d::{filter_2d, filter_2d_arbitrary, filter_2d_rgb, filter_2d_rgba}; pub use gamma_curves::TransferFunction; #[cfg(feature = "nightly_f16")] pub use gaussian::gaussian_blur_f16; pub use gaussian::{ - complex_gaussian_kernel, gaussian_blur, gaussian_blur_f32, gaussian_blur_u16, - gaussian_kernel_1d, gaussian_kernel_1d_f64, sigma_size, sigma_size_d, ConvolutionMode, - GaussianBlurParams, IeeeBinaryConvolutionMode, + ConvolutionMode, GaussianBlurParams, IeeeBinaryConvolutionMode, complex_gaussian_kernel, + gaussian_blur, gaussian_blur_f32, gaussian_blur_u16, gaussian_kernel_1d, + gaussian_kernel_1d_f64, sigma_size, sigma_size_d, }; #[cfg(feature = "image")] #[cfg_attr(docsrs, doc(cfg(feature = "image")))] diff --git a/src/motion_blur.rs b/src/motion_blur.rs index f3bc942a..22acb360 100644 --- a/src/motion_blur.rs +++ b/src/motion_blur.rs @@ -27,7 +27,7 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::{ - filter_2d, BlurError, BlurImage, BlurImageMut, EdgeMode2D, KernelShape, Scalar, ThreadingPolicy, + BlurError, BlurImage, BlurImageMut, EdgeMode2D, KernelShape, Scalar, ThreadingPolicy, filter_2d, }; #[derive(Copy, Clone)] diff --git a/src/neon/f16_utils.rs b/src/neon/f16_utils.rs index 57ed686f..6d36a764 100644 --- a/src/neon/f16_utils.rs +++ b/src/neon/f16_utils.rs @@ -45,18 +45,20 @@ use core::f16; #[inline] pub unsafe fn xvld_f16(ptr: *const f16) -> x_float16x4_t { - let store: uint16x4_t = vld1_u16(std::mem::transmute::<*const f16, *const u16>(ptr)); - std::mem::transmute(store) + unsafe { + let store: uint16x4_t = vld1_u16(std::mem::transmute::<*const f16, *const u16>(ptr)); + std::mem::transmute(store) + } } #[inline] -pub unsafe fn xreinterpret_u16_f16(x: x_float16x4_t) -> uint16x4_t { - std::mem::transmute(x) +pub fn xreinterpret_u16_f16(x: x_float16x4_t) -> uint16x4_t { + unsafe { std::mem::transmute(x) } } #[inline] -pub unsafe fn xreinterpret_f16_u16(x: uint16x4_t) -> x_float16x4_t { - std::mem::transmute(x) +pub fn xreinterpret_f16_u16(x: uint16x4_t) -> x_float16x4_t { + unsafe { std::mem::transmute(x) } } // #[inline] @@ -66,31 +68,37 @@ pub unsafe fn xreinterpret_f16_u16(x: uint16x4_t) -> x_float16x4_t { #[inline] pub unsafe fn xvcvt_f32_f16(x: x_float16x4_t) -> float32x4_t { - let src: uint16x4_t = xreinterpret_u16_f16(x); - let dst: float32x4_t; - asm!( - "fcvtl {0:v}.4s, {1:v}.4h", - out(vreg) dst, - in(vreg) src, - options(pure, nomem, nostack)); - dst + unsafe { + let src: uint16x4_t = xreinterpret_u16_f16(x); + let dst: float32x4_t; + asm!( + "fcvtl {0:v}.4s, {1:v}.4h", + out(vreg) dst, + in(vreg) src, + options(pure, nomem, nostack)); + dst + } } #[inline] pub(super) unsafe fn xvcvt_f16_f32(v: float32x4_t) -> x_float16x4_t { - let result: uint16x4_t; - asm!( - "fcvtn {0:v}.4h, {1:v}.4s", - out(vreg) result, - in(vreg) v, - options(pure, nomem, nostack)); - xreinterpret_f16_u16(result) + unsafe { + let result: uint16x4_t; + asm!( + "fcvtn {0:v}.4h, {1:v}.4s", + out(vreg) result, + in(vreg) v, + options(pure, nomem, nostack)); + xreinterpret_f16_u16(result) + } } #[inline] pub unsafe fn xvst_f16(ptr: *mut f16, x: x_float16x4_t) { - vst1_u16( - std::mem::transmute::<*mut f16, *mut u16>(ptr), - xreinterpret_u16_f16(x), - ) + unsafe { + vst1_u16( + std::mem::transmute::<*mut f16, *mut u16>(ptr), + xreinterpret_u16_f16(x), + ) + } } diff --git a/src/neon/fast_gaussian.rs b/src/neon/fast_gaussian.rs index d497e54e..96321434 100644 --- a/src/neon/fast_gaussian.rs +++ b/src/neon/fast_gaussian.rs @@ -25,18 +25,18 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::{load_u8_s32_fast, store_u8_s32_x4, store_u8x8_m4, vmulq_s32_f32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::aarch64::*; #[repr(C, align(16))] #[derive(Copy, Clone, Default)] pub(crate) struct NeonI32x4(pub(crate) [i32; 4]); -pub(crate) fn fg_horizontal_pass_neon_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_horizontal_pass_neon_u8( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -46,9 +46,7 @@ pub(crate) fn fg_horizontal_pass_neon_u8( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 4]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -62,7 +60,7 @@ pub(crate) fn fg_horizontal_pass_neon_u8( let mut yy = start; - while yy + 4 < height.min(end) { + while yy + 4 <= height.min(end) { let mut diffs0 = vdupq_n_s32(0); let mut diffs1 = vdupq_n_s32(0); let mut diffs2 = vdupq_n_s32(0); @@ -263,8 +261,8 @@ pub(crate) fn fg_horizontal_pass_neon_u8( } } -pub(crate) fn fg_vertical_pass_neon_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_vertical_pass_neon_u8( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -274,9 +272,7 @@ pub(crate) fn fg_vertical_pass_neon_u8( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 4]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -291,7 +287,7 @@ pub(crate) fn fg_vertical_pass_neon_u8( let mut xx = start; - while xx + 4 < width.min(end) { + while xx + 4 <= width.min(end) { let mut diffs0 = vdupq_n_s32(0); let mut diffs1 = vdupq_n_s32(0); let mut diffs2 = vdupq_n_s32(0); diff --git a/src/neon/fast_gaussian_f16.rs b/src/neon/fast_gaussian_f16.rs index 926e3750..1d5f8b1e 100644 --- a/src/neon/fast_gaussian_f16.rs +++ b/src/neon/fast_gaussian_f16.rs @@ -25,15 +25,15 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::{load_f32_f16, store_f32_f16}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use core::f16; use std::arch::aarch64::*; -pub(crate) fn fg_vertical_pass_neon_f16( - undef_bytes: &UnsafeSlice, +pub(crate) fn fg_vertical_pass_neon_f16( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -43,8 +43,7 @@ pub(crate) fn fg_vertical_pass_neon_f16( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f16> = std::mem::transmute(undef_bytes); - let mut buffer = Box::new([[0f32; 4]; 1024]); + let mut buffer = [[0f32; 4]; 1024]; let height_wide = height as i64; @@ -104,8 +103,8 @@ pub(crate) fn fg_vertical_pass_neon_f16( } } -pub(crate) fn fg_horizontal_pass_neon_f16( - undef_bytes: &UnsafeSlice, +pub(crate) fn fg_horizontal_pass_neon_f16( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -115,7 +114,6 @@ pub(crate) fn fg_horizontal_pass_neon_f16( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f16> = std::mem::transmute(undef_bytes); let mut buffer: [[f32; 4]; 1024] = [[0f32; 4]; 1024]; let radius_64 = radius as i64; let width_wide = width as i64; diff --git a/src/neon/fast_gaussian_f32.rs b/src/neon/fast_gaussian_f32.rs index 3ebb40e8..24a508c4 100644 --- a/src/neon/fast_gaussian_f32.rs +++ b/src/neon/fast_gaussian_f32.rs @@ -27,14 +27,14 @@ use std::arch::aarch64::*; +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::fast_gaussian_next_f32::NeonF32x4; use crate::neon::{load_f32_fast, store_f32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; -pub(crate) fn fg_vertical_pass_neon_f32( - undef_bytes: &UnsafeSlice, +pub(crate) fn fg_vertical_pass_neon_f32( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -44,9 +44,7 @@ pub(crate) fn fg_vertical_pass_neon_f32( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undef_bytes); - - let mut full_buffer = Box::new([NeonF32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonF32x4::default(); 1024 * 4]; let (bf0, rem) = full_buffer.split_at_mut(1024); let (bf1, rem) = rem.split_at_mut(1024); @@ -210,8 +208,8 @@ pub(crate) fn fg_vertical_pass_neon_f32( } } -pub(crate) fn fg_horizontal_pass_neon_f32( - undef_bytes: &UnsafeSlice, +pub(crate) fn fg_horizontal_pass_neon_f32( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -221,9 +219,7 @@ pub(crate) fn fg_horizontal_pass_neon_f32( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undef_bytes); - - let mut full_buffer = Box::new([NeonF32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonF32x4::default(); 1024 * 4]; let (bf0, rem) = full_buffer.split_at_mut(1024); let (bf1, rem) = rem.split_at_mut(1024); diff --git a/src/neon/fast_gaussian_next.rs b/src/neon/fast_gaussian_next.rs index d081faa4..43aa105a 100644 --- a/src/neon/fast_gaussian_next.rs +++ b/src/neon/fast_gaussian_next.rs @@ -25,15 +25,15 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::fast_gaussian::NeonI32x4; use crate::neon::{load_u8_s32_fast, store_u8_s32_x4, store_u8x8_m4, vmulq_by_3_s32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::aarch64::*; -pub(crate) fn fgn_vertical_pass_neon_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fgn_vertical_pass_neon_u8( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -43,9 +43,7 @@ pub(crate) fn fgn_vertical_pass_neon_u8( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 4]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -322,8 +320,8 @@ pub(crate) fn fgn_vertical_pass_neon_u8( } } -pub(crate) fn fgn_horizontal_pass_neon_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fgn_horizontal_pass_neon_u8( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -333,9 +331,7 @@ pub(crate) fn fgn_horizontal_pass_neon_u8( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 4]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); diff --git a/src/neon/fast_gaussian_next_f16.rs b/src/neon/fast_gaussian_next_f16.rs index 68b02c83..16ba45cc 100644 --- a/src/neon/fast_gaussian_next_f16.rs +++ b/src/neon/fast_gaussian_next_f16.rs @@ -29,13 +29,13 @@ use std::arch::aarch64::*; use core::f16; +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::{load_f32_f16, store_f32_f16}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; -pub(crate) fn fgn_vertical_pass_neon_f16( - undef_bytes: &UnsafeSlice, +pub(crate) fn fgn_vertical_pass_neon_f16( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -45,7 +45,6 @@ pub(crate) fn fgn_vertical_pass_neon_f16( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f16> = std::mem::transmute(undef_bytes); let mut buffer: [[f32; 4]; 1024] = [[0f32; 4]; 1024]; let height_wide = height as i64; @@ -124,8 +123,8 @@ pub(crate) fn fgn_vertical_pass_neon_f16( } } -pub(crate) fn fgn_horizontal_pass_neon_f16( - undef_bytes: &UnsafeSlice, +pub(crate) fn fgn_horizontal_pass_neon_f16( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -136,7 +135,6 @@ pub(crate) fn fgn_horizontal_pass_neon_f16( ) { unsafe { let mut buffer: [[f32; 4]; 1024] = [[0f32; 4]; 1024]; - let bytes: &UnsafeSlice<'_, f16> = std::mem::transmute(undef_bytes); let width_wide = width as i64; diff --git a/src/neon/fast_gaussian_next_f32.rs b/src/neon/fast_gaussian_next_f32.rs index 925e011e..760030e5 100644 --- a/src/neon/fast_gaussian_next_f32.rs +++ b/src/neon/fast_gaussian_next_f32.rs @@ -27,17 +27,17 @@ use std::arch::aarch64::*; +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::utils::{load_f32_fast, store_f32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; #[repr(C, align(16))] #[derive(Copy, Clone, Default)] pub(crate) struct NeonF32x4(pub(crate) [f32; 4]); -pub(crate) fn fgn_vertical_pass_neon_f32( - undef_bytes: &UnsafeSlice, +pub(crate) fn fgn_vertical_pass_neon_f32( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -47,9 +47,7 @@ pub(crate) fn fgn_vertical_pass_neon_f32( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undef_bytes); - - let mut full_buffer = Box::new([NeonF32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonF32x4::default(); 1024 * 4]; let (bf0, rem) = full_buffer.split_at_mut(1024); let (bf1, rem) = rem.split_at_mut(1024); @@ -63,7 +61,7 @@ pub(crate) fn fgn_vertical_pass_neon_f32( let mut xx = start as usize; - while xx + 4 < width.min(end) as usize { + while xx + 4 <= width.min(end) as usize { let mut diffs0 = vdupq_n_f32(0f32); let mut diffs1 = vdupq_n_f32(0f32); let mut diffs2 = vdupq_n_f32(0f32); @@ -79,10 +77,10 @@ pub(crate) fn fgn_vertical_pass_neon_f32( let mut summs2 = vdupq_n_f32(0f32); let mut summs3 = vdupq_n_f32(0f32); - let current_px0 = xx * CHANNELS_COUNT; - let current_px1 = (xx + 1) * CHANNELS_COUNT; - let current_px2 = (xx + 2) * CHANNELS_COUNT; - let current_px3 = (xx + 3) * CHANNELS_COUNT; + let current_px0 = xx * CN; + let current_px1 = (xx + 1) * CN; + let current_px2 = (xx + 2) * CN; + let current_px3 = (xx + 3) * CN; let start_y = 0 - 3 * radius as i64; for y in start_y..height_wide { @@ -99,10 +97,10 @@ pub(crate) fn fgn_vertical_pass_neon_f32( let dst_ptr2 = bytes.slice.as_ptr().add(current_y + current_px2) as *mut f32; let dst_ptr3 = bytes.slice.as_ptr().add(current_y + current_px3) as *mut f32; - store_f32::(dst_ptr0, prepared_px0); - store_f32::(dst_ptr1, prepared_px1); - store_f32::(dst_ptr2, prepared_px2); - store_f32::(dst_ptr3, prepared_px3); + store_f32::(dst_ptr0, prepared_px0); + store_f32::(dst_ptr1, prepared_px1); + store_f32::(dst_ptr2, prepared_px2); + store_f32::(dst_ptr3, prepared_px3); let d_a_1 = ((y + radius_64) & 1023) as usize; let d_a_2 = ((y - radius_64) & 1023) as usize; @@ -175,10 +173,10 @@ pub(crate) fn fgn_vertical_pass_neon_f32( let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut f32; let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut f32; - let pixel_color0 = load_f32_fast::(s_ptr0); - let pixel_color1 = load_f32_fast::(s_ptr1); - let pixel_color2 = load_f32_fast::(s_ptr2); - let pixel_color3 = load_f32_fast::(s_ptr3); + let pixel_color0 = load_f32_fast::(s_ptr0); + let pixel_color1 = load_f32_fast::(s_ptr1); + let pixel_color2 = load_f32_fast::(s_ptr2); + let pixel_color3 = load_f32_fast::(s_ptr3); let a_i = ((y + 2 * radius_64) & 1023) as usize; @@ -215,11 +213,11 @@ pub(crate) fn fgn_vertical_pass_neon_f32( for y in start_y..height_wide { if y >= 0 { let current_y = (y * (stride as i64)) as usize; - let current_px = x * CHANNELS_COUNT; + let current_px = x * CN; let prepared_px = vmulq_f32(summs, f_weight); let dst_ptr = bytes.slice.as_ptr().add(current_y + current_px) as *mut f32; - store_f32::(dst_ptr, prepared_px); + store_f32::(dst_ptr, prepared_px); let d_arr_index_1 = ((y + radius_64) & 1023) as usize; let d_arr_index_2 = ((y - radius_64) & 1023) as usize; @@ -256,11 +254,11 @@ pub(crate) fn fgn_vertical_pass_neon_f32( let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) * (stride as usize); - let next_row_x = x * CHANNELS_COUNT; + let next_row_x = x * CN; let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f32; - let pixel_color = load_f32_fast::(s_ptr); + let pixel_color = load_f32_fast::(s_ptr); let arr_index = ((y + 2 * radius_64) & 1023) as usize; let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; @@ -274,8 +272,8 @@ pub(crate) fn fgn_vertical_pass_neon_f32( } } -pub(crate) fn fgn_horizontal_pass_neon_f32( - undef_bytes: &UnsafeSlice, +pub(crate) fn fgn_horizontal_pass_neon_f32( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -285,14 +283,12 @@ pub(crate) fn fgn_horizontal_pass_neon_f32( edge_mode: EdgeMode, ) { unsafe { - let mut full_buffer = Box::new([NeonF32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonF32x4::default(); 1024 * 4]; let (bf0, rem) = full_buffer.split_at_mut(1024); let (bf1, rem) = rem.split_at_mut(1024); let (bf2, bf3) = rem.split_at_mut(1024); - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undef_bytes); - let width_wide = width as i64; let radius_64 = radius as i64; @@ -326,7 +322,7 @@ pub(crate) fn fgn_horizontal_pass_neon_f32( for x in start_x..(width as i64) { if x >= 0 { - let current_px = x as usize * CHANNELS_COUNT; + let current_px = x as usize * CN; let prepared_px0 = vmulq_f32(summs0, f_weight); let prepared_px1 = vmulq_f32(summs1, f_weight); @@ -338,10 +334,10 @@ pub(crate) fn fgn_horizontal_pass_neon_f32( let dst_ptr2 = bytes.slice.as_ptr().add(current_y2 + current_px) as *mut f32; let dst_ptr3 = bytes.slice.as_ptr().add(current_y3 + current_px) as *mut f32; - store_f32::(dst_ptr0, prepared_px0); - store_f32::(dst_ptr1, prepared_px1); - store_f32::(dst_ptr2, prepared_px2); - store_f32::(dst_ptr3, prepared_px3); + store_f32::(dst_ptr0, prepared_px0); + store_f32::(dst_ptr1, prepared_px1); + store_f32::(dst_ptr2, prepared_px2); + store_f32::(dst_ptr3, prepared_px3); let d_a_1 = ((x + radius_64) & 1023) as usize; let d_a_2 = ((x - radius_64) & 1023) as usize; @@ -407,17 +403,17 @@ pub(crate) fn fgn_horizontal_pass_neon_f32( } let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CHANNELS_COUNT; + let next_row_px = next_row_x * CN; let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut f32; let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut f32; let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut f32; let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut f32; - let pixel_color0 = load_f32_fast::(s_ptr0); - let pixel_color1 = load_f32_fast::(s_ptr1); - let pixel_color2 = load_f32_fast::(s_ptr2); - let pixel_color3 = load_f32_fast::(s_ptr3); + let pixel_color0 = load_f32_fast::(s_ptr0); + let pixel_color1 = load_f32_fast::(s_ptr1); + let pixel_color2 = load_f32_fast::(s_ptr2); + let pixel_color3 = load_f32_fast::(s_ptr3); let a_i = ((x + 2 * radius_64) & 1023) as usize; @@ -455,12 +451,12 @@ pub(crate) fn fgn_horizontal_pass_neon_f32( for x in start_x..(width as i64) { if x >= 0 { let current_y = ((y as i64) * (stride as i64)) as usize; - let current_px = x as usize * CHANNELS_COUNT; + let current_px = x as usize * CN; let prepared_px = vmulq_f32(summs, f_weight); let dst_ptr = bytes.slice.as_ptr().add(current_y + current_px) as *mut f32; - store_f32::(dst_ptr, prepared_px); + store_f32::(dst_ptr, prepared_px); let d_arr_index_1 = ((x + radius_64) & 1023) as usize; let d_arr_index_2 = ((x - radius_64) & 1023) as usize; @@ -497,10 +493,10 @@ pub(crate) fn fgn_horizontal_pass_neon_f32( let next_row_y = y * (stride as usize); let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CHANNELS_COUNT; + let next_row_px = next_row_x * CN; let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f32; - let pixel_color = load_f32_fast::(s_ptr); + let pixel_color = load_f32_fast::(s_ptr); let arr_index = ((x + 2 * radius_64) & 1023) as usize; let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; diff --git a/src/neon/fast_gaussian_next_q0_31.rs b/src/neon/fast_gaussian_next_q0_31.rs index 986ac4f3..68195aa7 100644 --- a/src/neon/fast_gaussian_next_q0_31.rs +++ b/src/neon/fast_gaussian_next_q0_31.rs @@ -25,15 +25,15 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::fast_gaussian::NeonI32x4; use crate::neon::{load_u8_s32_fast, store_u8_s32_x4, store_u8x8_m4, vmulq_by_3_s32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::aarch64::*; -pub(crate) fn fgn_vertical_pass_neon_u8_rdm( - undefined_slice: &UnsafeSlice, +pub(crate) fn fgn_vertical_pass_neon_u8_rdm( + undefined_slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -43,7 +43,7 @@ pub(crate) fn fgn_vertical_pass_neon_u8_rdm( edge_mode: EdgeMode, ) { unsafe { - fgn_vertical_pass_neon_impl_rdm::( + fgn_vertical_pass_neon_impl_rdm::( undefined_slice, stride, width, @@ -57,8 +57,8 @@ pub(crate) fn fgn_vertical_pass_neon_u8_rdm( } #[target_feature(enable = "rdm")] -unsafe fn fgn_vertical_pass_neon_impl_rdm( - undefined_slice: &UnsafeSlice, +fn fgn_vertical_pass_neon_impl_rdm( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -68,9 +68,7 @@ unsafe fn fgn_vertical_pass_neon_impl_rdm( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 4]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -85,7 +83,7 @@ unsafe fn fgn_vertical_pass_neon_impl_rdm( let mut xx = start; - while xx + 4 < width.min(end) { + while xx + 4 <= width.min(end) { let mut diffs0 = vdupq_n_s32(0); let mut diffs1 = vdupq_n_s32(0); let mut diffs2 = vdupq_n_s32(0); @@ -338,8 +336,8 @@ unsafe fn fgn_vertical_pass_neon_impl_rdm( } } -pub(crate) fn fgn_horizontal_pass_neon_u8_rdm( - undefined_slice: &UnsafeSlice, +pub(crate) fn fgn_horizontal_pass_neon_u8_rdm( + undefined_slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -349,7 +347,7 @@ pub(crate) fn fgn_horizontal_pass_neon_u8_rdm( edge_mode: EdgeMode, ) { unsafe { - fgn_horizontal_pass_neon_impl::( + fgn_horizontal_pass_neon_impl::( undefined_slice, stride, width, @@ -363,8 +361,8 @@ pub(crate) fn fgn_horizontal_pass_neon_u8_rdm( } #[target_feature(enable = "rdm")] -unsafe fn fgn_horizontal_pass_neon_impl( - undefined_slice: &UnsafeSlice, +fn fgn_horizontal_pass_neon_impl( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -374,9 +372,7 @@ unsafe fn fgn_horizontal_pass_neon_impl( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 4]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); diff --git a/src/neon/fast_gaussian_next_u16.rs b/src/neon/fast_gaussian_next_u16.rs index fcc02dad..311226c2 100644 --- a/src/neon/fast_gaussian_next_u16.rs +++ b/src/neon/fast_gaussian_next_u16.rs @@ -25,11 +25,11 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::fast_gaussian::NeonI32x4; use crate::neon::{load_u16_s32_fast, store_u16_s32_x4, store_u16x4, vmulq_by_3_s32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::aarch64::*; pub(crate) fn fgn_vertical_pass_neon_u16( @@ -43,7 +43,7 @@ pub(crate) fn fgn_vertical_pass_neon_u16( edge_mode: EdgeMode, ) { unsafe { - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 4]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -57,7 +57,7 @@ pub(crate) fn fgn_vertical_pass_neon_u16( let mut xx = start; - while xx + 4 < width.min(end) { + while xx + 4 <= width.min(end) { let mut diffs0 = vdupq_n_s32(0); let mut diffs1 = vdupq_n_s32(0); let mut diffs2 = vdupq_n_s32(0); @@ -330,7 +330,7 @@ pub(crate) fn fgn_horizontal_pass_neon_u16( edge_mode: EdgeMode, ) { unsafe { - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 4]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -344,7 +344,7 @@ pub(crate) fn fgn_horizontal_pass_neon_u16( let mut yy = start; - while yy + 4 < height.min(end) { + while yy + 4 <= height.min(end) { let mut diffs0 = vdupq_n_s32(0); let mut diffs1 = vdupq_n_s32(0); let mut diffs2 = vdupq_n_s32(0); diff --git a/src/neon/fast_gaussian_next_u16_q0_31.rs b/src/neon/fast_gaussian_next_u16_q0_31.rs index fa0dc590..ff3b120f 100644 --- a/src/neon/fast_gaussian_next_u16_q0_31.rs +++ b/src/neon/fast_gaussian_next_u16_q0_31.rs @@ -25,11 +25,11 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::fast_gaussian::NeonI32x4; use crate::neon::{load_u16_s32_fast, store_u16_s32_x4, store_u16x4, vmulq_by_3_s32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::aarch64::*; pub(crate) fn fgn_vertical_pass_neon_u16_q0_31( @@ -50,7 +50,7 @@ pub(crate) fn fgn_vertical_pass_neon_u16_q0_31( } #[target_feature(enable = "rdm")] -unsafe fn fgn_vertical_pass_q0_31_impl( +fn fgn_vertical_pass_q0_31_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -61,7 +61,7 @@ unsafe fn fgn_vertical_pass_q0_31_impl( edge_mode: EdgeMode, ) { unsafe { - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 4]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -347,7 +347,7 @@ pub(crate) fn fgn_horizontal_pass_neon_u16_q0_31( } #[target_feature(enable = "rdm")] -unsafe fn fgn_horizontal_pass_q0_31_impl( +fn fgn_horizontal_pass_q0_31_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -358,7 +358,7 @@ unsafe fn fgn_horizontal_pass_q0_31_impl( edge_mode: EdgeMode, ) { unsafe { - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 4]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 4]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -374,7 +374,7 @@ unsafe fn fgn_horizontal_pass_q0_31_impl( let mut yy = start; - while yy + 4 < height.min(end) { + while yy + 4 <= height.min(end) { let mut diffs0 = vdupq_n_s32(0); let mut diffs1 = vdupq_n_s32(0); let mut diffs2 = vdupq_n_s32(0); diff --git a/src/neon/fast_gaussian_q0_31.rs b/src/neon/fast_gaussian_q0_31.rs index 7fc0d644..363eca04 100644 --- a/src/neon/fast_gaussian_q0_31.rs +++ b/src/neon/fast_gaussian_q0_31.rs @@ -25,15 +25,15 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::fast_gaussian::NeonI32x4; use crate::neon::{load_u8_s32_fast, store_u8_s32_x5, store_u8x8_m4}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::aarch64::*; -pub(crate) fn fg_horizontal_pass_neon_u8_rdm( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_horizontal_pass_neon_u8_rdm( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -43,22 +43,15 @@ pub(crate) fn fg_horizontal_pass_neon_u8_rdm( edge_mode: EdgeMode, ) { unsafe { - fg_horizontal_pass_neon_rdm::( - undefined_slice, - stride, - width, - height, - radius, - start, - end, - edge_mode, + fg_horizontal_pass_neon_rdm::( + slice, stride, width, height, radius, start, end, edge_mode, ); } } #[target_feature(enable = "rdm")] -unsafe fn fg_horizontal_pass_neon_rdm( - undefined_slice: &UnsafeSlice, +fn fg_horizontal_pass_neon_rdm( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -68,9 +61,7 @@ unsafe fn fg_horizontal_pass_neon_rdm( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 5]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 5]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -88,7 +79,7 @@ unsafe fn fg_horizontal_pass_neon_rdm( let mut yy = start; - while yy + 5 < height.min(end) { + while yy + 5 <= height.min(end) { let mut diffs0 = vdupq_n_s32(0); let mut diffs1 = vdupq_n_s32(0); let mut diffs2 = vdupq_n_s32(0); @@ -304,8 +295,8 @@ unsafe fn fg_horizontal_pass_neon_rdm( } } -pub(crate) fn fg_vertical_pass_neon_u8_rdm( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_vertical_pass_neon_u8_rdm( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -315,22 +306,15 @@ pub(crate) fn fg_vertical_pass_neon_u8_rdm( edge_mode: EdgeMode, ) { unsafe { - fg_vertical_pass_neon_rdm::( - undefined_slice, - stride, - width, - height, - radius, - start, - end, - edge_mode, + fg_vertical_pass_neon_rdm::( + slice, stride, width, height, radius, start, end, edge_mode, ); } } #[target_feature(enable = "rdm")] -unsafe fn fg_vertical_pass_neon_rdm( - undefined_slice: &UnsafeSlice, +fn fg_vertical_pass_neon_rdm( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -340,9 +324,7 @@ unsafe fn fg_vertical_pass_neon_rdm( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 5]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 5]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -361,7 +343,7 @@ unsafe fn fg_vertical_pass_neon_rdm( let mut xx = start; - while xx + 5 < width.min(end) { + while xx + 5 <= width.min(end) { let mut diffs0 = vdupq_n_s32(0); let mut diffs1 = vdupq_n_s32(0); let mut diffs2 = vdupq_n_s32(0); diff --git a/src/neon/fast_gaussian_u16.rs b/src/neon/fast_gaussian_u16.rs index dcccb0dd..35191fcc 100644 --- a/src/neon/fast_gaussian_u16.rs +++ b/src/neon/fast_gaussian_u16.rs @@ -25,11 +25,11 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::neon::fast_gaussian::NeonI32x4; use crate::neon::{load_u16_s32_fast, store_u16_s32_x5, store_u16x4, vmulq_s32_f32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use std::arch::aarch64::*; pub(crate) fn fg_horizontal_pass_neon_u16( @@ -43,7 +43,7 @@ pub(crate) fn fg_horizontal_pass_neon_u16( edge_mode: EdgeMode, ) { unsafe { - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 5]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 5]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); @@ -296,7 +296,7 @@ pub(crate) fn fg_vertical_pass_neon_u16( edge_mode: EdgeMode, ) { unsafe { - let mut full_buffer = Box::new([NeonI32x4::default(); 1024 * 5]); + let mut full_buffer = [NeonI32x4::default(); 1024 * 5]; let (buffer0, rem) = full_buffer.split_at_mut(1024); let (buffer1, rem) = rem.split_at_mut(1024); diff --git a/src/neon/utils.rs b/src/neon/utils.rs index f8ddd6ac..8b13ae74 100644 --- a/src/neon/utils.rs +++ b/src/neon/utils.rs @@ -35,37 +35,41 @@ use std::arch::aarch64::*; #[inline(always)] pub unsafe fn load_u8_s32_fast(ptr: *const u8) -> int32x4_t { - vreinterpretq_s32_u32(load_u8_u32_fast::(ptr)) + unsafe { vreinterpretq_s32_u32(load_u8_u32_fast::(ptr)) } } #[inline(always)] pub unsafe fn store_f32(dst_ptr: *mut f32, regi: float32x4_t) { - if CN == 4 { - vst1q_f32(dst_ptr, regi); - } else if CN == 3 { - vst1q_lane_f64::<0>(dst_ptr as *mut f64, vreinterpretq_f64_f32(regi)); - vst1q_lane_f32::<2>(dst_ptr.add(2), regi); - } else if CN == 2 { - vst1q_lane_f64::<0>(dst_ptr as *mut f64, vreinterpretq_f64_f32(regi)); - } else { - vst1q_lane_f32::<0>(dst_ptr, regi); + unsafe { + if CN == 4 { + vst1q_f32(dst_ptr, regi); + } else if CN == 3 { + vst1q_lane_f64::<0>(dst_ptr as *mut f64, vreinterpretq_f64_f32(regi)); + vst1q_lane_f32::<2>(dst_ptr.add(2), regi); + } else if CN == 2 { + vst1q_lane_f64::<0>(dst_ptr as *mut f64, vreinterpretq_f64_f32(regi)); + } else { + vst1q_lane_f32::<0>(dst_ptr, regi); + } } } #[inline(always)] pub unsafe fn store_u8_s32(dst_ptr: *mut u8, regi: int32x4_t) { - let s16 = vqmovun_s32(regi); - let u16_f = vcombine_u16(s16, s16); - let v8 = vqmovn_u16(u16_f); - if CN == 4 { - vst1_lane_u32::<0>(dst_ptr as *mut u32, vreinterpret_u32_u8(v8)); - } else if CN == 3 { - vst1_lane_u16::<0>(dst_ptr as *mut u16, vreinterpret_u16_u8(v8)); - vst1_lane_u8::<2>(dst_ptr.add(2), v8); - } else if CN == 2 { - vst1_lane_u16::<0>(dst_ptr as *mut u16, vreinterpret_u16_u8(v8)); - } else { - vst1_lane_u8::<0>(dst_ptr, v8); + unsafe { + let s16 = vqmovun_s32(regi); + let u16_f = vcombine_u16(s16, s16); + let v8 = vqmovn_u16(u16_f); + if CN == 4 { + vst1_lane_u32::<0>(dst_ptr as *mut u32, vreinterpret_u32_u8(v8)); + } else if CN == 3 { + vst1_lane_u16::<0>(dst_ptr as *mut u16, vreinterpret_u16_u8(v8)); + vst1_lane_u8::<2>(dst_ptr.add(2), v8); + } else if CN == 2 { + vst1_lane_u16::<0>(dst_ptr as *mut u16, vreinterpret_u16_u8(v8)); + } else { + vst1_lane_u8::<0>(dst_ptr, v8); + } } } @@ -74,42 +78,44 @@ pub unsafe fn store_u8_s32_x4( dst_ptr: (*mut u8, *mut u8, *mut u8, *mut u8), regi: int32x4x4_t, ) { - let s16_0 = vreinterpret_u16_s16(vqmovn_s32(regi.0)); - let s16_1 = vreinterpret_u16_s16(vqmovn_s32(regi.1)); - let s16_2 = vreinterpret_u16_s16(vqmovn_s32(regi.2)); - let s16_3 = vreinterpret_u16_s16(vqmovn_s32(regi.3)); - let u16_f_0 = vcombine_u16(s16_0, s16_0); - let u16_f_1 = vcombine_u16(s16_1, s16_1); - let u16_f_2 = vcombine_u16(s16_2, s16_2); - let u16_f_3 = vcombine_u16(s16_3, s16_3); - let v8_0 = vqmovn_u16(u16_f_0); - let v8_1 = vqmovn_u16(u16_f_1); - let v8_2 = vqmovn_u16(u16_f_2); - let v8_3 = vqmovn_u16(u16_f_3); - if CN == 4 { - vst1_lane_u32::<0>(dst_ptr.0 as *mut u32, vreinterpret_u32_u8(v8_0)); - vst1_lane_u32::<0>(dst_ptr.1 as *mut u32, vreinterpret_u32_u8(v8_1)); - vst1_lane_u32::<0>(dst_ptr.2 as *mut u32, vreinterpret_u32_u8(v8_2)); - vst1_lane_u32::<0>(dst_ptr.3 as *mut u32, vreinterpret_u32_u8(v8_3)); - } else if CN == 3 { - vst1_lane_u16::<0>(dst_ptr.0 as *mut u16, vreinterpret_u16_u8(v8_0)); - vst1_lane_u8::<2>(dst_ptr.0.add(2), v8_0); - vst1_lane_u16::<0>(dst_ptr.1 as *mut u16, vreinterpret_u16_u8(v8_1)); - vst1_lane_u8::<2>(dst_ptr.1.add(2), v8_1); - vst1_lane_u16::<0>(dst_ptr.2 as *mut u16, vreinterpret_u16_u8(v8_2)); - vst1_lane_u8::<2>(dst_ptr.2.add(2), v8_2); - vst1_lane_u16::<0>(dst_ptr.3 as *mut u16, vreinterpret_u16_u8(v8_3)); - vst1_lane_u8::<2>(dst_ptr.3.add(2), v8_3); - } else if CN == 2 { - vst1_lane_u16::<0>(dst_ptr.0 as *mut u16, vreinterpret_u16_u8(v8_0)); - vst1_lane_u16::<0>(dst_ptr.1 as *mut u16, vreinterpret_u16_u8(v8_1)); - vst1_lane_u16::<0>(dst_ptr.2 as *mut u16, vreinterpret_u16_u8(v8_2)); - vst1_lane_u16::<0>(dst_ptr.3 as *mut u16, vreinterpret_u16_u8(v8_3)); - } else { - vst1_lane_u8::<0>(dst_ptr.0, v8_0); - vst1_lane_u8::<0>(dst_ptr.1, v8_1); - vst1_lane_u8::<0>(dst_ptr.2, v8_2); - vst1_lane_u8::<0>(dst_ptr.3, v8_3); + unsafe { + let s16_0 = vreinterpret_u16_s16(vqmovn_s32(regi.0)); + let s16_1 = vreinterpret_u16_s16(vqmovn_s32(regi.1)); + let s16_2 = vreinterpret_u16_s16(vqmovn_s32(regi.2)); + let s16_3 = vreinterpret_u16_s16(vqmovn_s32(regi.3)); + let u16_f_0 = vcombine_u16(s16_0, s16_0); + let u16_f_1 = vcombine_u16(s16_1, s16_1); + let u16_f_2 = vcombine_u16(s16_2, s16_2); + let u16_f_3 = vcombine_u16(s16_3, s16_3); + let v8_0 = vqmovn_u16(u16_f_0); + let v8_1 = vqmovn_u16(u16_f_1); + let v8_2 = vqmovn_u16(u16_f_2); + let v8_3 = vqmovn_u16(u16_f_3); + if CN == 4 { + vst1_lane_u32::<0>(dst_ptr.0 as *mut u32, vreinterpret_u32_u8(v8_0)); + vst1_lane_u32::<0>(dst_ptr.1 as *mut u32, vreinterpret_u32_u8(v8_1)); + vst1_lane_u32::<0>(dst_ptr.2 as *mut u32, vreinterpret_u32_u8(v8_2)); + vst1_lane_u32::<0>(dst_ptr.3 as *mut u32, vreinterpret_u32_u8(v8_3)); + } else if CN == 3 { + vst1_lane_u16::<0>(dst_ptr.0 as *mut u16, vreinterpret_u16_u8(v8_0)); + vst1_lane_u8::<2>(dst_ptr.0.add(2), v8_0); + vst1_lane_u16::<0>(dst_ptr.1 as *mut u16, vreinterpret_u16_u8(v8_1)); + vst1_lane_u8::<2>(dst_ptr.1.add(2), v8_1); + vst1_lane_u16::<0>(dst_ptr.2 as *mut u16, vreinterpret_u16_u8(v8_2)); + vst1_lane_u8::<2>(dst_ptr.2.add(2), v8_2); + vst1_lane_u16::<0>(dst_ptr.3 as *mut u16, vreinterpret_u16_u8(v8_3)); + vst1_lane_u8::<2>(dst_ptr.3.add(2), v8_3); + } else if CN == 2 { + vst1_lane_u16::<0>(dst_ptr.0 as *mut u16, vreinterpret_u16_u8(v8_0)); + vst1_lane_u16::<0>(dst_ptr.1 as *mut u16, vreinterpret_u16_u8(v8_1)); + vst1_lane_u16::<0>(dst_ptr.2 as *mut u16, vreinterpret_u16_u8(v8_2)); + vst1_lane_u16::<0>(dst_ptr.3 as *mut u16, vreinterpret_u16_u8(v8_3)); + } else { + vst1_lane_u8::<0>(dst_ptr.0, v8_0); + vst1_lane_u8::<0>(dst_ptr.1, v8_1); + vst1_lane_u8::<0>(dst_ptr.2, v8_2); + vst1_lane_u8::<0>(dst_ptr.3, v8_3); + } } } @@ -120,310 +126,343 @@ pub unsafe fn store_u8_s32_x5( regi: int32x4x4_t, add: int32x4_t, ) { - let s16_0 = vreinterpret_u16_s16(vqmovn_s32(regi.0)); - let s16_1 = vreinterpret_u16_s16(vqmovn_s32(regi.1)); - let s16_2 = vreinterpret_u16_s16(vqmovn_s32(regi.2)); - let s16_3 = vreinterpret_u16_s16(vqmovn_s32(regi.3)); - let s16_4 = vreinterpret_u16_s16(vqmovn_s32(add)); - let u16_f_0 = vcombine_u16(s16_0, s16_0); - let u16_f_1 = vcombine_u16(s16_1, s16_1); - let u16_f_2 = vcombine_u16(s16_2, s16_2); - let u16_f_3 = vcombine_u16(s16_3, s16_3); - let u16_f_4 = vcombine_u16(s16_4, s16_4); - let v8_0 = vqmovn_u16(u16_f_0); - let v8_1 = vqmovn_u16(u16_f_1); - let v8_2 = vqmovn_u16(u16_f_2); - let v8_3 = vqmovn_u16(u16_f_3); - let v8_4 = vqmovn_u16(u16_f_4); - if CN == 4 { - vst1_lane_u32::<0>(dst_ptr.0 as *mut u32, vreinterpret_u32_u8(v8_0)); - vst1_lane_u32::<0>(dst_ptr.1 as *mut u32, vreinterpret_u32_u8(v8_1)); - vst1_lane_u32::<0>(dst_ptr.2 as *mut u32, vreinterpret_u32_u8(v8_2)); - vst1_lane_u32::<0>(dst_ptr.3 as *mut u32, vreinterpret_u32_u8(v8_3)); - vst1_lane_u32::<0>(dst_ptr.4 as *mut u32, vreinterpret_u32_u8(v8_4)); - } else if CN == 3 { - vst1_lane_u16::<0>(dst_ptr.0 as *mut u16, vreinterpret_u16_u8(v8_0)); - vst1_lane_u8::<2>(dst_ptr.0.add(2), v8_0); - vst1_lane_u16::<0>(dst_ptr.1 as *mut u16, vreinterpret_u16_u8(v8_1)); - vst1_lane_u8::<2>(dst_ptr.1.add(2), v8_1); - vst1_lane_u16::<0>(dst_ptr.2 as *mut u16, vreinterpret_u16_u8(v8_2)); - vst1_lane_u8::<2>(dst_ptr.2.add(2), v8_2); - vst1_lane_u16::<0>(dst_ptr.3 as *mut u16, vreinterpret_u16_u8(v8_3)); - vst1_lane_u8::<2>(dst_ptr.3.add(2), v8_3); - vst1_lane_u16::<0>(dst_ptr.4 as *mut u16, vreinterpret_u16_u8(v8_4)); - vst1_lane_u8::<2>(dst_ptr.4.add(2), v8_4); - } else if CN == 2 { - vst1_lane_u16::<0>(dst_ptr.0 as *mut u16, vreinterpret_u16_u8(v8_0)); - vst1_lane_u16::<0>(dst_ptr.1 as *mut u16, vreinterpret_u16_u8(v8_1)); - vst1_lane_u16::<0>(dst_ptr.2 as *mut u16, vreinterpret_u16_u8(v8_2)); - vst1_lane_u16::<0>(dst_ptr.3 as *mut u16, vreinterpret_u16_u8(v8_3)); - vst1_lane_u16::<0>(dst_ptr.4 as *mut u16, vreinterpret_u16_u8(v8_4)); - } else { - vst1_lane_u8::<0>(dst_ptr.0, v8_0); - vst1_lane_u8::<0>(dst_ptr.1, v8_1); - vst1_lane_u8::<0>(dst_ptr.2, v8_2); - vst1_lane_u8::<0>(dst_ptr.3, v8_3); - vst1_lane_u8::<0>(dst_ptr.4, v8_4); + unsafe { + let s16_0 = vreinterpret_u16_s16(vqmovn_s32(regi.0)); + let s16_1 = vreinterpret_u16_s16(vqmovn_s32(regi.1)); + let s16_2 = vreinterpret_u16_s16(vqmovn_s32(regi.2)); + let s16_3 = vreinterpret_u16_s16(vqmovn_s32(regi.3)); + let s16_4 = vreinterpret_u16_s16(vqmovn_s32(add)); + let u16_f_0 = vcombine_u16(s16_0, s16_0); + let u16_f_1 = vcombine_u16(s16_1, s16_1); + let u16_f_2 = vcombine_u16(s16_2, s16_2); + let u16_f_3 = vcombine_u16(s16_3, s16_3); + let u16_f_4 = vcombine_u16(s16_4, s16_4); + let v8_0 = vqmovn_u16(u16_f_0); + let v8_1 = vqmovn_u16(u16_f_1); + let v8_2 = vqmovn_u16(u16_f_2); + let v8_3 = vqmovn_u16(u16_f_3); + let v8_4 = vqmovn_u16(u16_f_4); + if CN == 4 { + vst1_lane_u32::<0>(dst_ptr.0 as *mut u32, vreinterpret_u32_u8(v8_0)); + vst1_lane_u32::<0>(dst_ptr.1 as *mut u32, vreinterpret_u32_u8(v8_1)); + vst1_lane_u32::<0>(dst_ptr.2 as *mut u32, vreinterpret_u32_u8(v8_2)); + vst1_lane_u32::<0>(dst_ptr.3 as *mut u32, vreinterpret_u32_u8(v8_3)); + vst1_lane_u32::<0>(dst_ptr.4 as *mut u32, vreinterpret_u32_u8(v8_4)); + } else if CN == 3 { + vst1_lane_u16::<0>(dst_ptr.0 as *mut u16, vreinterpret_u16_u8(v8_0)); + vst1_lane_u8::<2>(dst_ptr.0.add(2), v8_0); + vst1_lane_u16::<0>(dst_ptr.1 as *mut u16, vreinterpret_u16_u8(v8_1)); + vst1_lane_u8::<2>(dst_ptr.1.add(2), v8_1); + vst1_lane_u16::<0>(dst_ptr.2 as *mut u16, vreinterpret_u16_u8(v8_2)); + vst1_lane_u8::<2>(dst_ptr.2.add(2), v8_2); + vst1_lane_u16::<0>(dst_ptr.3 as *mut u16, vreinterpret_u16_u8(v8_3)); + vst1_lane_u8::<2>(dst_ptr.3.add(2), v8_3); + vst1_lane_u16::<0>(dst_ptr.4 as *mut u16, vreinterpret_u16_u8(v8_4)); + vst1_lane_u8::<2>(dst_ptr.4.add(2), v8_4); + } else if CN == 2 { + vst1_lane_u16::<0>(dst_ptr.0 as *mut u16, vreinterpret_u16_u8(v8_0)); + vst1_lane_u16::<0>(dst_ptr.1 as *mut u16, vreinterpret_u16_u8(v8_1)); + vst1_lane_u16::<0>(dst_ptr.2 as *mut u16, vreinterpret_u16_u8(v8_2)); + vst1_lane_u16::<0>(dst_ptr.3 as *mut u16, vreinterpret_u16_u8(v8_3)); + vst1_lane_u16::<0>(dst_ptr.4 as *mut u16, vreinterpret_u16_u8(v8_4)); + } else { + vst1_lane_u8::<0>(dst_ptr.0, v8_0); + vst1_lane_u8::<0>(dst_ptr.1, v8_1); + vst1_lane_u8::<0>(dst_ptr.2, v8_2); + vst1_lane_u8::<0>(dst_ptr.3, v8_3); + vst1_lane_u8::<0>(dst_ptr.4, v8_4); + } } } #[inline(always)] pub unsafe fn store_u8_u32(dst_ptr: *mut u8, regi: uint32x4_t) { - let s16 = vqmovn_u32(regi); - let u16_f = vcombine_u16(s16, s16); - let v8 = vqmovn_u16(u16_f); - if CN == 4 { - vst1_lane_u32::<0>(dst_ptr as *mut u32, vreinterpret_u32_u8(v8)); - } else if CN == 3 { - vst1_lane_u16::<0>(dst_ptr as *mut u16, vreinterpret_u16_u8(v8)); - vst1_lane_u8::<2>(dst_ptr.add(2), v8); - } else if CN == 2 { - vst1_lane_u16::<0>(dst_ptr as *mut u16, vreinterpret_u16_u8(v8)); - } else { - vst1_lane_u8::<0>(dst_ptr, v8); + unsafe { + let s16 = vqmovn_u32(regi); + let u16_f = vcombine_u16(s16, s16); + let v8 = vqmovn_u16(u16_f); + if CN == 4 { + vst1_lane_u32::<0>(dst_ptr as *mut u32, vreinterpret_u32_u8(v8)); + } else if CN == 3 { + vst1_lane_u16::<0>(dst_ptr as *mut u16, vreinterpret_u16_u8(v8)); + vst1_lane_u8::<2>(dst_ptr.add(2), v8); + } else if CN == 2 { + vst1_lane_u16::<0>(dst_ptr as *mut u16, vreinterpret_u16_u8(v8)); + } else { + vst1_lane_u8::<0>(dst_ptr, v8); + } } } #[inline(always)] pub unsafe fn load_f32_fast(ptr: *const f32) -> float32x4_t { - if CN == 4 { - return vld1q_f32(ptr); - } else if CN == 3 { - return vld1q_f32( - [ - ptr.read_unaligned(), - ptr.add(1).read_unaligned(), - ptr.add(2).read_unaligned(), - 0f32, - ] - .as_ptr(), - ); - } else if CN == 2 { - return vld1q_f32( - [ - ptr.read_unaligned(), - ptr.add(1).read_unaligned(), - 0f32, - 0f32, - ] - .as_ptr(), - ); + unsafe { + if CN == 4 { + return vld1q_f32(ptr); + } else if CN == 3 { + return vld1q_f32( + [ + ptr.read_unaligned(), + ptr.add(1).read_unaligned(), + ptr.add(2).read_unaligned(), + 0f32, + ] + .as_ptr(), + ); + } else if CN == 2 { + return vld1q_f32( + [ + ptr.read_unaligned(), + ptr.add(1).read_unaligned(), + 0f32, + 0f32, + ] + .as_ptr(), + ); + } + vld1q_f32([ptr.read_unaligned(), 0f32, 0f32, 0f32].as_ptr()) } - vld1q_f32([ptr.read_unaligned(), 0f32, 0f32, 0f32].as_ptr()) } #[inline(always)] -pub unsafe fn load_u8_u32_fast(ptr: *const u8) -> uint32x4_t { - if CHANNELS_COUNT == 4 { - let v0 = vreinterpret_u8_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))); - vmovl_u16(vget_low_u16(vmovl_u8(v0))) - } else if CHANNELS_COUNT == 3 { - let mut v0 = vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const u16, vdup_n_u16(0))); - v0 = vld1_lane_u8::<2>(ptr.add(2), v0); - vmovl_u16(vget_low_u16(vmovl_u8(v0))) - } else if CHANNELS_COUNT == 2 { - let v0 = vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const u16, vdup_n_u16(0))); - vmovl_u16(vget_low_u16(vmovl_u8(v0))) - } else { - vmovl_u16(vget_low_u16(vmovl_u8(vld1_lane_u8::<0>(ptr, vdup_n_u8(0))))) +pub unsafe fn load_u8_u32_fast(ptr: *const u8) -> uint32x4_t { + unsafe { + if CN == 4 { + let v0 = vreinterpret_u8_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))); + vmovl_u16(vget_low_u16(vmovl_u8(v0))) + } else if CN == 3 { + let mut v0 = vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const u16, vdup_n_u16(0))); + v0 = vld1_lane_u8::<2>(ptr.add(2), v0); + vmovl_u16(vget_low_u16(vmovl_u8(v0))) + } else if CN == 2 { + let v0 = vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const u16, vdup_n_u16(0))); + vmovl_u16(vget_low_u16(vmovl_u8(v0))) + } else { + vmovl_u16(vget_low_u16(vmovl_u8(vld1_lane_u8::<0>(ptr, vdup_n_u8(0))))) + } } } #[inline(always)] pub unsafe fn load_u16_s32_fast(ptr: *const u16) -> int32x4_t { - if CN == 4 { - vreinterpretq_s32_u32(vmovl_u16(vld1_u16(ptr))) - } else if CN == 3 { - let mut v0 = vreinterpret_u16_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))); - v0 = vld1_lane_u16::<2>(ptr.add(2), v0); - vreinterpretq_s32_u32(vmovl_u16(v0)) - } else if CN == 2 { - let v0 = vreinterpret_u16_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))); - vreinterpretq_s32_u32(vmovl_u16(v0)) - } else { - vreinterpretq_s32_u32(vmovl_u16(vld1_lane_u16::<0>(ptr, vdup_n_u16(0)))) + unsafe { + if CN == 4 { + vreinterpretq_s32_u32(vmovl_u16(vld1_u16(ptr))) + } else if CN == 3 { + let mut v0 = vreinterpret_u16_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))); + v0 = vld1_lane_u16::<2>(ptr.add(2), v0); + vreinterpretq_s32_u32(vmovl_u16(v0)) + } else if CN == 2 { + let v0 = vreinterpret_u16_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))); + vreinterpretq_s32_u32(vmovl_u16(v0)) + } else { + vreinterpretq_s32_u32(vmovl_u16(vld1_lane_u16::<0>(ptr, vdup_n_u16(0)))) + } } } #[inline(always)] -pub(crate) unsafe fn vmulq_u32_f32(a: uint32x4_t, b: float32x4_t) -> uint32x4_t { - let cvt = vcvtq_f32_u32(a); - vcvtaq_u32_f32(vmulq_f32(cvt, b)) +pub(crate) fn vmulq_u32_f32(a: uint32x4_t, b: float32x4_t) -> uint32x4_t { + unsafe { + let cvt = vcvtq_f32_u32(a); + vcvtaq_u32_f32(vmulq_f32(cvt, b)) + } } #[inline(always)] -pub(crate) unsafe fn vmulq_u16_low_f32(a: uint16x8_t, b: float32x4_t) -> uint32x4_t { - let cvt = vcvtq_f32_u32(vmovl_u16(vget_low_u16(a))); - vcvtaq_u32_f32(vmulq_f32(cvt, b)) +pub(crate) fn vmulq_u16_low_f32(a: uint16x8_t, b: float32x4_t) -> uint32x4_t { + unsafe { + let cvt = vcvtq_f32_u32(vmovl_u16(vget_low_u16(a))); + vcvtaq_u32_f32(vmulq_f32(cvt, b)) + } } #[inline(always)] #[allow(dead_code)] -pub(crate) unsafe fn vmulq_u16_low_s32(a: uint16x8_t, b: int32x4_t) -> int32x4_t { - let cvt = vmovl_u16(vget_low_u16(a)); - vqrdmulhq_s32(vreinterpretq_s32_u32(cvt), b) +pub(crate) fn vmulq_u16_low_s32(a: uint16x8_t, b: int32x4_t) -> int32x4_t { + unsafe { + let cvt = vmovl_u16(vget_low_u16(a)); + vqrdmulhq_s32(vreinterpretq_s32_u32(cvt), b) + } } #[inline(always)] -pub(crate) unsafe fn vmulq_s32_f32(a: int32x4_t, b: float32x4_t) -> int32x4_t { - let cvt = vcvtq_f32_s32(a); - vcvtaq_s32_f32(vmulq_f32(cvt, b)) +pub(crate) fn vmulq_s32_f32(a: int32x4_t, b: float32x4_t) -> int32x4_t { + unsafe { + let cvt = vcvtq_f32_s32(a); + vcvtaq_s32_f32(vmulq_f32(cvt, b)) + } } #[inline(always)] -pub unsafe fn load_u8_u16(ptr: *const u8) -> uint16x4_t { - if CN == 4 { - vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vld1_lane_u32::<0>( - ptr as *mut u32, - vdup_n_u32(0), - )))) - } else if CN == 3 { - let a0 = vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const _, vdup_n_u16(0))); - let a1 = vld1_lane_u8::<2>(ptr.add(2) as *const _, a0); - vget_low_u16(vmovl_u8(a1)) - } else if CN == 2 { - let a0 = vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const _, vdup_n_u16(0))); - vget_low_u16(vmovl_u8(a0)) - } else { - let a0 = vld1_lane_u8::<0>(ptr as *const _, vdup_n_u8(0)); - vget_low_u16(vmovl_u8(a0)) +pub fn load_u8_u16(ptr: *const u8) -> uint16x4_t { + unsafe { + if CN == 4 { + vget_low_u16(vmovl_u8(vreinterpret_u8_u32(vld1_lane_u32::<0>( + ptr as *mut u32, + vdup_n_u32(0), + )))) + } else if CN == 3 { + let a0 = vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const _, vdup_n_u16(0))); + let a1 = vld1_lane_u8::<2>(ptr.add(2) as *const _, a0); + vget_low_u16(vmovl_u8(a1)) + } else if CN == 2 { + let a0 = vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const _, vdup_n_u16(0))); + vget_low_u16(vmovl_u8(a0)) + } else { + let a0 = vld1_lane_u8::<0>(ptr as *const _, vdup_n_u8(0)); + vget_low_u16(vmovl_u8(a0)) + } } } #[inline(always)] -pub(crate) unsafe fn vmulq_by_3_s32(k: int32x4_t) -> int32x4_t { - vaddq_s32(vshlq_n_s32::<1>(k), k) +pub(crate) fn vmulq_by_3_s32(k: int32x4_t) -> int32x4_t { + unsafe { vaddq_s32(vshlq_n_s32::<1>(k), k) } } #[inline(always)] pub unsafe fn load_u8(ptr: *const u8) -> uint8x8_t { - if CN == 4 { - vreinterpret_u8_u32(vld1_lane_u32::<0>(ptr as *mut u32, vdup_n_u32(0))) - } else if CN == 3 { - let a0 = vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const _, vdup_n_u16(0))); - vld1_lane_u8::<2>(ptr.add(2) as *const _, a0) - } else if CN == 2 { - vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const _, vdup_n_u16(0))) - } else { - vld1_lane_u8::<0>(ptr as *const _, vdup_n_u8(0)) + unsafe { + if CN == 4 { + vreinterpret_u8_u32(vld1_lane_u32::<0>(ptr as *mut u32, vdup_n_u32(0))) + } else if CN == 3 { + let a0 = vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const _, vdup_n_u16(0))); + vld1_lane_u8::<2>(ptr.add(2) as *const _, a0) + } else if CN == 2 { + vreinterpret_u8_u16(vld1_lane_u16::<0>(ptr as *const _, vdup_n_u16(0))) + } else { + vld1_lane_u8::<0>(ptr as *const _, vdup_n_u8(0)) + } } } #[inline(always)] pub(crate) unsafe fn load_u16(ptr: *const u16) -> uint16x4_t { - if CN == 4 { - vld1_u16(ptr) - } else if CN == 3 { - let v0 = vreinterpret_u16_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))); - vld1_lane_u16::<2>(ptr.add(2), v0) - } else if CN == 2 { - vreinterpret_u16_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))) - } else { - vld1_lane_u16::<0>(ptr, vdup_n_u16(0)) + unsafe { + if CN == 4 { + vld1_u16(ptr) + } else if CN == 3 { + let v0 = vreinterpret_u16_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))); + vld1_lane_u16::<2>(ptr.add(2), v0) + } else if CN == 2 { + vreinterpret_u16_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))) + } else { + vld1_lane_u16::<0>(ptr, vdup_n_u16(0)) + } } } #[inline(always)] -pub(crate) unsafe fn p_vfmaq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { - #[cfg(target_arch = "aarch64")] - { - vfmaq_f32(a, b, c) - } - #[cfg(target_arch = "arm")] - { - vmlaq_f32(a, b, c) +pub(crate) fn p_vfmaq_f32(a: float32x4_t, b: float32x4_t, c: float32x4_t) -> float32x4_t { + unsafe { + #[cfg(target_arch = "aarch64")] + { + vfmaq_f32(a, b, c) + } + #[cfg(target_arch = "arm")] + { + vmlaq_f32(a, b, c) + } } } #[inline(always)] -pub(crate) unsafe fn prefer_vfma_f32( - a: float32x2_t, - b: float32x2_t, - c: float32x2_t, -) -> float32x2_t { - #[cfg(target_arch = "aarch64")] - { - vfma_f32(a, b, c) - } - #[cfg(target_arch = "arm")] - { - vmla_f32(a, b, c) +pub(crate) fn prefer_vfma_f32(a: float32x2_t, b: float32x2_t, c: float32x2_t) -> float32x2_t { + unsafe { + #[cfg(target_arch = "aarch64")] + { + vfma_f32(a, b, c) + } + #[cfg(target_arch = "arm")] + { + vmla_f32(a, b, c) + } } } #[cfg(feature = "nightly_f16")] #[inline(always)] pub unsafe fn load_f32_f16(ptr: *const f16) -> float32x4_t { - if CN == 4 { - let cvt = xvld_f16(ptr); - return xvcvt_f32_f16(cvt); - } else if CN == 3 { - let mut jvet = vreinterpret_u16_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))); - jvet = vld1_lane_u16::<2>(ptr.add(2) as *const u16, jvet); - let cvt = xreinterpret_f16_u16(jvet); - return xvcvt_f32_f16(cvt); - } else if CN == 2 { - let jvet = vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0)); - let cvt = xreinterpret_f16_u16(vreinterpret_u16_u32(jvet)); - return xvcvt_f32_f16(cvt); + unsafe { + if CN == 4 { + let cvt = xvld_f16(ptr); + return xvcvt_f32_f16(cvt); + } else if CN == 3 { + let mut jvet = + vreinterpret_u16_u32(vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0))); + jvet = vld1_lane_u16::<2>(ptr.add(2) as *const u16, jvet); + let cvt = xreinterpret_f16_u16(jvet); + return xvcvt_f32_f16(cvt); + } else if CN == 2 { + let jvet = vld1_lane_u32::<0>(ptr as *const u32, vdup_n_u32(0)); + let cvt = xreinterpret_f16_u16(vreinterpret_u16_u32(jvet)); + return xvcvt_f32_f16(cvt); + } + let recast = ptr as *const u16; + let cvt = xreinterpret_f16_u16(vld1_u16([recast.read_unaligned(), 0, 0, 0].as_ptr())); + xvcvt_f32_f16(cvt) } - let recast = ptr as *const u16; - let cvt = xreinterpret_f16_u16(vld1_u16([recast.read_unaligned(), 0, 0, 0].as_ptr())); - xvcvt_f32_f16(cvt) } #[cfg(feature = "nightly_f16")] #[inline(always)] pub unsafe fn store_f32_f16(dst_ptr: *mut f16, in_regi: float32x4_t) { - let out_regi = xvcvt_f16_f32(in_regi); - if CN == 4 { - xvst_f16(dst_ptr, out_regi); - } else if CN == 3 { - let casted_out = xreinterpret_u16_f16(out_regi); - let casted_ptr = dst_ptr as *mut u16; - let lo_part = vreinterpret_u32_u16(casted_out); - vst1_lane_u32::<0>(casted_ptr as *mut _, lo_part); - vst1_lane_u16::<2>(casted_ptr.add(2) as *mut _, vreinterpret_u16_u32(lo_part)); - } else if CN == 2 { - let casted_out = xreinterpret_u16_f16(out_regi); - let casted_ptr = dst_ptr as *mut u32; - let lo_part = vreinterpret_u32_u16(casted_out); - vst1_lane_u32::<0>(casted_ptr, lo_part); - } else { - let casted_out = xreinterpret_u16_f16(out_regi); - vst1_lane_u16::<0>(dst_ptr as *mut u16, casted_out); + unsafe { + let out_regi = xvcvt_f16_f32(in_regi); + if CN == 4 { + xvst_f16(dst_ptr, out_regi); + } else if CN == 3 { + let casted_out = xreinterpret_u16_f16(out_regi); + let casted_ptr = dst_ptr as *mut u16; + let lo_part = vreinterpret_u32_u16(casted_out); + vst1_lane_u32::<0>(casted_ptr as *mut _, lo_part); + vst1_lane_u16::<2>(casted_ptr.add(2) as *mut _, vreinterpret_u16_u32(lo_part)); + } else if CN == 2 { + let casted_out = xreinterpret_u16_f16(out_regi); + let casted_ptr = dst_ptr as *mut u32; + let lo_part = vreinterpret_u32_u16(casted_out); + vst1_lane_u32::<0>(casted_ptr, lo_part); + } else { + let casted_out = xreinterpret_u16_f16(out_regi); + vst1_lane_u16::<0>(dst_ptr as *mut u16, casted_out); + } } } /// Stores up to 4 values from uint8x8_t #[inline(always)] pub(crate) unsafe fn store_u8x8_m4(dst_ptr: *mut u8, in_regi: uint8x8_t) { - let casted_u32 = vreinterpret_u32_u8(in_regi); + unsafe { + let casted_u32 = vreinterpret_u32_u8(in_regi); - if CN == 4 { - vst1_lane_u32::<0>(dst_ptr as *mut _, casted_u32); - } else if CN == 3 { - vst1_lane_u16::<0>(dst_ptr as *mut _, vreinterpret_u16_u32(casted_u32)); - vst1_lane_u8::<2>(dst_ptr.add(2) as *mut _, vreinterpret_u8_u32(casted_u32)); - } else if CN == 2 { - vst1_lane_u16::<0>(dst_ptr as *mut _, vreinterpret_u16_u32(casted_u32)); - } else { - vst1_lane_u8::<0>(dst_ptr, vreinterpret_u8_u32(casted_u32)); + if CN == 4 { + vst1_lane_u32::<0>(dst_ptr as *mut _, casted_u32); + } else if CN == 3 { + vst1_lane_u16::<0>(dst_ptr as *mut _, vreinterpret_u16_u32(casted_u32)); + vst1_lane_u8::<2>(dst_ptr.add(2) as *mut _, vreinterpret_u8_u32(casted_u32)); + } else if CN == 2 { + vst1_lane_u16::<0>(dst_ptr as *mut _, vreinterpret_u16_u32(casted_u32)); + } else { + vst1_lane_u8::<0>(dst_ptr, vreinterpret_u8_u32(casted_u32)); + } } } /// Stores up to 4 values from uint16x4_t #[inline(always)] pub(crate) unsafe fn store_u16x4(dst_ptr: *mut u16, a0: uint16x4_t) { - if CN == 4 { - vst1_u16(dst_ptr, a0); - } else if CN == 3 { - vst1_lane_u32::<0>(dst_ptr as *mut _, vreinterpret_u32_u16(a0)); - vst1_lane_u16::<2>(dst_ptr.add(2) as *mut _, a0); - } else if CN == 2 { - vst1_lane_u32::<0>(dst_ptr as *mut _, vreinterpret_u32_u16(a0)); - } else { - vst1_lane_u16::<0>(dst_ptr as *mut _, a0); + unsafe { + if CN == 4 { + vst1_u16(dst_ptr, a0); + } else if CN == 3 { + vst1_lane_u32::<0>(dst_ptr as *mut _, vreinterpret_u32_u16(a0)); + vst1_lane_u16::<2>(dst_ptr.add(2) as *mut _, a0); + } else if CN == 2 { + vst1_lane_u32::<0>(dst_ptr as *mut _, vreinterpret_u32_u16(a0)); + } else { + vst1_lane_u16::<0>(dst_ptr as *mut _, a0); + } } } @@ -432,34 +471,36 @@ pub unsafe fn store_u16_s32_x4( dst_ptr: (*mut u16, *mut u16, *mut u16, *mut u16), regi: int32x4x4_t, ) { - let a0 = vqmovun_s32(regi.0); - let a1 = vqmovun_s32(regi.1); - let a2 = vqmovun_s32(regi.2); - let a3 = vqmovun_s32(regi.3); - if CN == 4 { - vst1_u16(dst_ptr.0, a0); - vst1_u16(dst_ptr.1, a1); - vst1_u16(dst_ptr.2, a2); - vst1_u16(dst_ptr.3, a3); - } else if CN == 3 { - vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); - vst1_lane_u16::<2>(dst_ptr.0.add(2) as *mut _, a0); - vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); - vst1_lane_u16::<2>(dst_ptr.1.add(2) as *mut _, a1); - vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); - vst1_lane_u16::<2>(dst_ptr.2.add(2) as *mut _, a2); - vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); - vst1_lane_u16::<2>(dst_ptr.3.add(2) as *mut _, a3); - } else if CN == 2 { - vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); - vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); - vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); - vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); - } else { - vst1_lane_u16::<0>(dst_ptr.0 as *mut _, a0); - vst1_lane_u16::<0>(dst_ptr.1 as *mut _, a1); - vst1_lane_u16::<0>(dst_ptr.2 as *mut _, a2); - vst1_lane_u16::<0>(dst_ptr.3 as *mut _, a3); + unsafe { + let a0 = vqmovun_s32(regi.0); + let a1 = vqmovun_s32(regi.1); + let a2 = vqmovun_s32(regi.2); + let a3 = vqmovun_s32(regi.3); + if CN == 4 { + vst1_u16(dst_ptr.0, a0); + vst1_u16(dst_ptr.1, a1); + vst1_u16(dst_ptr.2, a2); + vst1_u16(dst_ptr.3, a3); + } else if CN == 3 { + vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); + vst1_lane_u16::<2>(dst_ptr.0.add(2) as *mut _, a0); + vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); + vst1_lane_u16::<2>(dst_ptr.1.add(2) as *mut _, a1); + vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); + vst1_lane_u16::<2>(dst_ptr.2.add(2) as *mut _, a2); + vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); + vst1_lane_u16::<2>(dst_ptr.3.add(2) as *mut _, a3); + } else if CN == 2 { + vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); + vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); + vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); + vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); + } else { + vst1_lane_u16::<0>(dst_ptr.0 as *mut _, a0); + vst1_lane_u16::<0>(dst_ptr.1 as *mut _, a1); + vst1_lane_u16::<0>(dst_ptr.2 as *mut _, a2); + vst1_lane_u16::<0>(dst_ptr.3 as *mut _, a3); + } } } @@ -468,34 +509,36 @@ pub unsafe fn store_u16_u32_x4( dst_ptr: (*mut u16, *mut u16, *mut u16, *mut u16), regi: uint32x4x4_t, ) { - let a0 = vqmovn_u32(regi.0); - let a1 = vqmovn_u32(regi.1); - let a2 = vqmovn_u32(regi.2); - let a3 = vqmovn_u32(regi.3); - if CN == 4 { - vst1_u16(dst_ptr.0, a0); - vst1_u16(dst_ptr.1, a1); - vst1_u16(dst_ptr.2, a2); - vst1_u16(dst_ptr.3, a3); - } else if CN == 3 { - vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); - vst1_lane_u16::<2>(dst_ptr.0.add(2) as *mut _, a0); - vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); - vst1_lane_u16::<2>(dst_ptr.1.add(2) as *mut _, a1); - vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); - vst1_lane_u16::<2>(dst_ptr.2.add(2) as *mut _, a2); - vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); - vst1_lane_u16::<2>(dst_ptr.3.add(2) as *mut _, a3); - } else if CN == 2 { - vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); - vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); - vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); - vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); - } else { - vst1_lane_u16::<0>(dst_ptr.0 as *mut _, a0); - vst1_lane_u16::<0>(dst_ptr.1 as *mut _, a1); - vst1_lane_u16::<0>(dst_ptr.2 as *mut _, a2); - vst1_lane_u16::<0>(dst_ptr.3 as *mut _, a3); + unsafe { + let a0 = vqmovn_u32(regi.0); + let a1 = vqmovn_u32(regi.1); + let a2 = vqmovn_u32(regi.2); + let a3 = vqmovn_u32(regi.3); + if CN == 4 { + vst1_u16(dst_ptr.0, a0); + vst1_u16(dst_ptr.1, a1); + vst1_u16(dst_ptr.2, a2); + vst1_u16(dst_ptr.3, a3); + } else if CN == 3 { + vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); + vst1_lane_u16::<2>(dst_ptr.0.add(2) as *mut _, a0); + vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); + vst1_lane_u16::<2>(dst_ptr.1.add(2) as *mut _, a1); + vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); + vst1_lane_u16::<2>(dst_ptr.2.add(2) as *mut _, a2); + vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); + vst1_lane_u16::<2>(dst_ptr.3.add(2) as *mut _, a3); + } else if CN == 2 { + vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); + vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); + vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); + vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); + } else { + vst1_lane_u16::<0>(dst_ptr.0 as *mut _, a0); + vst1_lane_u16::<0>(dst_ptr.1 as *mut _, a1); + vst1_lane_u16::<0>(dst_ptr.2 as *mut _, a2); + vst1_lane_u16::<0>(dst_ptr.3 as *mut _, a3); + } } } @@ -505,39 +548,41 @@ pub unsafe fn store_u16_s32_x5( regi: int32x4x4_t, add: int32x4_t, ) { - let a0 = vqmovun_s32(regi.0); - let a1 = vqmovun_s32(regi.1); - let a2 = vqmovun_s32(regi.2); - let a3 = vqmovun_s32(regi.3); - let a4 = vqmovun_s32(add); - if CN == 4 { - vst1_u16(dst_ptr.0, a0); - vst1_u16(dst_ptr.1, a1); - vst1_u16(dst_ptr.2, a2); - vst1_u16(dst_ptr.3, a3); - vst1_u16(dst_ptr.4, a4); - } else if CN == 3 { - vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); - vst1_lane_u16::<2>(dst_ptr.0.add(2) as *mut _, a0); - vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); - vst1_lane_u16::<2>(dst_ptr.1.add(2) as *mut _, a1); - vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); - vst1_lane_u16::<2>(dst_ptr.2.add(2) as *mut _, a2); - vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); - vst1_lane_u16::<2>(dst_ptr.3.add(2) as *mut _, a3); - vst1_lane_u32::<0>(dst_ptr.4 as *mut _, vreinterpret_u32_u16(a4)); - vst1_lane_u16::<2>(dst_ptr.4.add(2) as *mut _, a4); - } else if CN == 2 { - vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); - vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); - vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); - vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); - vst1_lane_u32::<0>(dst_ptr.4 as *mut _, vreinterpret_u32_u16(a4)); - } else { - vst1_lane_u16::<0>(dst_ptr.0 as *mut _, a0); - vst1_lane_u16::<0>(dst_ptr.1 as *mut _, a1); - vst1_lane_u16::<0>(dst_ptr.2 as *mut _, a2); - vst1_lane_u16::<0>(dst_ptr.3 as *mut _, a3); - vst1_lane_u16::<0>(dst_ptr.4 as *mut _, a4); + unsafe { + let a0 = vqmovun_s32(regi.0); + let a1 = vqmovun_s32(regi.1); + let a2 = vqmovun_s32(regi.2); + let a3 = vqmovun_s32(regi.3); + let a4 = vqmovun_s32(add); + if CN == 4 { + vst1_u16(dst_ptr.0, a0); + vst1_u16(dst_ptr.1, a1); + vst1_u16(dst_ptr.2, a2); + vst1_u16(dst_ptr.3, a3); + vst1_u16(dst_ptr.4, a4); + } else if CN == 3 { + vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); + vst1_lane_u16::<2>(dst_ptr.0.add(2) as *mut _, a0); + vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); + vst1_lane_u16::<2>(dst_ptr.1.add(2) as *mut _, a1); + vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); + vst1_lane_u16::<2>(dst_ptr.2.add(2) as *mut _, a2); + vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); + vst1_lane_u16::<2>(dst_ptr.3.add(2) as *mut _, a3); + vst1_lane_u32::<0>(dst_ptr.4 as *mut _, vreinterpret_u32_u16(a4)); + vst1_lane_u16::<2>(dst_ptr.4.add(2) as *mut _, a4); + } else if CN == 2 { + vst1_lane_u32::<0>(dst_ptr.0 as *mut _, vreinterpret_u32_u16(a0)); + vst1_lane_u32::<0>(dst_ptr.1 as *mut _, vreinterpret_u32_u16(a1)); + vst1_lane_u32::<0>(dst_ptr.2 as *mut _, vreinterpret_u32_u16(a2)); + vst1_lane_u32::<0>(dst_ptr.3 as *mut _, vreinterpret_u32_u16(a3)); + vst1_lane_u32::<0>(dst_ptr.4 as *mut _, vreinterpret_u32_u16(a4)); + } else { + vst1_lane_u16::<0>(dst_ptr.0 as *mut _, a0); + vst1_lane_u16::<0>(dst_ptr.1 as *mut _, a1); + vst1_lane_u16::<0>(dst_ptr.2 as *mut _, a2); + vst1_lane_u16::<0>(dst_ptr.3 as *mut _, a3); + vst1_lane_u16::<0>(dst_ptr.4 as *mut _, a4); + } } } diff --git a/src/sobel.rs b/src/sobel.rs index 321475a0..8bfba3b9 100644 --- a/src/sobel.rs +++ b/src/sobel.rs @@ -27,8 +27,8 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::{ - filter_1d_exact, BlurError, BlurImage, BlurImageMut, EdgeMode2D, FastBlurChannels, Scalar, - ThreadingPolicy, + BlurError, BlurImage, BlurImageMut, EdgeMode2D, FastBlurChannels, Scalar, ThreadingPolicy, + filter_1d_exact, }; /// Performs sobel operator on the image diff --git a/src/sse/fast_gaussian.rs b/src/sse/fast_gaussian.rs index b9bd0b83..0f778d45 100644 --- a/src/sse/fast_gaussian.rs +++ b/src/sse/fast_gaussian.rs @@ -25,11 +25,11 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::sse::store_u8_u32; use crate::sse::utils::load_u8_s32_fast; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] @@ -39,8 +39,8 @@ use std::arch::x86_64::*; #[derive(Copy, Clone, Default)] pub(crate) struct SseI32x4(pub(crate) [i32; 4]); -pub(crate) fn fg_horizontal_pass_sse_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_horizontal_pass_sse_u8( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -50,15 +50,14 @@ pub(crate) fn fg_horizontal_pass_sse_u8( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); fg_horizontal_pass_sse_u8_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } #[target_feature(enable = "sse4.1")] -unsafe fn fg_horizontal_pass_sse_u8_def( +fn fg_horizontal_pass_sse_u8_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -74,7 +73,7 @@ unsafe fn fg_horizontal_pass_sse_u8_def( } #[inline(always)] -unsafe fn fg_horizontal_pass_sse_u8_impl( +fn fg_horizontal_pass_sse_u8_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -84,227 +83,237 @@ unsafe fn fg_horizontal_pass_sse_u8_impl( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([SseI32x4::default(); 1024 * 4]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, rem) = rem.split_at_mut(1024); - let (buffer2, buffer3) = rem.split_at_mut(1024); - - let initial_sum = ((radius * radius) >> 1) as i32; - - let radius_64 = radius as i64; - let width_wide = width as i64; - - let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let mut yy = start; - - while yy + 4 < height.min(end) { - let mut diffs0 = _mm_setzero_si128(); - let mut diffs1 = _mm_setzero_si128(); - let mut diffs2 = _mm_setzero_si128(); - let mut diffs3 = _mm_setzero_si128(); - - let mut summs0 = _mm_set1_epi32(initial_sum); - let mut summs1 = _mm_set1_epi32(initial_sum); - let mut summs2 = _mm_set1_epi32(initial_sum); - let mut summs3 = _mm_set1_epi32(initial_sum); - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; - let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; - let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; - - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = (x as u32 * CN as u32) as usize; - - let ss0 = _mm_cvtepi32_ps(summs0); - let ss1 = _mm_cvtepi32_ps(summs1); - let ss2 = _mm_cvtepi32_ps(summs2); - let ss3 = _mm_cvtepi32_ps(summs3); - - let r0 = _mm_mul_ps(ss0, v_weight); - let r1 = _mm_mul_ps(ss1, v_weight); - let r2 = _mm_mul_ps(ss2, v_weight); - let r3 = _mm_mul_ps(ss3, v_weight); - - let prepared_px0 = _mm_cvtps_epi32(r0); - let prepared_px1 = _mm_cvtps_epi32(r1); - let prepared_px2 = _mm_cvtps_epi32(r2); - let prepared_px3 = _mm_cvtps_epi32(r3); - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y0 + current_px); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y1 + current_px); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y2 + current_px); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y3 + current_px); - - store_u8_u32::(dst_ptr0, prepared_px0); - store_u8_u32::(dst_ptr1, prepared_px1); - store_u8_u32::(dst_ptr2, prepared_px2); - store_u8_u32::(dst_ptr3, prepared_px3); - - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let mut d_stored0 = _mm_load_si128( - buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + unsafe { + let mut full_buffer = [SseI32x4::default(); 1024 * 4]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, rem) = rem.split_at_mut(1024); + let (buffer2, buffer3) = rem.split_at_mut(1024); + + let initial_sum = ((radius * radius) >> 1) as i32; + + let radius_64 = radius as i64; + let width_wide = width as i64; + + let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let mut yy = start; + + while yy + 4 <= height.min(end) { + let mut diffs0 = _mm_setzero_si128(); + let mut diffs1 = _mm_setzero_si128(); + let mut diffs2 = _mm_setzero_si128(); + let mut diffs3 = _mm_setzero_si128(); + + let mut summs0 = _mm_set1_epi32(initial_sum); + let mut summs1 = _mm_set1_epi32(initial_sum); + let mut summs2 = _mm_set1_epi32(initial_sum); + let mut summs3 = _mm_set1_epi32(initial_sum); + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; + let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; + let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = (x as u32 * CN as u32) as usize; + + let ss0 = _mm_cvtepi32_ps(summs0); + let ss1 = _mm_cvtepi32_ps(summs1); + let ss2 = _mm_cvtepi32_ps(summs2); + let ss3 = _mm_cvtepi32_ps(summs3); + + let r0 = _mm_mul_ps(ss0, v_weight); + let r1 = _mm_mul_ps(ss1, v_weight); + let r2 = _mm_mul_ps(ss2, v_weight); + let r3 = _mm_mul_ps(ss3, v_weight); + + let prepared_px0 = _mm_cvtps_epi32(r0); + let prepared_px1 = _mm_cvtps_epi32(r1); + let prepared_px2 = _mm_cvtps_epi32(r2); + let prepared_px3 = _mm_cvtps_epi32(r3); + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y0 + current_px); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y1 + current_px); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y2 + current_px); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y3 + current_px); + + store_u8_u32::(dst_ptr0, prepared_px0); + store_u8_u32::(dst_ptr1, prepared_px1); + store_u8_u32::(dst_ptr2, prepared_px2); + store_u8_u32::(dst_ptr3, prepared_px3); + + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let mut d_stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + + d_stored0 = _mm_slli_epi32::<1>(d_stored0); + d_stored1 = _mm_slli_epi32::<1>(d_stored1); + d_stored2 = _mm_slli_epi32::<1>(d_stored2); + d_stored3 = _mm_slli_epi32::<1>(d_stored3); + + let a_stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm_add_epi32(diffs0, _mm_sub_epi32(a_stored0, d_stored0)); + diffs1 = _mm_add_epi32(diffs1, _mm_sub_epi32(a_stored1, d_stored1)); + diffs2 = _mm_add_epi32(diffs2, _mm_sub_epi32(a_stored2, d_stored2)); + diffs3 = _mm_add_epi32(diffs3, _mm_sub_epi32(a_stored3, d_stored3)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let mut stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let mut stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let mut stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let mut stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + stored0 = _mm_slli_epi32::<1>(stored0); + stored1 = _mm_slli_epi32::<1>(stored1); + stored2 = _mm_slli_epi32::<1>(stored2); + stored3 = _mm_slli_epi32::<1>(stored3); + + diffs0 = _mm_sub_epi32(diffs0, stored0); + diffs1 = _mm_sub_epi32(diffs1, stored1); + diffs2 = _mm_sub_epi32(diffs2, stored2); + diffs3 = _mm_sub_epi32(diffs3, stored3); + } + + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u8; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u8; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u8; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u8; + + let pixel_color0 = load_u8_s32_fast::(s_ptr0); + let pixel_color1 = load_u8_s32_fast::(s_ptr1); + let pixel_color2 = load_u8_s32_fast::(s_ptr2); + let pixel_color3 = load_u8_s32_fast::(s_ptr3); + + let arr_index = ((x + radius_64) & 1023) as usize; + + _mm_store_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color0, ); - let mut d_stored1 = _mm_load_si128( - buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color1, ); - let mut d_stored2 = _mm_load_si128( - buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color2, ); - let mut d_stored3 = _mm_load_si128( - buffer3.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color3, ); - d_stored0 = _mm_slli_epi32::<1>(d_stored0); - d_stored1 = _mm_slli_epi32::<1>(d_stored1); - d_stored2 = _mm_slli_epi32::<1>(d_stored2); - d_stored3 = _mm_slli_epi32::<1>(d_stored3); - - let a_stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - diffs0 = _mm_add_epi32(diffs0, _mm_sub_epi32(a_stored0, d_stored0)); - diffs1 = _mm_add_epi32(diffs1, _mm_sub_epi32(a_stored1, d_stored1)); - diffs2 = _mm_add_epi32(diffs2, _mm_sub_epi32(a_stored2, d_stored2)); - diffs3 = _mm_add_epi32(diffs3, _mm_sub_epi32(a_stored3, d_stored3)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let mut stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let mut stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let mut stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let mut stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - stored0 = _mm_slli_epi32::<1>(stored0); - stored1 = _mm_slli_epi32::<1>(stored1); - stored2 = _mm_slli_epi32::<1>(stored2); - stored3 = _mm_slli_epi32::<1>(stored3); - - diffs0 = _mm_sub_epi32(diffs0, stored0); - diffs1 = _mm_sub_epi32(diffs1, stored1); - diffs2 = _mm_sub_epi32(diffs2, stored2); - diffs3 = _mm_sub_epi32(diffs3, stored3); + diffs0 = _mm_add_epi32(diffs0, pixel_color0); + diffs1 = _mm_add_epi32(diffs1, pixel_color1); + diffs2 = _mm_add_epi32(diffs2, pixel_color2); + diffs3 = _mm_add_epi32(diffs3, pixel_color3); + + summs0 = _mm_add_epi32(summs0, diffs0); + summs1 = _mm_add_epi32(summs1, diffs1); + summs2 = _mm_add_epi32(summs2, diffs2); + summs3 = _mm_add_epi32(summs3, diffs3); } - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u8; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u8; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u8; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u8; - - let pixel_color0 = load_u8_s32_fast::(s_ptr0); - let pixel_color1 = load_u8_s32_fast::(s_ptr1); - let pixel_color2 = load_u8_s32_fast::(s_ptr2); - let pixel_color3 = load_u8_s32_fast::(s_ptr3); - - let arr_index = ((x + radius_64) & 1023) as usize; - - _mm_store_si128( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color0, - ); - _mm_store_si128( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color1, - ); - _mm_store_si128( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color2, - ); - _mm_store_si128( - buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color3, - ); - - diffs0 = _mm_add_epi32(diffs0, pixel_color0); - diffs1 = _mm_add_epi32(diffs1, pixel_color1); - diffs2 = _mm_add_epi32(diffs2, pixel_color2); - diffs3 = _mm_add_epi32(diffs3, pixel_color3); - - summs0 = _mm_add_epi32(summs0, diffs0); - summs1 = _mm_add_epi32(summs1, diffs1); - summs2 = _mm_add_epi32(summs2, diffs2); - summs3 = _mm_add_epi32(summs3, diffs3); + yy += 4; } - yy += 4; - } - - for y in yy..height.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut summs = _mm_set1_epi32(initial_sum); + for y in yy..height.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut summs = _mm_set1_epi32(initial_sum); - let current_y = ((y as i64) * (stride as i64)) as usize; + let current_y = ((y as i64) * (stride as i64)) as usize; - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = (x as u32 * CN as u32) as usize; + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = (x as u32 * CN as u32) as usize; - let pixel_f32 = _mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight); - let pixel_u32 = _mm_cvtps_epi32(pixel_f32); + let pixel_f32 = _mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight); + let pixel_u32 = _mm_cvtps_epi32(pixel_f32); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - store_u8_u32::(dst_ptr, pixel_u32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + store_u8_u32::(dst_ptr, pixel_u32); - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let d_buf_ptr = buffer0.as_mut_ptr().add(d_arr_index) as *const i32; - let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); - d_stored = _mm_slli_epi32::<1>(d_stored); + let d_buf_ptr = buffer0.as_mut_ptr().add(d_arr_index) as *const i32; + let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); + d_stored = _mm_slli_epi32::<1>(d_stored); - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let a_stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let a_stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let mut stored = _mm_load_si128(buf_ptr as *const __m128i); - stored = _mm_slli_epi32::<1>(stored); - diffs = _mm_sub_epi32(diffs, stored); - } + diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let mut stored = _mm_load_si128(buf_ptr as *const __m128i); + stored = _mm_slli_epi32::<1>(stored); + diffs = _mm_sub_epi32(diffs, stored); + } - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; - let pixel_color = load_u8_s32_fast::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; + let pixel_color = load_u8_s32_fast::(s_ptr); - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); - summs = _mm_add_epi32(summs, diffs); + diffs = _mm_add_epi32(diffs, pixel_color); + summs = _mm_add_epi32(summs, diffs); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } -pub(crate) fn fg_vertical_pass_sse_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_vertical_pass_sse_u8( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -314,15 +323,14 @@ pub(crate) fn fg_vertical_pass_sse_u8( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); fg_vertical_pass_sse_u8_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } #[target_feature(enable = "sse4.1")] -unsafe fn fg_vertical_pass_sse_u8_def( +fn fg_vertical_pass_sse_u8_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -332,226 +340,236 @@ unsafe fn fg_vertical_pass_sse_u8_def( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([SseI32x4::default(); 1024 * 4]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, rem) = rem.split_at_mut(1024); - let (buffer2, buffer3) = rem.split_at_mut(1024); - - let initial_sum = ((radius * radius) >> 1) as i32; - - let height_wide = height as i64; - - let radius_64 = radius as i64; - - let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let mut xx = start; - - while xx + 4 < width.min(end) { - let mut diffs0 = _mm_setzero_si128(); - let mut diffs1 = _mm_setzero_si128(); - let mut diffs2 = _mm_setzero_si128(); - let mut diffs3 = _mm_setzero_si128(); - - let mut summs0 = _mm_set1_epi32(initial_sum); - let mut summs1 = _mm_set1_epi32(initial_sum); - let mut summs2 = _mm_set1_epi32(initial_sum); - let mut summs3 = _mm_set1_epi32(initial_sum); - - let start_y = 0 - 2 * radius as i64; - - let current_px0 = (xx * CN as u32) as usize; - let current_px1 = ((xx + 1) * CN as u32) as usize; - let current_px2 = ((xx + 2) * CN as u32) as usize; - let current_px3 = ((xx + 3) * CN as u32) as usize; - - for y in start_y..height_wide { - if y >= 0 { - let ss0 = _mm_cvtepi32_ps(summs0); - let ss1 = _mm_cvtepi32_ps(summs1); - let ss2 = _mm_cvtepi32_ps(summs2); - let ss3 = _mm_cvtepi32_ps(summs3); - - let r0 = _mm_mul_ps(ss0, v_weight); - let r1 = _mm_mul_ps(ss1, v_weight); - let r2 = _mm_mul_ps(ss2, v_weight); - let r3 = _mm_mul_ps(ss3, v_weight); - - let prepared_px0 = _mm_cvtps_epi32(r0); - let prepared_px1 = _mm_cvtps_epi32(r1); - let prepared_px2 = _mm_cvtps_epi32(r2); - let prepared_px3 = _mm_cvtps_epi32(r3); - - let current_y = (y * (stride as i64)) as usize; - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px0); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px1); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px2); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px3); - - store_u8_u32::(dst_ptr0, prepared_px0); - store_u8_u32::(dst_ptr1, prepared_px1); - store_u8_u32::(dst_ptr2, prepared_px2); - store_u8_u32::(dst_ptr3, prepared_px3); - - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let mut d_stored0 = _mm_load_si128( - buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + unsafe { + let mut full_buffer = [SseI32x4::default(); 1024 * 4]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, rem) = rem.split_at_mut(1024); + let (buffer2, buffer3) = rem.split_at_mut(1024); + + let initial_sum = ((radius * radius) >> 1) as i32; + + let height_wide = height as i64; + + let radius_64 = radius as i64; + + let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let mut xx = start; + + while xx + 4 <= width.min(end) { + let mut diffs0 = _mm_setzero_si128(); + let mut diffs1 = _mm_setzero_si128(); + let mut diffs2 = _mm_setzero_si128(); + let mut diffs3 = _mm_setzero_si128(); + + let mut summs0 = _mm_set1_epi32(initial_sum); + let mut summs1 = _mm_set1_epi32(initial_sum); + let mut summs2 = _mm_set1_epi32(initial_sum); + let mut summs3 = _mm_set1_epi32(initial_sum); + + let start_y = 0 - 2 * radius as i64; + + let current_px0 = (xx * CN as u32) as usize; + let current_px1 = ((xx + 1) * CN as u32) as usize; + let current_px2 = ((xx + 2) * CN as u32) as usize; + let current_px3 = ((xx + 3) * CN as u32) as usize; + + for y in start_y..height_wide { + if y >= 0 { + let ss0 = _mm_cvtepi32_ps(summs0); + let ss1 = _mm_cvtepi32_ps(summs1); + let ss2 = _mm_cvtepi32_ps(summs2); + let ss3 = _mm_cvtepi32_ps(summs3); + + let r0 = _mm_mul_ps(ss0, v_weight); + let r1 = _mm_mul_ps(ss1, v_weight); + let r2 = _mm_mul_ps(ss2, v_weight); + let r3 = _mm_mul_ps(ss3, v_weight); + + let prepared_px0 = _mm_cvtps_epi32(r0); + let prepared_px1 = _mm_cvtps_epi32(r1); + let prepared_px2 = _mm_cvtps_epi32(r2); + let prepared_px3 = _mm_cvtps_epi32(r3); + + let current_y = (y * (stride as i64)) as usize; + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px0); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px1); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px2); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px3); + + store_u8_u32::(dst_ptr0, prepared_px0); + store_u8_u32::(dst_ptr1, prepared_px1); + store_u8_u32::(dst_ptr2, prepared_px2); + store_u8_u32::(dst_ptr3, prepared_px3); + + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let mut d_stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + + d_stored0 = _mm_slli_epi32::<1>(d_stored0); + d_stored1 = _mm_slli_epi32::<1>(d_stored1); + d_stored2 = _mm_slli_epi32::<1>(d_stored2); + d_stored3 = _mm_slli_epi32::<1>(d_stored3); + + let a_stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm_add_epi32(diffs0, _mm_sub_epi32(a_stored0, d_stored0)); + diffs1 = _mm_add_epi32(diffs1, _mm_sub_epi32(a_stored1, d_stored1)); + diffs2 = _mm_add_epi32(diffs2, _mm_sub_epi32(a_stored2, d_stored2)); + diffs3 = _mm_add_epi32(diffs3, _mm_sub_epi32(a_stored3, d_stored3)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + + let mut stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + let mut stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + let mut stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + let mut stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + + stored0 = _mm_slli_epi32::<1>(stored0); + stored1 = _mm_slli_epi32::<1>(stored1); + stored2 = _mm_slli_epi32::<1>(stored2); + stored3 = _mm_slli_epi32::<1>(stored3); + + diffs0 = _mm_sub_epi32(diffs0, stored0); + diffs1 = _mm_sub_epi32(diffs1, stored1); + diffs2 = _mm_sub_epi32(diffs2, stored2); + diffs3 = _mm_sub_epi32(diffs3, stored3); + } + + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u8; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u8; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u8; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u8; + + let pixel_color0 = load_u8_s32_fast::(s_ptr0); + let pixel_color1 = load_u8_s32_fast::(s_ptr1); + let pixel_color2 = load_u8_s32_fast::(s_ptr2); + let pixel_color3 = load_u8_s32_fast::(s_ptr3); + + let arr_index = ((y + radius_64) & 1023) as usize; + + diffs0 = _mm_add_epi32(diffs0, pixel_color0); + diffs1 = _mm_add_epi32(diffs1, pixel_color1); + diffs2 = _mm_add_epi32(diffs2, pixel_color2); + diffs3 = _mm_add_epi32(diffs3, pixel_color3); + + _mm_store_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color0, ); - let mut d_stored1 = _mm_load_si128( - buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color1, ); - let mut d_stored2 = _mm_load_si128( - buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color2, ); - let mut d_stored3 = _mm_load_si128( - buffer3.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color3, ); - d_stored0 = _mm_slli_epi32::<1>(d_stored0); - d_stored1 = _mm_slli_epi32::<1>(d_stored1); - d_stored2 = _mm_slli_epi32::<1>(d_stored2); - d_stored3 = _mm_slli_epi32::<1>(d_stored3); - - let a_stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - diffs0 = _mm_add_epi32(diffs0, _mm_sub_epi32(a_stored0, d_stored0)); - diffs1 = _mm_add_epi32(diffs1, _mm_sub_epi32(a_stored1, d_stored1)); - diffs2 = _mm_add_epi32(diffs2, _mm_sub_epi32(a_stored2, d_stored2)); - diffs3 = _mm_add_epi32(diffs3, _mm_sub_epi32(a_stored3, d_stored3)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - - let mut stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _); - let mut stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _); - let mut stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _); - let mut stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _); - - stored0 = _mm_slli_epi32::<1>(stored0); - stored1 = _mm_slli_epi32::<1>(stored1); - stored2 = _mm_slli_epi32::<1>(stored2); - stored3 = _mm_slli_epi32::<1>(stored3); - - diffs0 = _mm_sub_epi32(diffs0, stored0); - diffs1 = _mm_sub_epi32(diffs1, stored1); - diffs2 = _mm_sub_epi32(diffs2, stored2); - diffs3 = _mm_sub_epi32(diffs3, stored3); + summs0 = _mm_add_epi32(summs0, diffs0); + summs1 = _mm_add_epi32(summs1, diffs1); + summs2 = _mm_add_epi32(summs2, diffs2); + summs3 = _mm_add_epi32(summs3, diffs3); } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u8; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u8; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u8; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u8; - - let pixel_color0 = load_u8_s32_fast::(s_ptr0); - let pixel_color1 = load_u8_s32_fast::(s_ptr1); - let pixel_color2 = load_u8_s32_fast::(s_ptr2); - let pixel_color3 = load_u8_s32_fast::(s_ptr3); - - let arr_index = ((y + radius_64) & 1023) as usize; - - diffs0 = _mm_add_epi32(diffs0, pixel_color0); - diffs1 = _mm_add_epi32(diffs1, pixel_color1); - diffs2 = _mm_add_epi32(diffs2, pixel_color2); - diffs3 = _mm_add_epi32(diffs3, pixel_color3); - - _mm_store_si128( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color0, - ); - _mm_store_si128( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color1, - ); - _mm_store_si128( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color2, - ); - _mm_store_si128( - buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color3, - ); - - summs0 = _mm_add_epi32(summs0, diffs0); - summs1 = _mm_add_epi32(summs1, diffs1); - summs2 = _mm_add_epi32(summs2, diffs2); - summs3 = _mm_add_epi32(summs3, diffs3); + xx += 4; } - xx += 4; - } - - for x in xx..width.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut summs = _mm_set1_epi32(initial_sum); + for x in xx..width.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut summs = _mm_set1_epi32(initial_sum); - let current_px = (x * CN as u32) as usize; + let current_px = (x * CN as u32) as usize; - let start_y = 0 - 2 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let pixel_f32 = _mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight); + let start_y = 0 - 2 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let pixel_f32 = _mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight); - let current_y = (y * (stride as i64)) as usize; + let current_y = (y * (stride as i64)) as usize; - let pixel_u32 = _mm_cvtps_epi32(pixel_f32); + let pixel_u32 = _mm_cvtps_epi32(pixel_f32); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - store_u8_u32::(dst_ptr, pixel_u32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + store_u8_u32::(dst_ptr, pixel_u32); - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let d_buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); - d_stored = _mm_slli_epi32::<1>(d_stored); + let d_buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); + d_stored = _mm_slli_epi32::<1>(d_stored); - let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; - let a_stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; + let a_stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let mut stored = _mm_load_si128(buf_ptr as *const __m128i); - stored = _mm_slli_epi32::<1>(stored); - diffs = _mm_sub_epi32(diffs, stored); - } + diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let mut stored = _mm_load_si128(buf_ptr as *const __m128i); + stored = _mm_slli_epi32::<1>(stored); + diffs = _mm_sub_epi32(diffs, stored); + } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; - let pixel_color = load_u8_s32_fast::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; + let pixel_color = load_u8_s32_fast::(s_ptr); - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); + diffs = _mm_add_epi32(diffs, pixel_color); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); - summs = _mm_add_epi32(summs, diffs); + summs = _mm_add_epi32(summs, diffs); + } } } } diff --git a/src/sse/fast_gaussian_f16.rs b/src/sse/fast_gaussian_f16.rs index c4c705f9..9542fa89 100644 --- a/src/sse/fast_gaussian_f16.rs +++ b/src/sse/fast_gaussian_f16.rs @@ -30,14 +30,14 @@ use std::arch::x86::*; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::sse::{load_f32_f16, store_f32_f16}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use core::f16; -pub(crate) fn fg_horizontal_pass_sse_f16( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_horizontal_pass_sse_f16( + undefined_slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -47,7 +47,7 @@ pub(crate) fn fg_horizontal_pass_sse_f16( edge_mode: EdgeMode, ) { unsafe { - fast_gaussian_horizontal_pass_sse_f16_impl::( + fast_gaussian_horizontal_pass_sse_f16_impl::( undefined_slice, stride, width, @@ -62,8 +62,8 @@ pub(crate) fn fg_horizontal_pass_sse_f16( #[inline] #[target_feature(enable = "sse4.1,f16c")] -unsafe fn fast_gaussian_horizontal_pass_sse_f16_impl( - undefined_slice: &UnsafeSlice, +fn fast_gaussian_horizontal_pass_sse_f16_impl( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -72,70 +72,71 @@ unsafe fn fast_gaussian_horizontal_pass_sse_f16_impl( end: u32, edge_mode: EdgeMode, ) { - let bytes: &UnsafeSlice<'_, f16> = std::mem::transmute(undefined_slice); - let mut buffer: [[f32; 4]; 1024] = [[0.; 4]; 1024]; + unsafe { + let mut buffer: [[f32; 4]; 1024] = [[0.; 4]; 1024]; - let radius_64 = radius as i64; - let width_wide = width as i64; + let radius_64 = radius as i64; + let width_wide = width as i64; - let v_half = _mm_set1_ps(2.); - let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); + let v_half = _mm_set1_ps(2.); + let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); - for y in start..std::cmp::min(height, end) { - let mut diffs = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); + for y in start..std::cmp::min(height, end) { + let mut diffs = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); - let current_y = ((y as i64) * (stride as i64)) as usize; + let current_y = ((y as i64) * (stride as i64)) as usize; - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; - let pixel = _mm_mul_ps(summs, v_weight); + let pixel = _mm_mul_ps(summs, v_weight); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f16; - store_f32_f16::(dst_ptr, pixel); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f16; + store_f32_f16::(dst_ptr, pixel); - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let d_buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let mut d_stored = _mm_loadu_ps(d_buf_ptr); - d_stored = _mm_mul_ps(d_stored, v_half); + let d_buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let mut d_stored = _mm_loadu_ps(d_buf_ptr); + d_stored = _mm_mul_ps(d_stored, v_half); - let buf_ptr = buffer.as_mut_ptr().add(arr_index) as *mut f32; - let a_stored = _mm_loadu_ps(buf_ptr); + let buf_ptr = buffer.as_mut_ptr().add(arr_index) as *mut f32; + let a_stored = _mm_loadu_ps(buf_ptr); - diffs = _mm_add_ps(diffs, _mm_sub_ps(a_stored, d_stored)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - let mut stored = _mm_loadu_ps(buf_ptr); - stored = _mm_mul_ps(stored, v_half); - diffs = _mm_sub_ps(diffs, stored); - } + diffs = _mm_add_ps(diffs, _mm_sub_ps(a_stored, d_stored)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let mut stored = _mm_loadu_ps(buf_ptr); + stored = _mm_mul_ps(stored, v_half); + diffs = _mm_sub_ps(diffs, stored); + } - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f16; - let pixel_color = load_f32_f16::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f16; + let pixel_color = load_f32_f16::(s_ptr); - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - diffs = _mm_add_ps(diffs, pixel_color); - summs = _mm_add_ps(summs, diffs); - _mm_storeu_ps(buf_ptr, pixel_color); + diffs = _mm_add_ps(diffs, pixel_color); + summs = _mm_add_ps(summs, diffs); + _mm_storeu_ps(buf_ptr, pixel_color); + } } } } -pub(crate) fn fg_vertical_pass_sse_f16( +pub(crate) fn fg_vertical_pass_sse_f16( undefined_slice: &UnsafeSlice, stride: u32, width: u32, @@ -146,7 +147,7 @@ pub(crate) fn fg_vertical_pass_sse_f16( edge_mode: EdgeMode, ) { unsafe { - fast_gaussian_vertical_pass_sse_f16_impl::( + fast_gaussian_vertical_pass_sse_f16_impl::( undefined_slice, stride, width, @@ -161,7 +162,7 @@ pub(crate) fn fg_vertical_pass_sse_f16( #[inline] #[target_feature(enable = "sse4.1,f16c")] -unsafe fn fast_gaussian_vertical_pass_sse_f16_impl( +fn fast_gaussian_vertical_pass_sse_f16_impl( undefined_slice: &UnsafeSlice, stride: u32, width: u32, @@ -171,65 +172,67 @@ unsafe fn fast_gaussian_vertical_pass_sse_f16_impl = std::mem::transmute(undefined_slice); - let mut buffer: [[f32; 4]; 1024] = [[0.; 4]; 1024]; + unsafe { + let bytes: &UnsafeSlice<'_, f16> = std::mem::transmute(undefined_slice); + let mut buffer: [[f32; 4]; 1024] = [[0.; 4]; 1024]; - let v_half = _mm_set1_ps(2.); - let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); + let v_half = _mm_set1_ps(2.); + let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); - let height_wide = height as i64; + let height_wide = height as i64; - let radius_64 = radius as i64; - for x in start..std::cmp::min(width, end) { - let mut diffs = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); + let radius_64 = radius as i64; + for x in start..std::cmp::min(width, end) { + let mut diffs = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); - let start_y = 0 - 2 * radius as i64; - for y in start_y..height_wide { - let current_y = (y * (stride as i64)) as usize; + let start_y = 0 - 2 * radius as i64; + for y in start_y..height_wide { + let current_y = (y * (stride as i64)) as usize; - if y >= 0 { - let current_px = ((std::cmp::max(x, 0)) * CHANNELS_COUNT as u32) as usize; + if y >= 0 { + let current_px = ((std::cmp::max(x, 0)) * CN as u32) as usize; - let pixel = _mm_mul_ps(summs, v_weight); + let pixel = _mm_mul_ps(summs, v_weight); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f16; - store_f32_f16::(dst_ptr, pixel); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f16; + store_f32_f16::(dst_ptr, pixel); - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let d_buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let mut d_stored = _mm_loadu_ps(d_buf_ptr); - d_stored = _mm_mul_ps(d_stored, v_half); + let d_buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let mut d_stored = _mm_loadu_ps(d_buf_ptr); + d_stored = _mm_mul_ps(d_stored, v_half); - let buf_ptr = buffer.as_mut_ptr().add(arr_index) as *mut f32; - let a_stored = _mm_loadu_ps(buf_ptr); + let buf_ptr = buffer.as_mut_ptr().add(arr_index) as *mut f32; + let a_stored = _mm_loadu_ps(buf_ptr); - diffs = _mm_add_ps(diffs, _mm_sub_ps(a_stored, d_stored)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - let mut stored = _mm_loadu_ps(buf_ptr); - stored = _mm_mul_ps(stored, v_half); - diffs = _mm_sub_ps(diffs, stored); - } + diffs = _mm_add_ps(diffs, _mm_sub_ps(a_stored, d_stored)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let mut stored = _mm_loadu_ps(buf_ptr); + stored = _mm_mul_ps(stored, v_half); + diffs = _mm_sub_ps(diffs, stored); + } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - let next_row_x = (x * CHANNELS_COUNT as u32) as usize; + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f16; - let pixel_color = load_f32_f16::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f16; + let pixel_color = load_f32_f16::(s_ptr); - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - diffs = _mm_add_ps(diffs, pixel_color); - summs = _mm_add_ps(summs, diffs); - _mm_storeu_ps(buf_ptr, pixel_color); + diffs = _mm_add_ps(diffs, pixel_color); + summs = _mm_add_ps(summs, diffs); + _mm_storeu_ps(buf_ptr, pixel_color); + } } } } diff --git a/src/sse/fast_gaussian_f32.rs b/src/sse/fast_gaussian_f32.rs index 20fb269b..e9008060 100644 --- a/src/sse/fast_gaussian_f32.rs +++ b/src/sse/fast_gaussian_f32.rs @@ -25,17 +25,17 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::sse::{_mm_opt_fnmlaf_ps, load_f32, store_f32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; -pub(crate) fn fg_horizontal_pass_sse_f32( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_horizontal_pass_sse_f32( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -45,21 +45,20 @@ pub(crate) fn fg_horizontal_pass_sse_f32( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undefined_slice); if std::arch::is_x86_feature_detected!("fma") { fg_horizontal_pass_sse_f32_fma::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } else { fg_horizontal_pass_sse_f32_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } } #[target_feature(enable = "sse4.1")] -unsafe fn fg_horizontal_pass_sse_f32_def( +fn fg_horizontal_pass_sse_f32_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -75,7 +74,7 @@ unsafe fn fg_horizontal_pass_sse_f32_def( } #[target_feature(enable = "sse4.1", enable = "fma")] -unsafe fn fg_horizontal_pass_sse_f32_fma( +fn fg_horizontal_pass_sse_f32_fma( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -91,7 +90,7 @@ unsafe fn fg_horizontal_pass_sse_f32_fma( } #[inline(always)] -unsafe fn fg_horizontal_pass_sse_f32_impl( +fn fg_horizontal_pass_sse_f32_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -101,176 +100,178 @@ unsafe fn fg_horizontal_pass_sse_f32_impl( end: u32, edge_mode: EdgeMode, ) { - let mut bf0 = Box::new([[0f32; 4]; 1024]); - let mut bf1 = Box::new([[0f32; 4]; 1024]); - let mut bf2 = Box::new([[0f32; 4]; 1024]); - let mut bf3 = Box::new([[0f32; 4]; 1024]); - - let radius_64 = radius as i64; - let width_wide = width as i64; - - let v_double = _mm_set1_ps(2.); - let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let mut yy = start as usize; - - while yy + 4 < height.min(end) as usize { - let mut diffs0 = _mm_setzero_ps(); - let mut diffs1 = _mm_setzero_ps(); - let mut diffs2 = _mm_setzero_ps(); - let mut diffs3 = _mm_setzero_ps(); - - let mut sums0 = _mm_setzero_ps(); - let mut sums1 = _mm_setzero_ps(); - let mut sums2 = _mm_setzero_ps(); - let mut sums3 = _mm_setzero_ps(); - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; - let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; - let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; - - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; - - let prepared_px0 = _mm_mul_ps(sums0, v_weight); - let prepared_px1 = _mm_mul_ps(sums1, v_weight); - let prepared_px2 = _mm_mul_ps(sums2, v_weight); - let prepared_px3 = _mm_mul_ps(sums3, v_weight); - - let dst_ptr0 = bytes.slice.as_ptr().add(current_y0 + current_px) as *mut f32; - let dst_ptr1 = bytes.slice.as_ptr().add(current_y1 + current_px) as *mut f32; - let dst_ptr2 = bytes.slice.as_ptr().add(current_y2 + current_px) as *mut f32; - let dst_ptr3 = bytes.slice.as_ptr().add(current_y3 + current_px) as *mut f32; - - store_f32::(dst_ptr0, prepared_px0); - store_f32::(dst_ptr1, prepared_px1); - store_f32::(dst_ptr2, prepared_px2); - store_f32::(dst_ptr3, prepared_px3); - - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let d_s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_arr_index) as *mut f32); - let d_s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_arr_index) as *mut f32); - let d_s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_arr_index) as *mut f32); - let d_s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(d_arr_index) as *mut f32); - - let a_s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let a_s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let a_s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - let a_s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = _mm_add_ps(diffs0, _mm_opt_fnmlaf_ps::(a_s0, d_s0, v_double)); - diffs1 = _mm_add_ps(diffs1, _mm_opt_fnmlaf_ps::(a_s1, d_s1, v_double)); - diffs2 = _mm_add_ps(diffs2, _mm_opt_fnmlaf_ps::(a_s2, d_s2, v_double)); - diffs3 = _mm_add_ps(diffs3, _mm_opt_fnmlaf_ps::(a_s3, d_s3, v_double)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - let s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = _mm_opt_fnmlaf_ps::(diffs0, s0, v_double); - diffs1 = _mm_opt_fnmlaf_ps::(diffs1, s1, v_double); - diffs2 = _mm_opt_fnmlaf_ps::(diffs2, s2, v_double); - diffs3 = _mm_opt_fnmlaf_ps::(diffs3, s3, v_double); + unsafe { + let mut bf0 = [[0f32; 4]; 1024]; + let mut bf1 = [[0f32; 4]; 1024]; + let mut bf2 = [[0f32; 4]; 1024]; + let mut bf3 = [[0f32; 4]; 1024]; + + let radius_64 = radius as i64; + let width_wide = width as i64; + + let v_double = _mm_set1_ps(2.); + let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let mut yy = start as usize; + + while yy + 4 <= height.min(end) as usize { + let mut diffs0 = _mm_setzero_ps(); + let mut diffs1 = _mm_setzero_ps(); + let mut diffs2 = _mm_setzero_ps(); + let mut diffs3 = _mm_setzero_ps(); + + let mut sums0 = _mm_setzero_ps(); + let mut sums1 = _mm_setzero_ps(); + let mut sums2 = _mm_setzero_ps(); + let mut sums3 = _mm_setzero_ps(); + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; + let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; + let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; + + let prepared_px0 = _mm_mul_ps(sums0, v_weight); + let prepared_px1 = _mm_mul_ps(sums1, v_weight); + let prepared_px2 = _mm_mul_ps(sums2, v_weight); + let prepared_px3 = _mm_mul_ps(sums3, v_weight); + + let dst_ptr0 = bytes.slice.as_ptr().add(current_y0 + current_px) as *mut f32; + let dst_ptr1 = bytes.slice.as_ptr().add(current_y1 + current_px) as *mut f32; + let dst_ptr2 = bytes.slice.as_ptr().add(current_y2 + current_px) as *mut f32; + let dst_ptr3 = bytes.slice.as_ptr().add(current_y3 + current_px) as *mut f32; + + store_f32::(dst_ptr0, prepared_px0); + store_f32::(dst_ptr1, prepared_px1); + store_f32::(dst_ptr2, prepared_px2); + store_f32::(dst_ptr3, prepared_px3); + + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let d_s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_arr_index) as *mut f32); + let d_s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_arr_index) as *mut f32); + let d_s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_arr_index) as *mut f32); + let d_s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(d_arr_index) as *mut f32); + + let a_s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let a_s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let a_s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + let a_s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm_add_ps(diffs0, _mm_opt_fnmlaf_ps::(a_s0, d_s0, v_double)); + diffs1 = _mm_add_ps(diffs1, _mm_opt_fnmlaf_ps::(a_s1, d_s1, v_double)); + diffs2 = _mm_add_ps(diffs2, _mm_opt_fnmlaf_ps::(a_s2, d_s2, v_double)); + diffs3 = _mm_add_ps(diffs3, _mm_opt_fnmlaf_ps::(a_s3, d_s3, v_double)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + let s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm_opt_fnmlaf_ps::(diffs0, s0, v_double); + diffs1 = _mm_opt_fnmlaf_ps::(diffs1, s1, v_double); + diffs2 = _mm_opt_fnmlaf_ps::(diffs2, s2, v_double); + diffs3 = _mm_opt_fnmlaf_ps::(diffs3, s3, v_double); + } + + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut f32; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut f32; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut f32; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut f32; + + let px0 = load_f32::(s_ptr0); + let px1 = load_f32::(s_ptr1); + let px2 = load_f32::(s_ptr2); + let px3 = load_f32::(s_ptr3); + + let arr_index = ((x + radius_64) & 1023) as usize; + + _mm_storeu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32, px0); + _mm_storeu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32, px1); + _mm_storeu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32, px2); + _mm_storeu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32, px3); + + diffs0 = _mm_add_ps(diffs0, px0); + diffs1 = _mm_add_ps(diffs1, px1); + diffs2 = _mm_add_ps(diffs2, px2); + diffs3 = _mm_add_ps(diffs3, px3); + + sums0 = _mm_add_ps(sums0, diffs0); + sums1 = _mm_add_ps(sums1, diffs1); + sums2 = _mm_add_ps(sums2, diffs2); + sums3 = _mm_add_ps(sums3, diffs3); } - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut f32; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut f32; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut f32; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut f32; - - let px0 = load_f32::(s_ptr0); - let px1 = load_f32::(s_ptr1); - let px2 = load_f32::(s_ptr2); - let px3 = load_f32::(s_ptr3); - - let arr_index = ((x + radius_64) & 1023) as usize; - - _mm_storeu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32, px0); - _mm_storeu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32, px1); - _mm_storeu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32, px2); - _mm_storeu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32, px3); - - diffs0 = _mm_add_ps(diffs0, px0); - diffs1 = _mm_add_ps(diffs1, px1); - diffs2 = _mm_add_ps(diffs2, px2); - diffs3 = _mm_add_ps(diffs3, px3); - - sums0 = _mm_add_ps(sums0, diffs0); - sums1 = _mm_add_ps(sums1, diffs1); - sums2 = _mm_add_ps(sums2, diffs2); - sums3 = _mm_add_ps(sums3, diffs3); + yy += 4; } - yy += 4; - } - - for y in yy..height.min(end) as usize { - let mut diffs = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); + for y in yy..height.min(end) as usize { + let mut diffs = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); - let current_y = ((y as i64) * (stride as i64)) as usize; + let current_y = ((y as i64) * (stride as i64)) as usize; - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; - let pixel = _mm_mul_ps(summs, v_weight); + let pixel = _mm_mul_ps(summs, v_weight); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; - store_f32::(dst_ptr, pixel); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; + store_f32::(dst_ptr, pixel); - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let d_buf_ptr = bf0.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let d_stored = _mm_loadu_ps(d_buf_ptr); + let d_buf_ptr = bf0.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let d_stored = _mm_loadu_ps(d_buf_ptr); - let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; - let a_stored = _mm_loadu_ps(buf_ptr); + let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; + let a_stored = _mm_loadu_ps(buf_ptr); - diffs = _mm_add_ps( - diffs, - _mm_opt_fnmlaf_ps::(a_stored, d_stored, v_double), - ); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); - diffs = _mm_opt_fnmlaf_ps::(diffs, stored, v_double); - } + diffs = _mm_add_ps( + diffs, + _mm_opt_fnmlaf_ps::(a_stored, d_stored, v_double), + ); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); + diffs = _mm_opt_fnmlaf_ps::(diffs, stored, v_double); + } - let next_row_y = y * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_y = y * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f32; - let pixel_color = load_f32::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f32; + let pixel_color = load_f32::(s_ptr); - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); - diffs = _mm_add_ps(diffs, pixel_color); - summs = _mm_add_ps(summs, diffs); - _mm_storeu_ps(buf_ptr, pixel_color); + diffs = _mm_add_ps(diffs, pixel_color); + summs = _mm_add_ps(summs, diffs); + _mm_storeu_ps(buf_ptr, pixel_color); + } } } } -pub(crate) fn fg_vertical_pass_sse_f32( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_vertical_pass_sse_f32( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -280,22 +281,20 @@ pub(crate) fn fg_vertical_pass_sse_f32( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undefined_slice); - if std::arch::is_x86_feature_detected!("fma") { fg_vertical_pass_sse_f32_fma::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } else { fg_vertical_pass_sse_f32_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } } #[target_feature(enable = "sse4.1")] -unsafe fn fg_vertical_pass_sse_f32_def( +fn fg_vertical_pass_sse_f32_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -311,7 +310,7 @@ unsafe fn fg_vertical_pass_sse_f32_def( } #[target_feature(enable = "sse4.1", enable = "fma")] -unsafe fn fg_vertical_pass_sse_f32_fma( +fn fg_vertical_pass_sse_f32_fma( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -327,7 +326,7 @@ unsafe fn fg_vertical_pass_sse_f32_fma( } #[inline(always)] -unsafe fn fg_vertical_pass_sse_f32_impl( +fn fg_vertical_pass_sse_f32_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -337,172 +336,174 @@ unsafe fn fg_vertical_pass_sse_f32_impl( end: u32, edge_mode: EdgeMode, ) { - let mut bf0 = Box::new([[0f32; 4]; 1024]); - let mut bf1 = Box::new([[0f32; 4]; 1024]); - let mut bf2 = Box::new([[0f32; 4]; 1024]); - let mut bf3 = Box::new([[0f32; 4]; 1024]); - - let v_double = _mm_set1_ps(2.); - let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let height_wide = height as i64; - - let radius_64 = radius as i64; - - let mut xx = start as usize; - - while xx + 4 < width.min(end) as usize { - let mut diffs0 = _mm_setzero_ps(); - let mut diffs1 = _mm_setzero_ps(); - let mut diffs2 = _mm_setzero_ps(); - let mut diffs3 = _mm_setzero_ps(); - - let mut sums0 = _mm_setzero_ps(); - let mut sums1 = _mm_setzero_ps(); - let mut sums2 = _mm_setzero_ps(); - let mut sums3 = _mm_setzero_ps(); - - let current_px0 = xx * CN; - let current_px1 = (xx + 1) * CN; - let current_px2 = (xx + 2) * CN; - let current_px3 = (xx + 3) * CN; - - let start_y = 0 - 2 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_y = (y * (stride as i64)) as usize; - - let prepared_px0 = _mm_mul_ps(sums0, v_weight); - let prepared_px1 = _mm_mul_ps(sums1, v_weight); - let prepared_px2 = _mm_mul_ps(sums2, v_weight); - let prepared_px3 = _mm_mul_ps(sums3, v_weight); - - let dst_ptr0 = bytes.slice.as_ptr().add(current_y + current_px0) as *mut f32; - let dst_ptr1 = bytes.slice.as_ptr().add(current_y + current_px1) as *mut f32; - let dst_ptr2 = bytes.slice.as_ptr().add(current_y + current_px2) as *mut f32; - let dst_ptr3 = bytes.slice.as_ptr().add(current_y + current_px3) as *mut f32; - - store_f32::(dst_ptr0, prepared_px0); - store_f32::(dst_ptr1, prepared_px1); - store_f32::(dst_ptr2, prepared_px2); - store_f32::(dst_ptr3, prepared_px3); - - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let d_s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_arr_index) as *mut f32); - let d_s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_arr_index) as *mut f32); - let d_s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_arr_index) as *mut f32); - let d_s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(d_arr_index) as *mut f32); - - let a_s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let a_s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let a_s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - let a_s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = _mm_add_ps(diffs0, _mm_opt_fnmlaf_ps::(a_s0, d_s0, v_double)); - diffs1 = _mm_add_ps(diffs1, _mm_opt_fnmlaf_ps::(a_s1, d_s1, v_double)); - diffs2 = _mm_add_ps(diffs2, _mm_opt_fnmlaf_ps::(a_s2, d_s2, v_double)); - diffs3 = _mm_add_ps(diffs3, _mm_opt_fnmlaf_ps::(a_s3, d_s3, v_double)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - let s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = _mm_opt_fnmlaf_ps::(diffs0, s0, v_double); - diffs1 = _mm_opt_fnmlaf_ps::(diffs1, s1, v_double); - diffs2 = _mm_opt_fnmlaf_ps::(diffs2, s2, v_double); - diffs3 = _mm_opt_fnmlaf_ps::(diffs3, s3, v_double); + unsafe { + let mut bf0 = [[0f32; 4]; 1024]; + let mut bf1 = [[0f32; 4]; 1024]; + let mut bf2 = [[0f32; 4]; 1024]; + let mut bf3 = [[0f32; 4]; 1024]; + + let v_double = _mm_set1_ps(2.); + let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let height_wide = height as i64; + + let radius_64 = radius as i64; + + let mut xx = start as usize; + + while xx + 4 <= width.min(end) as usize { + let mut diffs0 = _mm_setzero_ps(); + let mut diffs1 = _mm_setzero_ps(); + let mut diffs2 = _mm_setzero_ps(); + let mut diffs3 = _mm_setzero_ps(); + + let mut sums0 = _mm_setzero_ps(); + let mut sums1 = _mm_setzero_ps(); + let mut sums2 = _mm_setzero_ps(); + let mut sums3 = _mm_setzero_ps(); + + let current_px0 = xx * CN; + let current_px1 = (xx + 1) * CN; + let current_px2 = (xx + 2) * CN; + let current_px3 = (xx + 3) * CN; + + let start_y = 0 - 2 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_y = (y * (stride as i64)) as usize; + + let prepared_px0 = _mm_mul_ps(sums0, v_weight); + let prepared_px1 = _mm_mul_ps(sums1, v_weight); + let prepared_px2 = _mm_mul_ps(sums2, v_weight); + let prepared_px3 = _mm_mul_ps(sums3, v_weight); + + let dst_ptr0 = bytes.slice.as_ptr().add(current_y + current_px0) as *mut f32; + let dst_ptr1 = bytes.slice.as_ptr().add(current_y + current_px1) as *mut f32; + let dst_ptr2 = bytes.slice.as_ptr().add(current_y + current_px2) as *mut f32; + let dst_ptr3 = bytes.slice.as_ptr().add(current_y + current_px3) as *mut f32; + + store_f32::(dst_ptr0, prepared_px0); + store_f32::(dst_ptr1, prepared_px1); + store_f32::(dst_ptr2, prepared_px2); + store_f32::(dst_ptr3, prepared_px3); + + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let d_s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_arr_index) as *mut f32); + let d_s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_arr_index) as *mut f32); + let d_s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_arr_index) as *mut f32); + let d_s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(d_arr_index) as *mut f32); + + let a_s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let a_s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let a_s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + let a_s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm_add_ps(diffs0, _mm_opt_fnmlaf_ps::(a_s0, d_s0, v_double)); + diffs1 = _mm_add_ps(diffs1, _mm_opt_fnmlaf_ps::(a_s1, d_s1, v_double)); + diffs2 = _mm_add_ps(diffs2, _mm_opt_fnmlaf_ps::(a_s2, d_s2, v_double)); + diffs3 = _mm_add_ps(diffs3, _mm_opt_fnmlaf_ps::(a_s3, d_s3, v_double)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let s0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let s1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let s2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + let s3 = _mm_loadu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm_opt_fnmlaf_ps::(diffs0, s0, v_double); + diffs1 = _mm_opt_fnmlaf_ps::(diffs1, s1, v_double); + diffs2 = _mm_opt_fnmlaf_ps::(diffs2, s2, v_double); + diffs3 = _mm_opt_fnmlaf_ps::(diffs3, s3, v_double); + } + + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut f32; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut f32; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut f32; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut f32; + + let px0 = load_f32::(s_ptr0); + let px1 = load_f32::(s_ptr1); + let px2 = load_f32::(s_ptr2); + let px3 = load_f32::(s_ptr3); + + let arr_index = ((y + radius_64) & 1023) as usize; + + _mm_storeu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32, px0); + _mm_storeu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32, px1); + _mm_storeu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32, px2); + _mm_storeu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32, px3); + + diffs0 = _mm_add_ps(diffs0, px0); + diffs1 = _mm_add_ps(diffs1, px1); + diffs2 = _mm_add_ps(diffs2, px2); + diffs3 = _mm_add_ps(diffs3, px3); + + sums0 = _mm_add_ps(sums0, diffs0); + sums1 = _mm_add_ps(sums1, diffs1); + sums2 = _mm_add_ps(sums2, diffs2); + sums3 = _mm_add_ps(sums3, diffs3); } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut f32; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut f32; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut f32; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut f32; - - let px0 = load_f32::(s_ptr0); - let px1 = load_f32::(s_ptr1); - let px2 = load_f32::(s_ptr2); - let px3 = load_f32::(s_ptr3); - - let arr_index = ((y + radius_64) & 1023) as usize; - - _mm_storeu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32, px0); - _mm_storeu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32, px1); - _mm_storeu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32, px2); - _mm_storeu_ps(bf3.as_mut_ptr().add(arr_index) as *mut f32, px3); - - diffs0 = _mm_add_ps(diffs0, px0); - diffs1 = _mm_add_ps(diffs1, px1); - diffs2 = _mm_add_ps(diffs2, px2); - diffs3 = _mm_add_ps(diffs3, px3); - - sums0 = _mm_add_ps(sums0, diffs0); - sums1 = _mm_add_ps(sums1, diffs1); - sums2 = _mm_add_ps(sums2, diffs2); - sums3 = _mm_add_ps(sums3, diffs3); + xx += 4; } - xx += 4; - } - - for x in xx..width.min(end) as usize { - let mut diffs = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); + for x in xx..width.min(end) as usize { + let mut diffs = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); - let start_y = 0 - 2 * radius as i64; + let start_y = 0 - 2 * radius as i64; - let current_px = x * CN; + let current_px = x * CN; - for y in start_y..height_wide { - let current_y = (y * (stride as i64)) as usize; + for y in start_y..height_wide { + let current_y = (y * (stride as i64)) as usize; - if y >= 0 { - let pixel = _mm_mul_ps(summs, v_weight); + if y >= 0 { + let pixel = _mm_mul_ps(summs, v_weight); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; - store_f32::(dst_ptr, pixel); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; + store_f32::(dst_ptr, pixel); - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let d_buf_ptr = bf0.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let d_stored = _mm_loadu_ps(d_buf_ptr); + let d_buf_ptr = bf0.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let d_stored = _mm_loadu_ps(d_buf_ptr); - let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; - let a_stored = _mm_loadu_ps(buf_ptr); + let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; + let a_stored = _mm_loadu_ps(buf_ptr); - diffs = _mm_add_ps( - diffs, - _mm_opt_fnmlaf_ps::(a_stored, d_stored, v_double), - ); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); - diffs = _mm_opt_fnmlaf_ps::(diffs, stored, v_double); - } + diffs = _mm_add_ps( + diffs, + _mm_opt_fnmlaf_ps::(a_stored, d_stored, v_double), + ); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); + diffs = _mm_opt_fnmlaf_ps::(diffs, stored, v_double); + } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - let next_row_x = x * CN; + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + let next_row_x = x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f32; - let pixel_color = load_f32::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f32; + let pixel_color = load_f32::(s_ptr); - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); - diffs = _mm_add_ps(diffs, pixel_color); - summs = _mm_add_ps(summs, diffs); - _mm_storeu_ps(buf_ptr, pixel_color); + diffs = _mm_add_ps(diffs, pixel_color); + summs = _mm_add_ps(summs, diffs); + _mm_storeu_ps(buf_ptr, pixel_color); + } } } } diff --git a/src/sse/fast_gaussian_next.rs b/src/sse/fast_gaussian_next.rs index 2609a360..d6f1a248 100644 --- a/src/sse/fast_gaussian_next.rs +++ b/src/sse/fast_gaussian_next.rs @@ -25,19 +25,19 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::sse::fast_gaussian::SseI32x4; use crate::sse::utils::load_u8_s32_fast; use crate::sse::{_mm_mul_by_3_epi32, store_u8_u32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; -pub(crate) fn fgn_vertical_pass_sse_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fgn_vertical_pass_sse_u8( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -47,16 +47,14 @@ pub(crate) fn fgn_vertical_pass_sse_u8( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - fgn_vertical_pass_sse_u8_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } #[target_feature(enable = "sse4.1")] -unsafe fn fgn_vertical_pass_sse_u8_def( +fn fgn_vertical_pass_sse_u8_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -72,7 +70,7 @@ unsafe fn fgn_vertical_pass_sse_u8_def( } #[inline(always)] -unsafe fn fgn_vertical_pass_sse_u8_impl( +fn fgn_vertical_pass_sse_u8_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -82,276 +80,295 @@ unsafe fn fgn_vertical_pass_sse_u8_impl( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([SseI32x4::default(); 1024 * 4]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, rem) = rem.split_at_mut(1024); - let (buffer2, buffer3) = rem.split_at_mut(1024); - - let height_wide = height as i64; - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm_set1_ps(weight); - - let mut xx = start; - - while xx + 4 < width.min(end) { - let mut diffs0 = _mm_setzero_si128(); - let mut diffs1 = _mm_setzero_si128(); - let mut diffs2 = _mm_setzero_si128(); - let mut diffs3 = _mm_setzero_si128(); - - let mut ders0 = _mm_setzero_si128(); - let mut ders1 = _mm_setzero_si128(); - let mut ders2 = _mm_setzero_si128(); - let mut ders3 = _mm_setzero_si128(); - - let mut summs0 = _mm_setzero_si128(); - let mut summs1 = _mm_setzero_si128(); - let mut summs2 = _mm_setzero_si128(); - let mut summs3 = _mm_setzero_si128(); - - let start_y = 0 - 3 * radius as i64; - - let current_px0 = (xx * CN as u32) as usize; - let current_px1 = ((xx + 1) * CN as u32) as usize; - let current_px2 = ((xx + 2) * CN as u32) as usize; - let current_px3 = ((xx + 3) * CN as u32) as usize; - - for y in start_y..height_wide { - if y >= 0 { - let ss0 = _mm_cvtepi32_ps(summs0); - let ss1 = _mm_cvtepi32_ps(summs1); - let ss2 = _mm_cvtepi32_ps(summs2); - let ss3 = _mm_cvtepi32_ps(summs3); - - let r0 = _mm_mul_ps(ss0, v_weight); - let r1 = _mm_mul_ps(ss1, v_weight); - let r2 = _mm_mul_ps(ss2, v_weight); - let r3 = _mm_mul_ps(ss3, v_weight); - - let prepared_px0 = _mm_cvtps_epi32(r0); - let prepared_px1 = _mm_cvtps_epi32(r1); - let prepared_px2 = _mm_cvtps_epi32(r2); - let prepared_px3 = _mm_cvtps_epi32(r3); - - let current_y = (y * (stride as i64)) as usize; - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px0); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px1); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px2); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px3); - - store_u8_u32::(dst_ptr0, prepared_px0); - store_u8_u32::(dst_ptr1, prepared_px1); - store_u8_u32::(dst_ptr2, prepared_px2); - store_u8_u32::(dst_ptr3, prepared_px3); - - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let stored0 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index) as *const _); - let stored1 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index) as *const _); - let stored2 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index) as *const _); - let stored3 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index) as *const _); - - let stored_10 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_11 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_12 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_13 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_1) as *const _); - - let j0 = _mm_sub_epi32(stored0, stored_10); - let j1 = _mm_sub_epi32(stored1, stored_11); - let j2 = _mm_sub_epi32(stored2, stored_12); - let j3 = _mm_sub_epi32(stored3, stored_13); - - let stored_20 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_21 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_22 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_23 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_2) as *const _); - - let k0 = _mm_mul_by_3_epi32(j0); - let k1 = _mm_mul_by_3_epi32(j1); - let k2 = _mm_mul_by_3_epi32(j2); - let k3 = _mm_mul_by_3_epi32(j3); - - let new_diff0 = _mm_sub_epi32(k0, stored_20); - let new_diff1 = _mm_sub_epi32(k1, stored_21); - let new_diff2 = _mm_sub_epi32(k2, stored_22); - let new_diff3 = _mm_sub_epi32(k3, stored_23); - - diffs0 = _mm_add_epi32(diffs0, new_diff0); - diffs1 = _mm_add_epi32(diffs1, new_diff1); - diffs2 = _mm_add_epi32(diffs2, new_diff2); - diffs3 = _mm_add_epi32(diffs3, new_diff3); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; - - let stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - let stored_10 = _mm_load_si128( - buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_11 = _mm_load_si128( - buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_12 = _mm_load_si128( - buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_13 = _mm_load_si128( - buffer3.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - - let new_diff0 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored0, stored_10)); - let new_diff1 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored1, stored_11)); - let new_diff2 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored2, stored_12)); - let new_diff3 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored3, stored_13)); - - diffs0 = _mm_add_epi32(diffs0, new_diff0); - diffs1 = _mm_add_epi32(diffs1, new_diff1); - diffs2 = _mm_add_epi32(diffs2, new_diff2); - diffs3 = _mm_add_epi32(diffs3, new_diff3); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - - let stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - diffs0 = _mm_sub_epi32(diffs0, _mm_mul_by_3_epi32(stored0)); - diffs1 = _mm_sub_epi32(diffs1, _mm_mul_by_3_epi32(stored1)); - diffs2 = _mm_sub_epi32(diffs2, _mm_mul_by_3_epi32(stored2)); - diffs3 = _mm_sub_epi32(diffs3, _mm_mul_by_3_epi32(stored3)); + unsafe { + let mut full_buffer = [SseI32x4::default(); 1024 * 4]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, rem) = rem.split_at_mut(1024); + let (buffer2, buffer3) = rem.split_at_mut(1024); + + let height_wide = height as i64; + + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm_set1_ps(weight); + + let mut xx = start; + + while xx + 4 <= width.min(end) { + let mut diffs0 = _mm_setzero_si128(); + let mut diffs1 = _mm_setzero_si128(); + let mut diffs2 = _mm_setzero_si128(); + let mut diffs3 = _mm_setzero_si128(); + + let mut ders0 = _mm_setzero_si128(); + let mut ders1 = _mm_setzero_si128(); + let mut ders2 = _mm_setzero_si128(); + let mut ders3 = _mm_setzero_si128(); + + let mut summs0 = _mm_setzero_si128(); + let mut summs1 = _mm_setzero_si128(); + let mut summs2 = _mm_setzero_si128(); + let mut summs3 = _mm_setzero_si128(); + + let start_y = 0 - 3 * radius as i64; + + let current_px0 = (xx * CN as u32) as usize; + let current_px1 = ((xx + 1) * CN as u32) as usize; + let current_px2 = ((xx + 2) * CN as u32) as usize; + let current_px3 = ((xx + 3) * CN as u32) as usize; + + for y in start_y..height_wide { + if y >= 0 { + let ss0 = _mm_cvtepi32_ps(summs0); + let ss1 = _mm_cvtepi32_ps(summs1); + let ss2 = _mm_cvtepi32_ps(summs2); + let ss3 = _mm_cvtepi32_ps(summs3); + + let r0 = _mm_mul_ps(ss0, v_weight); + let r1 = _mm_mul_ps(ss1, v_weight); + let r2 = _mm_mul_ps(ss2, v_weight); + let r3 = _mm_mul_ps(ss3, v_weight); + + let prepared_px0 = _mm_cvtps_epi32(r0); + let prepared_px1 = _mm_cvtps_epi32(r1); + let prepared_px2 = _mm_cvtps_epi32(r2); + let prepared_px3 = _mm_cvtps_epi32(r3); + + let current_y = (y * (stride as i64)) as usize; + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px0); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px1); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px2); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y + current_px3); + + store_u8_u32::(dst_ptr0, prepared_px0); + store_u8_u32::(dst_ptr1, prepared_px1); + store_u8_u32::(dst_ptr2, prepared_px2); + store_u8_u32::(dst_ptr3, prepared_px3); + + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let stored0 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index) as *const _); + let stored1 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index) as *const _); + let stored2 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index) as *const _); + let stored3 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index) as *const _); + + let stored_10 = + _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_11 = + _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_12 = + _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_13 = + _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_1) as *const _); + + let j0 = _mm_sub_epi32(stored0, stored_10); + let j1 = _mm_sub_epi32(stored1, stored_11); + let j2 = _mm_sub_epi32(stored2, stored_12); + let j3 = _mm_sub_epi32(stored3, stored_13); + + let stored_20 = + _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_21 = + _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_22 = + _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_23 = + _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_2) as *const _); + + let k0 = _mm_mul_by_3_epi32(j0); + let k1 = _mm_mul_by_3_epi32(j1); + let k2 = _mm_mul_by_3_epi32(j2); + let k3 = _mm_mul_by_3_epi32(j3); + + let new_diff0 = _mm_sub_epi32(k0, stored_20); + let new_diff1 = _mm_sub_epi32(k1, stored_21); + let new_diff2 = _mm_sub_epi32(k2, stored_22); + let new_diff3 = _mm_sub_epi32(k3, stored_23); + + diffs0 = _mm_add_epi32(diffs0, new_diff0); + diffs1 = _mm_add_epi32(diffs1, new_diff1); + diffs2 = _mm_add_epi32(diffs2, new_diff2); + diffs3 = _mm_add_epi32(diffs3, new_diff3); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + + let stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + let stored_10 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_11 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_12 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_13 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + + let new_diff0 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored0, stored_10)); + let new_diff1 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored1, stored_11)); + let new_diff2 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored2, stored_12)); + let new_diff3 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored3, stored_13)); + + diffs0 = _mm_add_epi32(diffs0, new_diff0); + diffs1 = _mm_add_epi32(diffs1, new_diff1); + diffs2 = _mm_add_epi32(diffs2, new_diff2); + diffs3 = _mm_add_epi32(diffs3, new_diff3); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + + let stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm_sub_epi32(diffs0, _mm_mul_by_3_epi32(stored0)); + diffs1 = _mm_sub_epi32(diffs1, _mm_mul_by_3_epi32(stored1)); + diffs2 = _mm_sub_epi32(diffs2, _mm_mul_by_3_epi32(stored2)); + diffs3 = _mm_sub_epi32(diffs3, _mm_mul_by_3_epi32(stored3)); + } + + let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u8; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u8; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u8; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u8; + + let pixel_color0 = load_u8_s32_fast::(s_ptr0); + let pixel_color1 = load_u8_s32_fast::(s_ptr1); + let pixel_color2 = load_u8_s32_fast::(s_ptr2); + let pixel_color3 = load_u8_s32_fast::(s_ptr3); + + let arr_index = ((y + 2 * radius_64) & 1023) as usize; + + let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr3 = buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr(); + + diffs0 = _mm_add_epi32(diffs0, pixel_color0); + diffs1 = _mm_add_epi32(diffs1, pixel_color1); + diffs2 = _mm_add_epi32(diffs2, pixel_color2); + diffs3 = _mm_add_epi32(diffs3, pixel_color3); + + _mm_store_si128(buf_ptr0 as *mut _, pixel_color0); + _mm_store_si128(buf_ptr1 as *mut _, pixel_color1); + _mm_store_si128(buf_ptr2 as *mut _, pixel_color2); + _mm_store_si128(buf_ptr3 as *mut _, pixel_color3); + + ders0 = _mm_add_epi32(ders0, diffs0); + ders1 = _mm_add_epi32(ders1, diffs1); + ders2 = _mm_add_epi32(ders2, diffs2); + ders3 = _mm_add_epi32(ders3, diffs3); + + summs0 = _mm_add_epi32(summs0, ders0); + summs1 = _mm_add_epi32(summs1, ders1); + summs2 = _mm_add_epi32(summs2, ders2); + summs3 = _mm_add_epi32(summs3, ders3); } - - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u8; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u8; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u8; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u8; - - let pixel_color0 = load_u8_s32_fast::(s_ptr0); - let pixel_color1 = load_u8_s32_fast::(s_ptr1); - let pixel_color2 = load_u8_s32_fast::(s_ptr2); - let pixel_color3 = load_u8_s32_fast::(s_ptr3); - - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - - let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr3 = buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr(); - - diffs0 = _mm_add_epi32(diffs0, pixel_color0); - diffs1 = _mm_add_epi32(diffs1, pixel_color1); - diffs2 = _mm_add_epi32(diffs2, pixel_color2); - diffs3 = _mm_add_epi32(diffs3, pixel_color3); - - _mm_store_si128(buf_ptr0 as *mut _, pixel_color0); - _mm_store_si128(buf_ptr1 as *mut _, pixel_color1); - _mm_store_si128(buf_ptr2 as *mut _, pixel_color2); - _mm_store_si128(buf_ptr3 as *mut _, pixel_color3); - - ders0 = _mm_add_epi32(ders0, diffs0); - ders1 = _mm_add_epi32(ders1, diffs1); - ders2 = _mm_add_epi32(ders2, diffs2); - ders3 = _mm_add_epi32(ders3, diffs3); - - summs0 = _mm_add_epi32(summs0, ders0); - summs1 = _mm_add_epi32(summs1, ders1); - summs2 = _mm_add_epi32(summs2, ders2); - summs3 = _mm_add_epi32(summs3, ders3); + xx += 4; } - xx += 4; - } - for x in xx..width.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut ders = _mm_setzero_si128(); - let mut summs = _mm_setzero_si128(); + for x in xx..width.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut ders = _mm_setzero_si128(); + let mut summs = _mm_setzero_si128(); - let start_y = 0 - 3 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_px = (x * CN as u32) as usize; - let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight)); + let start_y = 0 - 3 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_px = (x * CN as u32) as usize; + let prepared_px_s32 = + _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight)); - let current_y = (y * (stride as i64)) as usize; + let current_y = (y * (stride as i64)) as usize; - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - store_u8_u32::(dst_ptr, prepared_px_s32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + store_u8_u32::(dst_ptr, prepared_px_s32); - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.as_mut_ptr().add(d_arr_index_1); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.as_mut_ptr().add(d_arr_index_1); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let buf_ptr_2 = buffer0.as_mut_ptr().add(d_arr_index_2); - let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); + let buf_ptr_2 = buffer0.as_mut_ptr().add(d_arr_index_2); + let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); - let new_diff = _mm_sub_epi32( - _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), - stored_2, - ); - diffs = _mm_add_epi32(diffs, new_diff); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let new_diff = _mm_sub_epi32( + _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), + stored_2, + ); + diffs = _mm_add_epi32(diffs, new_diff); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr(); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr(); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); + let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); - diffs = _mm_add_epi32(diffs, new_diff); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); - } + diffs = _mm_add_epi32(diffs, new_diff); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); + diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); + } - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; - let pixel_color = load_u8_s32_fast::(s_ptr); + let pixel_color = load_u8_s32_fast::(s_ptr); - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((y + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); - ders = _mm_add_epi32(ders, diffs); - summs = _mm_add_epi32(summs, ders); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + diffs = _mm_add_epi32(diffs, pixel_color); + ders = _mm_add_epi32(ders, diffs); + summs = _mm_add_epi32(summs, ders); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } @@ -375,7 +392,7 @@ pub(crate) fn fgn_horizontal_pass_sse_u8( } #[target_feature(enable = "sse4.1")] -unsafe fn fgn_horizontal_pass_sse_u8_def( +fn fgn_horizontal_pass_sse_u8_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -391,7 +408,7 @@ unsafe fn fgn_horizontal_pass_sse_u8_def( } #[inline(always)] -unsafe fn fgn_horizontal_pass_sse_u8_impl( +fn fgn_horizontal_pass_sse_u8_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -401,273 +418,292 @@ unsafe fn fgn_horizontal_pass_sse_u8_impl( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([SseI32x4::default(); 1024 * 4]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, rem) = rem.split_at_mut(1024); - let (buffer2, buffer3) = rem.split_at_mut(1024); - - let width_wide = width as i64; - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm_set1_ps(weight); - - let mut yy = start; - - while yy + 4 < height.min(end) { - let mut diffs0 = _mm_setzero_si128(); - let mut diffs1 = _mm_setzero_si128(); - let mut diffs2 = _mm_setzero_si128(); - let mut diffs3 = _mm_setzero_si128(); - - let mut ders0 = _mm_setzero_si128(); - let mut ders1 = _mm_setzero_si128(); - let mut ders2 = _mm_setzero_si128(); - let mut ders3 = _mm_setzero_si128(); - - let mut summs0 = _mm_setzero_si128(); - let mut summs1 = _mm_setzero_si128(); - let mut summs2 = _mm_setzero_si128(); - let mut summs3 = _mm_setzero_si128(); - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; - let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; - let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; - - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; - - let ss0 = _mm_cvtepi32_ps(summs0); - let ss1 = _mm_cvtepi32_ps(summs1); - let ss2 = _mm_cvtepi32_ps(summs2); - let ss3 = _mm_cvtepi32_ps(summs3); - - let r0 = _mm_mul_ps(ss0, v_weight); - let r1 = _mm_mul_ps(ss1, v_weight); - let r2 = _mm_mul_ps(ss2, v_weight); - let r3 = _mm_mul_ps(ss3, v_weight); - - let prepared_px0 = _mm_cvtps_epi32(r0); - let prepared_px1 = _mm_cvtps_epi32(r1); - let prepared_px2 = _mm_cvtps_epi32(r2); - let prepared_px3 = _mm_cvtps_epi32(r3); - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y0 + current_px); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y1 + current_px); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y2 + current_px); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y3 + current_px); - - store_u8_u32::(dst_ptr0, prepared_px0); - store_u8_u32::(dst_ptr1, prepared_px1); - store_u8_u32::(dst_ptr2, prepared_px2); - store_u8_u32::(dst_ptr3, prepared_px3); - - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let stored0 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index) as *const _); - let stored1 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index) as *const _); - let stored2 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index) as *const _); - let stored3 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index) as *const _); - - let stored_10 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_11 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_12 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_13 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_1) as *const _); - - let j0 = _mm_sub_epi32(stored0, stored_10); - let j1 = _mm_sub_epi32(stored1, stored_11); - let j2 = _mm_sub_epi32(stored2, stored_12); - let j3 = _mm_sub_epi32(stored3, stored_13); - - let stored_20 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_21 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_22 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_23 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_2) as *const _); - - let k0 = _mm_mul_by_3_epi32(j0); - let k1 = _mm_mul_by_3_epi32(j1); - let k2 = _mm_mul_by_3_epi32(j2); - let k3 = _mm_mul_by_3_epi32(j3); - - let new_diff0 = _mm_sub_epi32(k0, stored_20); - let new_diff1 = _mm_sub_epi32(k1, stored_21); - let new_diff2 = _mm_sub_epi32(k2, stored_22); - let new_diff3 = _mm_sub_epi32(k3, stored_23); - - diffs0 = _mm_add_epi32(diffs0, new_diff0); - diffs1 = _mm_add_epi32(diffs1, new_diff1); - diffs2 = _mm_add_epi32(diffs2, new_diff2); - diffs3 = _mm_add_epi32(diffs3, new_diff3); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - - let stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - let stored_10 = _mm_load_si128( - buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_11 = _mm_load_si128( - buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_12 = _mm_load_si128( - buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_13 = _mm_load_si128( - buffer3.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - - let new_diff0 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored0, stored_10)); - let new_diff1 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored1, stored_11)); - let new_diff2 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored2, stored_12)); - let new_diff3 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored3, stored_13)); - - diffs0 = _mm_add_epi32(diffs0, new_diff0); - diffs1 = _mm_add_epi32(diffs1, new_diff1); - diffs2 = _mm_add_epi32(diffs2, new_diff2); - diffs3 = _mm_add_epi32(diffs3, new_diff3); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - diffs0 = _mm_sub_epi32(diffs0, _mm_mul_by_3_epi32(stored0)); - diffs1 = _mm_sub_epi32(diffs1, _mm_mul_by_3_epi32(stored1)); - diffs2 = _mm_sub_epi32(diffs2, _mm_mul_by_3_epi32(stored2)); - diffs3 = _mm_sub_epi32(diffs3, _mm_mul_by_3_epi32(stored3)); + unsafe { + let mut full_buffer = [SseI32x4::default(); 1024 * 4]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, rem) = rem.split_at_mut(1024); + let (buffer2, buffer3) = rem.split_at_mut(1024); + + let width_wide = width as i64; + + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm_set1_ps(weight); + + let mut yy = start; + + while yy + 4 <= height.min(end) { + let mut diffs0 = _mm_setzero_si128(); + let mut diffs1 = _mm_setzero_si128(); + let mut diffs2 = _mm_setzero_si128(); + let mut diffs3 = _mm_setzero_si128(); + + let mut ders0 = _mm_setzero_si128(); + let mut ders1 = _mm_setzero_si128(); + let mut ders2 = _mm_setzero_si128(); + let mut ders3 = _mm_setzero_si128(); + + let mut summs0 = _mm_setzero_si128(); + let mut summs1 = _mm_setzero_si128(); + let mut summs2 = _mm_setzero_si128(); + let mut summs3 = _mm_setzero_si128(); + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; + let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; + let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; + + let ss0 = _mm_cvtepi32_ps(summs0); + let ss1 = _mm_cvtepi32_ps(summs1); + let ss2 = _mm_cvtepi32_ps(summs2); + let ss3 = _mm_cvtepi32_ps(summs3); + + let r0 = _mm_mul_ps(ss0, v_weight); + let r1 = _mm_mul_ps(ss1, v_weight); + let r2 = _mm_mul_ps(ss2, v_weight); + let r3 = _mm_mul_ps(ss3, v_weight); + + let prepared_px0 = _mm_cvtps_epi32(r0); + let prepared_px1 = _mm_cvtps_epi32(r1); + let prepared_px2 = _mm_cvtps_epi32(r2); + let prepared_px3 = _mm_cvtps_epi32(r3); + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u8).add(current_y0 + current_px); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u8).add(current_y1 + current_px); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u8).add(current_y2 + current_px); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u8).add(current_y3 + current_px); + + store_u8_u32::(dst_ptr0, prepared_px0); + store_u8_u32::(dst_ptr1, prepared_px1); + store_u8_u32::(dst_ptr2, prepared_px2); + store_u8_u32::(dst_ptr3, prepared_px3); + + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let stored0 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index) as *const _); + let stored1 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index) as *const _); + let stored2 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index) as *const _); + let stored3 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index) as *const _); + + let stored_10 = + _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_11 = + _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_12 = + _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_13 = + _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_1) as *const _); + + let j0 = _mm_sub_epi32(stored0, stored_10); + let j1 = _mm_sub_epi32(stored1, stored_11); + let j2 = _mm_sub_epi32(stored2, stored_12); + let j3 = _mm_sub_epi32(stored3, stored_13); + + let stored_20 = + _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_21 = + _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_22 = + _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_23 = + _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_2) as *const _); + + let k0 = _mm_mul_by_3_epi32(j0); + let k1 = _mm_mul_by_3_epi32(j1); + let k2 = _mm_mul_by_3_epi32(j2); + let k3 = _mm_mul_by_3_epi32(j3); + + let new_diff0 = _mm_sub_epi32(k0, stored_20); + let new_diff1 = _mm_sub_epi32(k1, stored_21); + let new_diff2 = _mm_sub_epi32(k2, stored_22); + let new_diff3 = _mm_sub_epi32(k3, stored_23); + + diffs0 = _mm_add_epi32(diffs0, new_diff0); + diffs1 = _mm_add_epi32(diffs1, new_diff1); + diffs2 = _mm_add_epi32(diffs2, new_diff2); + diffs3 = _mm_add_epi32(diffs3, new_diff3); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + + let stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + let stored_10 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_11 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_12 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_13 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + + let new_diff0 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored0, stored_10)); + let new_diff1 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored1, stored_11)); + let new_diff2 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored2, stored_12)); + let new_diff3 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored3, stored_13)); + + diffs0 = _mm_add_epi32(diffs0, new_diff0); + diffs1 = _mm_add_epi32(diffs1, new_diff1); + diffs2 = _mm_add_epi32(diffs2, new_diff2); + diffs3 = _mm_add_epi32(diffs3, new_diff3); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm_sub_epi32(diffs0, _mm_mul_by_3_epi32(stored0)); + diffs1 = _mm_sub_epi32(diffs1, _mm_mul_by_3_epi32(stored1)); + diffs2 = _mm_sub_epi32(diffs2, _mm_mul_by_3_epi32(stored2)); + diffs3 = _mm_sub_epi32(diffs3, _mm_mul_by_3_epi32(stored3)); + } + + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u8; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u8; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u8; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u8; + + let pixel_color0 = load_u8_s32_fast::(s_ptr0); + let pixel_color1 = load_u8_s32_fast::(s_ptr1); + let pixel_color2 = load_u8_s32_fast::(s_ptr2); + let pixel_color3 = load_u8_s32_fast::(s_ptr3); + + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr3 = buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr(); + + diffs0 = _mm_add_epi32(diffs0, pixel_color0); + diffs1 = _mm_add_epi32(diffs1, pixel_color1); + diffs2 = _mm_add_epi32(diffs2, pixel_color2); + diffs3 = _mm_add_epi32(diffs3, pixel_color3); + + _mm_store_si128(buf_ptr0 as *mut _, pixel_color0); + _mm_store_si128(buf_ptr1 as *mut _, pixel_color1); + _mm_store_si128(buf_ptr2 as *mut _, pixel_color2); + _mm_store_si128(buf_ptr3 as *mut _, pixel_color3); + + ders0 = _mm_add_epi32(ders0, diffs0); + ders1 = _mm_add_epi32(ders1, diffs1); + ders2 = _mm_add_epi32(ders2, diffs2); + ders3 = _mm_add_epi32(ders3, diffs3); + + summs0 = _mm_add_epi32(summs0, ders0); + summs1 = _mm_add_epi32(summs1, ders1); + summs2 = _mm_add_epi32(summs2, ders2); + summs3 = _mm_add_epi32(summs3, ders3); } - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u8; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u8; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u8; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u8; - - let pixel_color0 = load_u8_s32_fast::(s_ptr0); - let pixel_color1 = load_u8_s32_fast::(s_ptr1); - let pixel_color2 = load_u8_s32_fast::(s_ptr2); - let pixel_color3 = load_u8_s32_fast::(s_ptr3); - - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr3 = buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr(); - - diffs0 = _mm_add_epi32(diffs0, pixel_color0); - diffs1 = _mm_add_epi32(diffs1, pixel_color1); - diffs2 = _mm_add_epi32(diffs2, pixel_color2); - diffs3 = _mm_add_epi32(diffs3, pixel_color3); - - _mm_store_si128(buf_ptr0 as *mut _, pixel_color0); - _mm_store_si128(buf_ptr1 as *mut _, pixel_color1); - _mm_store_si128(buf_ptr2 as *mut _, pixel_color2); - _mm_store_si128(buf_ptr3 as *mut _, pixel_color3); - - ders0 = _mm_add_epi32(ders0, diffs0); - ders1 = _mm_add_epi32(ders1, diffs1); - ders2 = _mm_add_epi32(ders2, diffs2); - ders3 = _mm_add_epi32(ders3, diffs3); - - summs0 = _mm_add_epi32(summs0, ders0); - summs1 = _mm_add_epi32(summs1, ders1); - summs2 = _mm_add_epi32(summs2, ders2); - summs3 = _mm_add_epi32(summs3, ders3); + yy += 4; } - yy += 4; - } - - for y in yy..height.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut ders = _mm_setzero_si128(); - let mut summs = _mm_setzero_si128(); + for y in yy..height.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut ders = _mm_setzero_si128(); + let mut summs = _mm_setzero_si128(); - let current_y = ((y as i64) * (stride as i64)) as usize; + let current_y = ((y as i64) * (stride as i64)) as usize; - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; - let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight)); + let prepared_px_s32 = + _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight)); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - store_u8_u32::(dst_ptr, prepared_px_s32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + store_u8_u32::(dst_ptr, prepared_px_s32); - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.get_unchecked_mut(d_arr_index_1).0.as_mut_ptr(); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.get_unchecked_mut(d_arr_index_1).0.as_mut_ptr(); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let buf_ptr_2 = buffer0.get_unchecked_mut(d_arr_index_2).0.as_mut_ptr(); - let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); + let buf_ptr_2 = buffer0.get_unchecked_mut(d_arr_index_2).0.as_mut_ptr(); + let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); - let new_diff = _mm_sub_epi32( - _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), - stored_2, - ); - diffs = _mm_add_epi32(diffs, new_diff); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let new_diff = _mm_sub_epi32( + _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), + stored_2, + ); + diffs = _mm_add_epi32(diffs, new_diff); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.as_mut_ptr().add(arr_index_1); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.as_mut_ptr().add(arr_index_1); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); + let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); - diffs = _mm_add_epi32(diffs, new_diff); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.as_mut_ptr().add(arr_index); - let stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); - } + diffs = _mm_add_epi32(diffs, new_diff); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.as_mut_ptr().add(arr_index); + let stored = _mm_load_si128(buf_ptr as *const __m128i); + diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); + } - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; - let pixel_color = load_u8_s32_fast::(s_ptr); + let pixel_color = load_u8_s32_fast::(s_ptr); - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); - ders = _mm_add_epi32(ders, diffs); - summs = _mm_add_epi32(summs, ders); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + diffs = _mm_add_epi32(diffs, pixel_color); + ders = _mm_add_epi32(ders, diffs); + summs = _mm_add_epi32(summs, ders); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } diff --git a/src/sse/fast_gaussian_next_f16.rs b/src/sse/fast_gaussian_next_f16.rs index 4bce3870..f122eb60 100644 --- a/src/sse/fast_gaussian_next_f16.rs +++ b/src/sse/fast_gaussian_next_f16.rs @@ -30,14 +30,14 @@ use std::arch::x86::*; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::sse::{load_f32_f16, store_f32_f16}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; use core::f16; -pub(crate) fn fast_gaussian_next_vertical_pass_sse_f16( - undefined_slice: &UnsafeSlice, +pub(crate) fn fast_gaussian_next_vertical_pass_sse_f16( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -47,22 +47,15 @@ pub(crate) fn fast_gaussian_next_vertical_pass_sse_f16( edge_mode: EdgeMode, ) { unsafe { - fast_gaussian_next_vertical_pass_sse_f16_impl::( - undefined_slice, - stride, - width, - height, - radius, - start, - end, - edge_mode, + fast_gaussian_next_vertical_pass_sse_f16_impl::( + slice, stride, width, height, radius, start, end, edge_mode, ); } } #[target_feature(enable = "sse4.1", enable = "f16c")] -unsafe fn fast_gaussian_next_vertical_pass_sse_f16_impl( - undefined_slice: &UnsafeSlice, +fn fast_gaussian_next_vertical_pass_sse_f16_impl( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -71,89 +64,90 @@ unsafe fn fast_gaussian_next_vertical_pass_sse_f16_impl( end: u32, edge_mode: EdgeMode, ) { - let bytes: &UnsafeSlice<'_, f16> = std::mem::transmute(undefined_slice); - let mut buffer: [[f32; 4]; 1024] = [[0.; 4]; 1024]; + unsafe { + let mut buffer: [[f32; 4]; 1024] = [[0.; 4]; 1024]; - let height_wide = height as i64; + let height_wide = height as i64; - let threes = _mm_set1_ps(3.); + let threes = _mm_set1_ps(3.); - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm_set1_ps(weight); - for x in start..std::cmp::min(width, end) { - let mut diffs = _mm_setzero_ps(); - let mut ders = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm_set1_ps(weight); + for x in start..std::cmp::min(width, end) { + let mut diffs = _mm_setzero_ps(); + let mut ders = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); - let current_px = (x * CN as u32) as usize; + let current_px = (x * CN as u32) as usize; - let start_y = 0 - 3 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_y = (y * (stride as i64)) as usize; - let bytes_offset = current_y + current_px; + let start_y = 0 - 3 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_y = (y * (stride as i64)) as usize; + let bytes_offset = current_y + current_px; - let pixel = _mm_mul_ps(summs, v_weight); - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f16; - store_f32_f16::(dst_ptr, pixel); + let pixel = _mm_mul_ps(summs, v_weight); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f16; + store_f32_f16::(dst_ptr, pixel); - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); + let buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); - let buf_ptr_1 = buffer.as_mut_ptr().add(d_arr_index_1) as *mut f32; - let stored_1 = _mm_loadu_ps(buf_ptr_1); + let buf_ptr_1 = buffer.as_mut_ptr().add(d_arr_index_1) as *mut f32; + let stored_1 = _mm_loadu_ps(buf_ptr_1); - let buf_ptr_2 = buffer.as_mut_ptr().add(d_arr_index_2) as *mut f32; - let stored_2 = _mm_loadu_ps(buf_ptr_2); + let buf_ptr_2 = buffer.as_mut_ptr().add(d_arr_index_2) as *mut f32; + let stored_2 = _mm_loadu_ps(buf_ptr_2); - let new_diff = - _mm_sub_ps(_mm_mul_ps(_mm_sub_ps(stored, stored_1), threes), stored_2); - diffs = _mm_add_ps(diffs, new_diff); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); + let new_diff = + _mm_sub_ps(_mm_mul_ps(_mm_sub_ps(stored, stored_1), threes), stored_2); + diffs = _mm_add_ps(diffs, new_diff); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); - let buf_ptr_1 = buffer.get_unchecked_mut(arr_index_1).as_mut_ptr(); - let stored_1 = _mm_loadu_ps(buf_ptr_1); + let buf_ptr_1 = buffer.get_unchecked_mut(arr_index_1).as_mut_ptr(); + let stored_1 = _mm_loadu_ps(buf_ptr_1); - let new_diff = _mm_mul_ps(_mm_sub_ps(stored, stored_1), threes); + let new_diff = _mm_mul_ps(_mm_sub_ps(stored, stored_1), threes); - diffs = _mm_add_ps(diffs, new_diff); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); - diffs = _mm_sub_ps(diffs, _mm_mul_ps(stored, threes)); - } + diffs = _mm_add_ps(diffs, new_diff); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); + diffs = _mm_sub_ps(diffs, _mm_mul_ps(stored, threes)); + } - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f16; + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f16; - let pixel_color = load_f32_f16::(s_ptr); + let pixel_color = load_f32_f16::(s_ptr); - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let arr_index = ((y + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - diffs = _mm_add_ps(diffs, pixel_color); - ders = _mm_add_ps(ders, diffs); - summs = _mm_add_ps(summs, ders); - _mm_storeu_ps(buf_ptr, pixel_color); + diffs = _mm_add_ps(diffs, pixel_color); + ders = _mm_add_ps(ders, diffs); + summs = _mm_add_ps(summs, ders); + _mm_storeu_ps(buf_ptr, pixel_color); + } } } } -pub(crate) fn fast_gaussian_next_horizontal_pass_sse_f16( - undefined_slice: &UnsafeSlice, +pub(crate) fn fast_gaussian_next_horizontal_pass_sse_f16( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -163,22 +157,15 @@ pub(crate) fn fast_gaussian_next_horizontal_pass_sse_f16( edge_mode: EdgeMode, ) { unsafe { - fast_gaussian_next_horizontal_pass_sse_f16_impl::( - undefined_slice, - stride, - width, - height, - radius, - start, - end, - edge_mode, + fast_gaussian_next_horizontal_pass_sse_f16_impl::( + slice, stride, width, height, radius, start, end, edge_mode, ); } } #[target_feature(enable = "sse4.1", enable = "f16c")] -unsafe fn fast_gaussian_next_horizontal_pass_sse_f16_impl( - undefined_slice: &UnsafeSlice, +fn fast_gaussian_next_horizontal_pass_sse_f16_impl( + undefined_slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -187,83 +174,85 @@ unsafe fn fast_gaussian_next_horizontal_pass_sse_f16_impl( end: u32, edge_mode: EdgeMode, ) { - let bytes: &UnsafeSlice<'_, f16> = std::mem::transmute(undefined_slice); - let mut buffer: [[f32; 4]; 1024] = [[0.; 4]; 1024]; + unsafe { + let bytes: &UnsafeSlice<'_, f16> = std::mem::transmute(undefined_slice); + let mut buffer: [[f32; 4]; 1024] = [[0.; 4]; 1024]; - let width_wide = width as i64; + let width_wide = width as i64; - let threes = _mm_set1_ps(3.); + let threes = _mm_set1_ps(3.); - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm_set1_ps(weight); - for y in start..std::cmp::min(height, end) { - let mut diffs = _mm_setzero_ps(); - let mut ders = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm_set1_ps(weight); + for y in start..std::cmp::min(height, end) { + let mut diffs = _mm_setzero_ps(); + let mut ders = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); - let current_y = ((y as i64) * (stride as i64)) as usize; + let current_y = ((y as i64) * (stride as i64)) as usize; - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let pixel = _mm_mul_ps(summs, v_weight); - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f16; - store_f32_f16::(dst_ptr, pixel); + let pixel = _mm_mul_ps(summs, v_weight); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f16; + store_f32_f16::(dst_ptr, pixel); - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); + let buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); - let buf_ptr_1 = buffer.get_unchecked_mut(d_arr_index_1).as_mut_ptr(); - let stored_1 = _mm_loadu_ps(buf_ptr_1); + let buf_ptr_1 = buffer.get_unchecked_mut(d_arr_index_1).as_mut_ptr(); + let stored_1 = _mm_loadu_ps(buf_ptr_1); - let buf_ptr_2 = buffer.get_unchecked_mut(d_arr_index_2).as_mut_ptr(); - let stored_2 = _mm_loadu_ps(buf_ptr_2); + let buf_ptr_2 = buffer.get_unchecked_mut(d_arr_index_2).as_mut_ptr(); + let stored_2 = _mm_loadu_ps(buf_ptr_2); - let new_diff = - _mm_sub_ps(_mm_mul_ps(_mm_sub_ps(stored, stored_1), threes), stored_2); - diffs = _mm_add_ps(diffs, new_diff); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer.as_mut_ptr().add(arr_index) as *mut f32; - let stored = _mm_loadu_ps(buf_ptr); + let new_diff = + _mm_sub_ps(_mm_mul_ps(_mm_sub_ps(stored, stored_1), threes), stored_2); + diffs = _mm_add_ps(diffs, new_diff); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer.as_mut_ptr().add(arr_index) as *mut f32; + let stored = _mm_loadu_ps(buf_ptr); - let buf_ptr_1 = buffer.as_mut_ptr().add(arr_index_1); - let stored_1 = _mm_loadu_ps(buf_ptr_1 as *const f32); + let buf_ptr_1 = buffer.as_mut_ptr().add(arr_index_1); + let stored_1 = _mm_loadu_ps(buf_ptr_1 as *const f32); - let new_diff = _mm_mul_ps(_mm_sub_ps(stored, stored_1), threes); + let new_diff = _mm_mul_ps(_mm_sub_ps(stored, stored_1), threes); - diffs = _mm_add_ps(diffs, new_diff); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer.as_mut_ptr().add(arr_index); - let stored = _mm_loadu_ps(buf_ptr as *const f32); - diffs = _mm_sub_ps(diffs, _mm_mul_ps(stored, threes)); - } + diffs = _mm_add_ps(diffs, new_diff); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer.as_mut_ptr().add(arr_index); + let stored = _mm_loadu_ps(buf_ptr as *const f32); + diffs = _mm_sub_ps(diffs, _mm_mul_ps(stored, threes)); + } - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f16; + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f16; - let pixel_color = load_f32_f16::(s_ptr); + let pixel_color = load_f32_f16::(s_ptr); - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - diffs = _mm_add_ps(diffs, pixel_color); - ders = _mm_add_ps(ders, diffs); - summs = _mm_add_ps(summs, ders); - _mm_storeu_ps(buf_ptr, pixel_color); + diffs = _mm_add_ps(diffs, pixel_color); + ders = _mm_add_ps(ders, diffs); + summs = _mm_add_ps(summs, ders); + _mm_storeu_ps(buf_ptr, pixel_color); + } } } } diff --git a/src/sse/fast_gaussian_next_f32.rs b/src/sse/fast_gaussian_next_f32.rs index 96c92e9a..0783ffa5 100644 --- a/src/sse/fast_gaussian_next_f32.rs +++ b/src/sse/fast_gaussian_next_f32.rs @@ -25,17 +25,17 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::sse::{_mm_opt_fmlaf_ps, _mm_opt_fnmlaf_ps, _mm_opt_fnmlsf_ps, load_f32, store_f32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; -pub(crate) fn fgn_vertical_pass_sse_f32( - undefined_slice: &UnsafeSlice, +pub(crate) fn fgn_vertical_pass_sse_f32( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -45,21 +45,20 @@ pub(crate) fn fgn_vertical_pass_sse_f32( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undefined_slice); if std::arch::is_x86_feature_detected!("fma") { fgn_vertical_pass_sse_f32_fma::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } else { fgn_vertical_pass_sse_f32_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } } #[target_feature(enable = "sse4.1")] -unsafe fn fgn_vertical_pass_sse_f32_def( +fn fgn_vertical_pass_sse_f32_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -75,7 +74,7 @@ unsafe fn fgn_vertical_pass_sse_f32_def( } #[target_feature(enable = "sse4.1", enable = "fma")] -unsafe fn fgn_vertical_pass_sse_f32_fma( +fn fgn_vertical_pass_sse_f32_fma( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -85,13 +84,13 @@ unsafe fn fgn_vertical_pass_sse_f32_fma( end: u32, edge_mode: EdgeMode, ) { - fgn_vertical_pass_sse_f32_impl::( + fgn_vertical_pass_sse_f32_impl::( bytes, stride, width, height, radius, start, end, edge_mode, ); } #[inline(always)] -unsafe fn fgn_vertical_pass_sse_f32_impl( +fn fgn_vertical_pass_sse_f32_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -101,210 +100,212 @@ unsafe fn fgn_vertical_pass_sse_f32_impl( end: u32, edge_mode: EdgeMode, ) { - let mut bf0 = Box::new([[0f32; 4]; 1024]); - let mut bf1 = Box::new([[0f32; 4]; 1024]); - let mut bf2 = Box::new([[0f32; 4]; 1024]); - - let height_wide = height as i64; - - let threes = _mm_set1_ps(3.); - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm_set1_ps(weight); - - let mut xx = start as usize; - - while xx + 3 < width.min(end) as usize { - let mut diffs0 = _mm_setzero_ps(); - let mut diffs1 = _mm_setzero_ps(); - let mut diffs2 = _mm_setzero_ps(); - - let mut ders0 = _mm_setzero_ps(); - let mut ders1 = _mm_setzero_ps(); - let mut ders2 = _mm_setzero_ps(); - - let mut summs0 = _mm_setzero_ps(); - let mut summs1 = _mm_setzero_ps(); - let mut summs2 = _mm_setzero_ps(); - - let current_px0 = xx * CN; - let current_px1 = (xx + 1) * CN; - let current_px2 = (xx + 2) * CN; - - let start_y = 0 - 3 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_y = (y * (stride as i64)) as usize; - - let prepared_px0 = _mm_mul_ps(summs0, v_weight); - let prepared_px1 = _mm_mul_ps(summs1, v_weight); - let prepared_px2 = _mm_mul_ps(summs2, v_weight); - - let dst_ptr0 = bytes.slice.as_ptr().add(current_y + current_px0) as *mut f32; - let dst_ptr1 = bytes.slice.as_ptr().add(current_y + current_px1) as *mut f32; - let dst_ptr2 = bytes.slice.as_ptr().add(current_y + current_px2) as *mut f32; - - store_f32::(dst_ptr0, prepared_px0); - store_f32::(dst_ptr1, prepared_px1); - store_f32::(dst_ptr2, prepared_px2); - - let d_a_1 = ((y + radius_64) & 1023) as usize; - let d_a_2 = ((y - radius_64) & 1023) as usize; - let d_i = (y & 1023) as usize; - - let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_i) as *mut f32); - let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_i) as *mut f32); - let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_i) as *mut f32); - - let sd_1_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_a_1) as *mut f32); - let sd_1_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_a_1) as *mut f32); - let sd_1_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_a_1) as *mut f32); - - let j0 = _mm_sub_ps(sd0, sd_1_0); - let j1 = _mm_sub_ps(sd1, sd_1_1); - let j2 = _mm_sub_ps(sd2, sd_1_2); - - let sd_2_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_a_2) as *mut f32); - let sd_2_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_a_2) as *mut f32); - let sd_2_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_a_2) as *mut f32); - - let new_diff0 = _mm_opt_fnmlsf_ps::(sd_2_0, j0, threes); - let new_diff1 = _mm_opt_fnmlsf_ps::(sd_2_1, j1, threes); - let new_diff2 = _mm_opt_fnmlsf_ps::(sd_2_2, j2, threes); - - diffs0 = _mm_add_ps(diffs0, new_diff0); - diffs1 = _mm_add_ps(diffs1, new_diff1); - diffs2 = _mm_add_ps(diffs2, new_diff2); - } else if y + radius_64 >= 0 { - let a_i = (y & 1023) as usize; - let a_i_1 = ((y + radius_64) & 1023) as usize; - let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(a_i) as *mut f32); - let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(a_i) as *mut f32); - let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(a_i) as *mut f32); - - let sd_1_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(a_i_1) as *mut f32); - let sd_1_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(a_i_1) as *mut f32); - let sd_1_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(a_i_1) as *mut f32); - - diffs0 = _mm_opt_fmlaf_ps::(diffs0, _mm_sub_ps(sd0, sd_1_0), threes); - diffs1 = _mm_opt_fmlaf_ps::(diffs1, _mm_sub_ps(sd1, sd_1_1), threes); - diffs2 = _mm_opt_fmlaf_ps::(diffs2, _mm_sub_ps(sd2, sd_1_2), threes); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = _mm_opt_fnmlaf_ps::(diffs0, sd0, threes); - diffs1 = _mm_opt_fnmlaf_ps::(diffs1, sd1, threes); - diffs2 = _mm_opt_fnmlaf_ps::(diffs2, sd2, threes); + unsafe { + let mut bf0 = [[0f32; 4]; 1024]; + let mut bf1 = [[0f32; 4]; 1024]; + let mut bf2 = [[0f32; 4]; 1024]; + + let height_wide = height as i64; + + let threes = _mm_set1_ps(3.); + + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm_set1_ps(weight); + + let mut xx = start as usize; + + while xx + 3 <= width.min(end) as usize { + let mut diffs0 = _mm_setzero_ps(); + let mut diffs1 = _mm_setzero_ps(); + let mut diffs2 = _mm_setzero_ps(); + + let mut ders0 = _mm_setzero_ps(); + let mut ders1 = _mm_setzero_ps(); + let mut ders2 = _mm_setzero_ps(); + + let mut summs0 = _mm_setzero_ps(); + let mut summs1 = _mm_setzero_ps(); + let mut summs2 = _mm_setzero_ps(); + + let current_px0 = xx * CN; + let current_px1 = (xx + 1) * CN; + let current_px2 = (xx + 2) * CN; + + let start_y = 0 - 3 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_y = (y * (stride as i64)) as usize; + + let prepared_px0 = _mm_mul_ps(summs0, v_weight); + let prepared_px1 = _mm_mul_ps(summs1, v_weight); + let prepared_px2 = _mm_mul_ps(summs2, v_weight); + + let dst_ptr0 = bytes.slice.as_ptr().add(current_y + current_px0) as *mut f32; + let dst_ptr1 = bytes.slice.as_ptr().add(current_y + current_px1) as *mut f32; + let dst_ptr2 = bytes.slice.as_ptr().add(current_y + current_px2) as *mut f32; + + store_f32::(dst_ptr0, prepared_px0); + store_f32::(dst_ptr1, prepared_px1); + store_f32::(dst_ptr2, prepared_px2); + + let d_a_1 = ((y + radius_64) & 1023) as usize; + let d_a_2 = ((y - radius_64) & 1023) as usize; + let d_i = (y & 1023) as usize; + + let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_i) as *mut f32); + let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_i) as *mut f32); + let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_i) as *mut f32); + + let sd_1_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_a_1) as *mut f32); + let sd_1_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_a_1) as *mut f32); + let sd_1_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_a_1) as *mut f32); + + let j0 = _mm_sub_ps(sd0, sd_1_0); + let j1 = _mm_sub_ps(sd1, sd_1_1); + let j2 = _mm_sub_ps(sd2, sd_1_2); + + let sd_2_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_a_2) as *mut f32); + let sd_2_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_a_2) as *mut f32); + let sd_2_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_a_2) as *mut f32); + + let new_diff0 = _mm_opt_fnmlsf_ps::(sd_2_0, j0, threes); + let new_diff1 = _mm_opt_fnmlsf_ps::(sd_2_1, j1, threes); + let new_diff2 = _mm_opt_fnmlsf_ps::(sd_2_2, j2, threes); + + diffs0 = _mm_add_ps(diffs0, new_diff0); + diffs1 = _mm_add_ps(diffs1, new_diff1); + diffs2 = _mm_add_ps(diffs2, new_diff2); + } else if y + radius_64 >= 0 { + let a_i = (y & 1023) as usize; + let a_i_1 = ((y + radius_64) & 1023) as usize; + let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(a_i) as *mut f32); + let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(a_i) as *mut f32); + let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(a_i) as *mut f32); + + let sd_1_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(a_i_1) as *mut f32); + let sd_1_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(a_i_1) as *mut f32); + let sd_1_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(a_i_1) as *mut f32); + + diffs0 = _mm_opt_fmlaf_ps::(diffs0, _mm_sub_ps(sd0, sd_1_0), threes); + diffs1 = _mm_opt_fmlaf_ps::(diffs1, _mm_sub_ps(sd1, sd_1_1), threes); + diffs2 = _mm_opt_fmlaf_ps::(diffs2, _mm_sub_ps(sd2, sd_1_2), threes); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm_opt_fnmlaf_ps::(diffs0, sd0, threes); + diffs1 = _mm_opt_fnmlaf_ps::(diffs1, sd1, threes); + diffs2 = _mm_opt_fnmlaf_ps::(diffs2, sd2, threes); + } + + let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut f32; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut f32; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut f32; + + let pixel_color0 = load_f32::(s_ptr0); + let pixel_color1 = load_f32::(s_ptr1); + let pixel_color2 = load_f32::(s_ptr2); + + let a_i = ((y + 2 * radius_64) & 1023) as usize; + + _mm_storeu_ps(bf0.as_mut_ptr().add(a_i) as *mut f32, pixel_color0); + _mm_storeu_ps(bf1.as_mut_ptr().add(a_i) as *mut f32, pixel_color1); + _mm_storeu_ps(bf2.as_mut_ptr().add(a_i) as *mut f32, pixel_color2); + + diffs0 = _mm_add_ps(diffs0, pixel_color0); + diffs1 = _mm_add_ps(diffs1, pixel_color1); + diffs2 = _mm_add_ps(diffs2, pixel_color2); + + ders0 = _mm_add_ps(ders0, diffs0); + ders1 = _mm_add_ps(ders1, diffs1); + ders2 = _mm_add_ps(ders2, diffs2); + + summs0 = _mm_add_ps(summs0, ders0); + summs1 = _mm_add_ps(summs1, ders1); + summs2 = _mm_add_ps(summs2, ders2); } - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut f32; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut f32; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut f32; + xx += 3; + } - let pixel_color0 = load_f32::(s_ptr0); - let pixel_color1 = load_f32::(s_ptr1); - let pixel_color2 = load_f32::(s_ptr2); + for x in xx..width.min(end) as usize { + let mut diffs = _mm_setzero_ps(); + let mut ders = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); - let a_i = ((y + 2 * radius_64) & 1023) as usize; + let current_px = x * CN; - _mm_storeu_ps(bf0.as_mut_ptr().add(a_i) as *mut f32, pixel_color0); - _mm_storeu_ps(bf1.as_mut_ptr().add(a_i) as *mut f32, pixel_color1); - _mm_storeu_ps(bf2.as_mut_ptr().add(a_i) as *mut f32, pixel_color2); + let start_y = 0 - 3 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_y = (y * (stride as i64)) as usize; + let bytes_offset = current_y + current_px; - diffs0 = _mm_add_ps(diffs0, pixel_color0); - diffs1 = _mm_add_ps(diffs1, pixel_color1); - diffs2 = _mm_add_ps(diffs2, pixel_color2); + let pixel = _mm_mul_ps(summs, v_weight); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; + store_f32::(dst_ptr, pixel); - ders0 = _mm_add_ps(ders0, diffs0); - ders1 = _mm_add_ps(ders1, diffs1); - ders2 = _mm_add_ps(ders2, diffs2); + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - summs0 = _mm_add_ps(summs0, ders0); - summs1 = _mm_add_ps(summs1, ders1); - summs2 = _mm_add_ps(summs2, ders2); - } + let buf_ptr = bf0.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); - xx += 3; - } + let buf_ptr_1 = bf0.as_mut_ptr().add(d_arr_index_1) as *mut f32; + let stored_1 = _mm_loadu_ps(buf_ptr_1); - for x in xx..width.min(end) as usize { - let mut diffs = _mm_setzero_ps(); - let mut ders = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); + let buf_ptr_2 = bf0.as_mut_ptr().add(d_arr_index_2) as *mut f32; + let stored_2 = _mm_loadu_ps(buf_ptr_2); - let current_px = x * CN; + let new_diff = + _mm_opt_fnmlsf_ps::(stored_2, _mm_sub_ps(stored, stored_1), threes); + diffs = _mm_add_ps(diffs, new_diff); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); - let start_y = 0 - 3 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_y = (y * (stride as i64)) as usize; - let bytes_offset = current_y + current_px; + let buf_ptr_1 = bf0.get_unchecked_mut(arr_index_1).as_mut_ptr(); + let stored_1 = _mm_loadu_ps(buf_ptr_1); - let pixel = _mm_mul_ps(summs, v_weight); - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; - store_f32::(dst_ptr, pixel); + diffs = _mm_opt_fmlaf_ps::(diffs, _mm_sub_ps(stored, stored_1), threes); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); + diffs = _mm_opt_fnmlaf_ps::(diffs, stored, threes); + } - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + let next_row_x = x * CN; - let buf_ptr = bf0.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f32; - let buf_ptr_1 = bf0.as_mut_ptr().add(d_arr_index_1) as *mut f32; - let stored_1 = _mm_loadu_ps(buf_ptr_1); + let pixel_color = load_f32::(s_ptr); - let buf_ptr_2 = bf0.as_mut_ptr().add(d_arr_index_2) as *mut f32; - let stored_2 = _mm_loadu_ps(buf_ptr_2); - - let new_diff = - _mm_opt_fnmlsf_ps::(stored_2, _mm_sub_ps(stored, stored_1), threes); - diffs = _mm_add_ps(diffs, new_diff); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; + let arr_index = ((y + 2 * radius_64) & 1023) as usize; let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); - - let buf_ptr_1 = bf0.get_unchecked_mut(arr_index_1).as_mut_ptr(); - let stored_1 = _mm_loadu_ps(buf_ptr_1); - diffs = _mm_opt_fmlaf_ps::(diffs, _mm_sub_ps(stored, stored_1), threes); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); - diffs = _mm_opt_fnmlaf_ps::(diffs, stored, threes); + diffs = _mm_add_ps(diffs, pixel_color); + ders = _mm_add_ps(ders, diffs); + summs = _mm_add_ps(summs, ders); + _mm_storeu_ps(buf_ptr, pixel_color); } - - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - let next_row_x = x * CN; - - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut f32; - - let pixel_color = load_f32::(s_ptr); - - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); - - diffs = _mm_add_ps(diffs, pixel_color); - ders = _mm_add_ps(ders, diffs); - summs = _mm_add_ps(summs, ders); - _mm_storeu_ps(buf_ptr, pixel_color); } } } -pub(crate) fn fgn_horizontal_pass_sse_f32( - undefined_slice: &UnsafeSlice, +pub(crate) fn fgn_horizontal_pass_sse_f32( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -314,21 +315,20 @@ pub(crate) fn fgn_horizontal_pass_sse_f32( edge_mode: EdgeMode, ) { unsafe { - let bytes: &UnsafeSlice<'_, f32> = std::mem::transmute(undefined_slice); if std::arch::is_x86_feature_detected!("fma") { fgn_horizontal_pass_sse_f32_fma::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } else { fgn_horizontal_pass_sse_f32_def::( - bytes, stride, width, height, radius, start, end, edge_mode, + slice, stride, width, height, radius, start, end, edge_mode, ); } } } #[target_feature(enable = "sse4.1")] -unsafe fn fgn_horizontal_pass_sse_f32_def( +fn fgn_horizontal_pass_sse_f32_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -344,7 +344,7 @@ unsafe fn fgn_horizontal_pass_sse_f32_def( } #[target_feature(enable = "sse4.1", enable = "fma")] -unsafe fn fgn_horizontal_pass_sse_f32_fma( +fn fgn_horizontal_pass_sse_f32_fma( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -360,7 +360,7 @@ unsafe fn fgn_horizontal_pass_sse_f32_fma( } #[inline(always)] -unsafe fn fgn_horizontal_pass_sse_f32_impl( +fn fgn_horizontal_pass_sse_f32_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -370,205 +370,207 @@ unsafe fn fgn_horizontal_pass_sse_f32_impl( end: u32, edge_mode: EdgeMode, ) { - let mut bf0 = Box::new([[0f32; 4]; 1024]); - let mut bf1 = Box::new([[0f32; 4]; 1024]); - let mut bf2 = Box::new([[0f32; 4]; 1024]); - - let width_wide = width as i64; - - let threes = _mm_set1_ps(3.); - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm_set1_ps(weight); - - let mut yy = start as usize; - - while yy + 3 < height.min(end) as usize { - let mut diffs0 = _mm_setzero_ps(); - let mut diffs1 = _mm_setzero_ps(); - let mut diffs2 = _mm_setzero_ps(); - - let mut ders0 = _mm_setzero_ps(); - let mut ders1 = _mm_setzero_ps(); - let mut ders2 = _mm_setzero_ps(); - - let mut summs0 = _mm_setzero_ps(); - let mut summs1 = _mm_setzero_ps(); - let mut summs2 = _mm_setzero_ps(); - - let start_x = 0 - 3 * radius_64; - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = (((yy + 1) as i64) * (stride as i64)) as usize; - let current_y2 = (((yy + 2) as i64) * (stride as i64)) as usize; - - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; - - let prepared_px0 = _mm_mul_ps(summs0, v_weight); - let prepared_px1 = _mm_mul_ps(summs1, v_weight); - let prepared_px2 = _mm_mul_ps(summs2, v_weight); - - let dst_ptr0 = bytes.slice.as_ptr().add(current_y0 + current_px) as *mut f32; - let dst_ptr1 = bytes.slice.as_ptr().add(current_y1 + current_px) as *mut f32; - let dst_ptr2 = bytes.slice.as_ptr().add(current_y2 + current_px) as *mut f32; - - store_f32::(dst_ptr0, prepared_px0); - store_f32::(dst_ptr1, prepared_px1); - store_f32::(dst_ptr2, prepared_px2); - - let d_a_1 = ((x + radius_64) & 1023) as usize; - let d_a_2 = ((x - radius_64) & 1023) as usize; - let d_i = (x & 1023) as usize; - - let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_i) as *mut f32); - let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_i) as *mut f32); - let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_i) as *mut f32); - - let sd_1_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_a_1) as *mut f32); - let sd_1_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_a_1) as *mut f32); - let sd_1_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_a_1) as *mut f32); - - let j0 = _mm_sub_ps(sd0, sd_1_0); - let j1 = _mm_sub_ps(sd1, sd_1_1); - let j2 = _mm_sub_ps(sd2, sd_1_2); - - let sd_2_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_a_2) as *mut f32); - let sd_2_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_a_2) as *mut f32); - let sd_2_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_a_2) as *mut f32); - - let new_diff0 = _mm_opt_fnmlsf_ps::(sd_2_0, j0, threes); - let new_diff1 = _mm_opt_fnmlsf_ps::(sd_2_1, j1, threes); - let new_diff2 = _mm_opt_fnmlsf_ps::(sd_2_2, j2, threes); - - diffs0 = _mm_add_ps(diffs0, new_diff0); - diffs1 = _mm_add_ps(diffs1, new_diff1); - diffs2 = _mm_add_ps(diffs2, new_diff2); - } else if x + radius_64 >= 0 { - let a_i = (x & 1023) as usize; - let a_i_1 = ((x + radius_64) & 1023) as usize; - let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(a_i) as *mut f32); - let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(a_i) as *mut f32); - let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(a_i) as *mut f32); - - let sd_1_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(a_i_1) as *mut f32); - let sd_1_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(a_i_1) as *mut f32); - let sd_1_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(a_i_1) as *mut f32); - - diffs0 = _mm_opt_fmlaf_ps::(diffs0, _mm_sub_ps(sd0, sd_1_0), threes); - diffs1 = _mm_opt_fmlaf_ps::(diffs1, _mm_sub_ps(sd1, sd_1_1), threes); - diffs2 = _mm_opt_fmlaf_ps::(diffs2, _mm_sub_ps(sd2, sd_1_2), threes); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); - let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); - let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); - - diffs0 = _mm_opt_fnmlaf_ps::(diffs0, sd0, threes); - diffs1 = _mm_opt_fnmlaf_ps::(diffs1, sd1, threes); - diffs2 = _mm_opt_fnmlaf_ps::(diffs2, sd2, threes); - } - - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut f32; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut f32; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut f32; + unsafe { + let mut bf0 = [[0f32; 4]; 1024]; + let mut bf1 = [[0f32; 4]; 1024]; + let mut bf2 = [[0f32; 4]; 1024]; + + let width_wide = width as i64; + + let threes = _mm_set1_ps(3.); + + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm_set1_ps(weight); + + let mut yy = start as usize; - let pixel_color0 = load_f32::(s_ptr0); - let pixel_color1 = load_f32::(s_ptr1); - let pixel_color2 = load_f32::(s_ptr2); + while yy + 3 <= height.min(end) as usize { + let mut diffs0 = _mm_setzero_ps(); + let mut diffs1 = _mm_setzero_ps(); + let mut diffs2 = _mm_setzero_ps(); - let a_i = ((x + 2 * radius_64) & 1023) as usize; + let mut ders0 = _mm_setzero_ps(); + let mut ders1 = _mm_setzero_ps(); + let mut ders2 = _mm_setzero_ps(); - _mm_storeu_ps(bf0.as_mut_ptr().add(a_i) as *mut f32, pixel_color0); - _mm_storeu_ps(bf1.as_mut_ptr().add(a_i) as *mut f32, pixel_color1); - _mm_storeu_ps(bf2.as_mut_ptr().add(a_i) as *mut f32, pixel_color2); + let mut summs0 = _mm_setzero_ps(); + let mut summs1 = _mm_setzero_ps(); + let mut summs2 = _mm_setzero_ps(); - diffs0 = _mm_add_ps(diffs0, pixel_color0); - diffs1 = _mm_add_ps(diffs1, pixel_color1); - diffs2 = _mm_add_ps(diffs2, pixel_color2); + let start_x = 0 - 3 * radius_64; + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = (((yy + 1) as i64) * (stride as i64)) as usize; + let current_y2 = (((yy + 2) as i64) * (stride as i64)) as usize; + + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; + + let prepared_px0 = _mm_mul_ps(summs0, v_weight); + let prepared_px1 = _mm_mul_ps(summs1, v_weight); + let prepared_px2 = _mm_mul_ps(summs2, v_weight); + + let dst_ptr0 = bytes.slice.as_ptr().add(current_y0 + current_px) as *mut f32; + let dst_ptr1 = bytes.slice.as_ptr().add(current_y1 + current_px) as *mut f32; + let dst_ptr2 = bytes.slice.as_ptr().add(current_y2 + current_px) as *mut f32; + + store_f32::(dst_ptr0, prepared_px0); + store_f32::(dst_ptr1, prepared_px1); + store_f32::(dst_ptr2, prepared_px2); + + let d_a_1 = ((x + radius_64) & 1023) as usize; + let d_a_2 = ((x - radius_64) & 1023) as usize; + let d_i = (x & 1023) as usize; + + let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_i) as *mut f32); + let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_i) as *mut f32); + let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_i) as *mut f32); + + let sd_1_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_a_1) as *mut f32); + let sd_1_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_a_1) as *mut f32); + let sd_1_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_a_1) as *mut f32); + + let j0 = _mm_sub_ps(sd0, sd_1_0); + let j1 = _mm_sub_ps(sd1, sd_1_1); + let j2 = _mm_sub_ps(sd2, sd_1_2); + + let sd_2_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(d_a_2) as *mut f32); + let sd_2_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(d_a_2) as *mut f32); + let sd_2_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(d_a_2) as *mut f32); + + let new_diff0 = _mm_opt_fnmlsf_ps::(sd_2_0, j0, threes); + let new_diff1 = _mm_opt_fnmlsf_ps::(sd_2_1, j1, threes); + let new_diff2 = _mm_opt_fnmlsf_ps::(sd_2_2, j2, threes); + + diffs0 = _mm_add_ps(diffs0, new_diff0); + diffs1 = _mm_add_ps(diffs1, new_diff1); + diffs2 = _mm_add_ps(diffs2, new_diff2); + } else if x + radius_64 >= 0 { + let a_i = (x & 1023) as usize; + let a_i_1 = ((x + radius_64) & 1023) as usize; + let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(a_i) as *mut f32); + let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(a_i) as *mut f32); + let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(a_i) as *mut f32); + + let sd_1_0 = _mm_loadu_ps(bf0.as_mut_ptr().add(a_i_1) as *mut f32); + let sd_1_1 = _mm_loadu_ps(bf1.as_mut_ptr().add(a_i_1) as *mut f32); + let sd_1_2 = _mm_loadu_ps(bf2.as_mut_ptr().add(a_i_1) as *mut f32); + + diffs0 = _mm_opt_fmlaf_ps::(diffs0, _mm_sub_ps(sd0, sd_1_0), threes); + diffs1 = _mm_opt_fmlaf_ps::(diffs1, _mm_sub_ps(sd1, sd_1_1), threes); + diffs2 = _mm_opt_fmlaf_ps::(diffs2, _mm_sub_ps(sd2, sd_1_2), threes); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let sd0 = _mm_loadu_ps(bf0.as_mut_ptr().add(arr_index) as *mut f32); + let sd1 = _mm_loadu_ps(bf1.as_mut_ptr().add(arr_index) as *mut f32); + let sd2 = _mm_loadu_ps(bf2.as_mut_ptr().add(arr_index) as *mut f32); + + diffs0 = _mm_opt_fnmlaf_ps::(diffs0, sd0, threes); + diffs1 = _mm_opt_fnmlaf_ps::(diffs1, sd1, threes); + diffs2 = _mm_opt_fnmlaf_ps::(diffs2, sd2, threes); + } + + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; - ders0 = _mm_add_ps(ders0, diffs0); - ders1 = _mm_add_ps(ders1, diffs1); - ders2 = _mm_add_ps(ders2, diffs2); + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut f32; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut f32; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut f32; + + let pixel_color0 = load_f32::(s_ptr0); + let pixel_color1 = load_f32::(s_ptr1); + let pixel_color2 = load_f32::(s_ptr2); + + let a_i = ((x + 2 * radius_64) & 1023) as usize; + + _mm_storeu_ps(bf0.as_mut_ptr().add(a_i) as *mut f32, pixel_color0); + _mm_storeu_ps(bf1.as_mut_ptr().add(a_i) as *mut f32, pixel_color1); + _mm_storeu_ps(bf2.as_mut_ptr().add(a_i) as *mut f32, pixel_color2); + + diffs0 = _mm_add_ps(diffs0, pixel_color0); + diffs1 = _mm_add_ps(diffs1, pixel_color1); + diffs2 = _mm_add_ps(diffs2, pixel_color2); + + ders0 = _mm_add_ps(ders0, diffs0); + ders1 = _mm_add_ps(ders1, diffs1); + ders2 = _mm_add_ps(ders2, diffs2); + + summs0 = _mm_add_ps(summs0, ders0); + summs1 = _mm_add_ps(summs1, ders1); + summs2 = _mm_add_ps(summs2, ders2); + } - summs0 = _mm_add_ps(summs0, ders0); - summs1 = _mm_add_ps(summs1, ders1); - summs2 = _mm_add_ps(summs2, ders2); + yy += 3; } - yy += 3; - } - - for y in yy..height.min(end) as usize { - let mut diffs = _mm_setzero_ps(); - let mut ders = _mm_setzero_ps(); - let mut summs = _mm_setzero_ps(); + for y in yy..height.min(end) as usize { + let mut diffs = _mm_setzero_ps(); + let mut ders = _mm_setzero_ps(); + let mut summs = _mm_setzero_ps(); - let current_y = ((y as i64) * (stride as i64)) as usize; + let current_y = ((y as i64) * (stride as i64)) as usize; - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let pixel = _mm_mul_ps(summs, v_weight); - let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; - store_f32::(dst_ptr, pixel); + let pixel = _mm_mul_ps(summs, v_weight); + let dst_ptr = bytes.slice.as_ptr().add(bytes_offset) as *mut f32; + store_f32::(dst_ptr, pixel); - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let stored = _mm_loadu_ps(buf_ptr); + let buf_ptr = bf0.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let stored = _mm_loadu_ps(buf_ptr); - let buf_ptr_1 = bf0.get_unchecked_mut(d_arr_index_1).as_mut_ptr(); - let stored_1 = _mm_loadu_ps(buf_ptr_1); + let buf_ptr_1 = bf0.get_unchecked_mut(d_arr_index_1).as_mut_ptr(); + let stored_1 = _mm_loadu_ps(buf_ptr_1); - let buf_ptr_2 = bf0.get_unchecked_mut(d_arr_index_2).as_mut_ptr(); - let stored_2 = _mm_loadu_ps(buf_ptr_2); + let buf_ptr_2 = bf0.get_unchecked_mut(d_arr_index_2).as_mut_ptr(); + let stored_2 = _mm_loadu_ps(buf_ptr_2); - let new_diff = - _mm_opt_fnmlsf_ps::(stored_2, _mm_sub_ps(stored, stored_1), threes); - diffs = _mm_add_ps(diffs, new_diff); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; - let stored = _mm_loadu_ps(buf_ptr); + let new_diff = + _mm_opt_fnmlsf_ps::(stored_2, _mm_sub_ps(stored, stored_1), threes); + diffs = _mm_add_ps(diffs, new_diff); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + let buf_ptr = bf0.as_mut_ptr().add(arr_index) as *mut f32; + let stored = _mm_loadu_ps(buf_ptr); - let buf_ptr_1 = bf0.as_mut_ptr().add(arr_index_1); - let stored_1 = _mm_loadu_ps(buf_ptr_1 as *const f32); + let buf_ptr_1 = bf0.as_mut_ptr().add(arr_index_1); + let stored_1 = _mm_loadu_ps(buf_ptr_1 as *const f32); - diffs = _mm_opt_fmlaf_ps::(diffs, _mm_sub_ps(stored, stored_1), threes); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = bf0.as_mut_ptr().add(arr_index); - let stored = _mm_loadu_ps(buf_ptr as *const f32); - diffs = _mm_opt_fnmlaf_ps::(diffs, stored, threes); - } + diffs = _mm_opt_fmlaf_ps::(diffs, _mm_sub_ps(stored, stored_1), threes); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = bf0.as_mut_ptr().add(arr_index); + let stored = _mm_loadu_ps(buf_ptr as *const f32); + diffs = _mm_opt_fnmlaf_ps::(diffs, stored, threes); + } - let next_row_y = y * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_y = y * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f32; + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut f32; - let pixel_color = load_f32::(s_ptr); + let pixel_color = load_f32::(s_ptr); - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr = bf0.get_unchecked_mut(arr_index).as_mut_ptr(); - diffs = _mm_add_ps(diffs, pixel_color); - ders = _mm_add_ps(ders, diffs); - summs = _mm_add_ps(summs, ders); - _mm_storeu_ps(buf_ptr, pixel_color); + diffs = _mm_add_ps(diffs, pixel_color); + ders = _mm_add_ps(ders, diffs); + summs = _mm_add_ps(summs, ders); + _mm_storeu_ps(buf_ptr, pixel_color); + } } } } diff --git a/src/sse/fast_gaussian_next_u16.rs b/src/sse/fast_gaussian_next_u16.rs index 339cc429..358be49e 100644 --- a/src/sse/fast_gaussian_next_u16.rs +++ b/src/sse/fast_gaussian_next_u16.rs @@ -25,12 +25,12 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::sse::fast_gaussian::SseI32x4; use crate::sse::utils::load_u16_s32_fast; use crate::sse::{_mm_mul_by_3_epi32, store_u16_u32}; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] @@ -54,7 +54,7 @@ pub(crate) fn fgn_vertical_pass_sse_u16( } #[target_feature(enable = "sse4.1")] -unsafe fn fgn_vertical_pass_sse_u16_def( +fn fgn_vertical_pass_sse_u16_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -70,7 +70,7 @@ unsafe fn fgn_vertical_pass_sse_u16_def( } #[inline(always)] -unsafe fn fgn_vertical_pass_sse_u16_impl( +fn fgn_vertical_pass_sse_u16_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -80,276 +80,295 @@ unsafe fn fgn_vertical_pass_sse_u16_impl( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([SseI32x4::default(); 1024 * 4]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, rem) = rem.split_at_mut(1024); - let (buffer2, buffer3) = rem.split_at_mut(1024); - - let height_wide = height as i64; - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm_set1_ps(weight); - - let mut xx = start; - - while xx + 4 < width.min(end) { - let mut diffs0 = _mm_setzero_si128(); - let mut diffs1 = _mm_setzero_si128(); - let mut diffs2 = _mm_setzero_si128(); - let mut diffs3 = _mm_setzero_si128(); - - let mut ders0 = _mm_setzero_si128(); - let mut ders1 = _mm_setzero_si128(); - let mut ders2 = _mm_setzero_si128(); - let mut ders3 = _mm_setzero_si128(); - - let mut summs0 = _mm_setzero_si128(); - let mut summs1 = _mm_setzero_si128(); - let mut summs2 = _mm_setzero_si128(); - let mut summs3 = _mm_setzero_si128(); - - let start_y = 0 - 3 * radius as i64; - - let current_px0 = (xx * CN as u32) as usize; - let current_px1 = ((xx + 1) * CN as u32) as usize; - let current_px2 = ((xx + 2) * CN as u32) as usize; - let current_px3 = ((xx + 3) * CN as u32) as usize; - - for y in start_y..height_wide { - if y >= 0 { - let ss0 = _mm_cvtepi32_ps(summs0); - let ss1 = _mm_cvtepi32_ps(summs1); - let ss2 = _mm_cvtepi32_ps(summs2); - let ss3 = _mm_cvtepi32_ps(summs3); - - let r0 = _mm_mul_ps(ss0, v_weight); - let r1 = _mm_mul_ps(ss1, v_weight); - let r2 = _mm_mul_ps(ss2, v_weight); - let r3 = _mm_mul_ps(ss3, v_weight); - - let prepared_px0 = _mm_cvtps_epi32(r0); - let prepared_px1 = _mm_cvtps_epi32(r1); - let prepared_px2 = _mm_cvtps_epi32(r2); - let prepared_px3 = _mm_cvtps_epi32(r3); - - let current_y = (y * (stride as i64)) as usize; - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px0); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px1); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px2); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px3); - - store_u16_u32::(dst_ptr0, prepared_px0); - store_u16_u32::(dst_ptr1, prepared_px1); - store_u16_u32::(dst_ptr2, prepared_px2); - store_u16_u32::(dst_ptr3, prepared_px3); - - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let stored0 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index) as *const _); - let stored1 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index) as *const _); - let stored2 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index) as *const _); - let stored3 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index) as *const _); - - let stored_10 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_11 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_12 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_13 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_1) as *const _); - - let j0 = _mm_sub_epi32(stored0, stored_10); - let j1 = _mm_sub_epi32(stored1, stored_11); - let j2 = _mm_sub_epi32(stored2, stored_12); - let j3 = _mm_sub_epi32(stored3, stored_13); - - let stored_20 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_21 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_22 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_23 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_2) as *const _); - - let k0 = _mm_mul_by_3_epi32(j0); - let k1 = _mm_mul_by_3_epi32(j1); - let k2 = _mm_mul_by_3_epi32(j2); - let k3 = _mm_mul_by_3_epi32(j3); - - let new_diff0 = _mm_sub_epi32(k0, stored_20); - let new_diff1 = _mm_sub_epi32(k1, stored_21); - let new_diff2 = _mm_sub_epi32(k2, stored_22); - let new_diff3 = _mm_sub_epi32(k3, stored_23); - - diffs0 = _mm_add_epi32(diffs0, new_diff0); - diffs1 = _mm_add_epi32(diffs1, new_diff1); - diffs2 = _mm_add_epi32(diffs2, new_diff2); - diffs3 = _mm_add_epi32(diffs3, new_diff3); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; - - let stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - let stored_10 = _mm_load_si128( - buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_11 = _mm_load_si128( - buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_12 = _mm_load_si128( - buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_13 = _mm_load_si128( - buffer3.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - - let new_diff0 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored0, stored_10)); - let new_diff1 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored1, stored_11)); - let new_diff2 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored2, stored_12)); - let new_diff3 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored3, stored_13)); - - diffs0 = _mm_add_epi32(diffs0, new_diff0); - diffs1 = _mm_add_epi32(diffs1, new_diff1); - diffs2 = _mm_add_epi32(diffs2, new_diff2); - diffs3 = _mm_add_epi32(diffs3, new_diff3); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - - let stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - diffs0 = _mm_sub_epi32(diffs0, _mm_mul_by_3_epi32(stored0)); - diffs1 = _mm_sub_epi32(diffs1, _mm_mul_by_3_epi32(stored1)); - diffs2 = _mm_sub_epi32(diffs2, _mm_mul_by_3_epi32(stored2)); - diffs3 = _mm_sub_epi32(diffs3, _mm_mul_by_3_epi32(stored3)); + unsafe { + let mut full_buffer = [SseI32x4::default(); 1024 * 4]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, rem) = rem.split_at_mut(1024); + let (buffer2, buffer3) = rem.split_at_mut(1024); + + let height_wide = height as i64; + + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm_set1_ps(weight); + + let mut xx = start; + + while xx + 4 <= width.min(end) { + let mut diffs0 = _mm_setzero_si128(); + let mut diffs1 = _mm_setzero_si128(); + let mut diffs2 = _mm_setzero_si128(); + let mut diffs3 = _mm_setzero_si128(); + + let mut ders0 = _mm_setzero_si128(); + let mut ders1 = _mm_setzero_si128(); + let mut ders2 = _mm_setzero_si128(); + let mut ders3 = _mm_setzero_si128(); + + let mut summs0 = _mm_setzero_si128(); + let mut summs1 = _mm_setzero_si128(); + let mut summs2 = _mm_setzero_si128(); + let mut summs3 = _mm_setzero_si128(); + + let start_y = 0 - 3 * radius as i64; + + let current_px0 = (xx * CN as u32) as usize; + let current_px1 = ((xx + 1) * CN as u32) as usize; + let current_px2 = ((xx + 2) * CN as u32) as usize; + let current_px3 = ((xx + 3) * CN as u32) as usize; + + for y in start_y..height_wide { + if y >= 0 { + let ss0 = _mm_cvtepi32_ps(summs0); + let ss1 = _mm_cvtepi32_ps(summs1); + let ss2 = _mm_cvtepi32_ps(summs2); + let ss3 = _mm_cvtepi32_ps(summs3); + + let r0 = _mm_mul_ps(ss0, v_weight); + let r1 = _mm_mul_ps(ss1, v_weight); + let r2 = _mm_mul_ps(ss2, v_weight); + let r3 = _mm_mul_ps(ss3, v_weight); + + let prepared_px0 = _mm_cvtps_epi32(r0); + let prepared_px1 = _mm_cvtps_epi32(r1); + let prepared_px2 = _mm_cvtps_epi32(r2); + let prepared_px3 = _mm_cvtps_epi32(r3); + + let current_y = (y * (stride as i64)) as usize; + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px0); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px1); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px2); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px3); + + store_u16_u32::(dst_ptr0, prepared_px0); + store_u16_u32::(dst_ptr1, prepared_px1); + store_u16_u32::(dst_ptr2, prepared_px2); + store_u16_u32::(dst_ptr3, prepared_px3); + + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let stored0 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index) as *const _); + let stored1 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index) as *const _); + let stored2 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index) as *const _); + let stored3 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index) as *const _); + + let stored_10 = + _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_11 = + _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_12 = + _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_13 = + _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_1) as *const _); + + let j0 = _mm_sub_epi32(stored0, stored_10); + let j1 = _mm_sub_epi32(stored1, stored_11); + let j2 = _mm_sub_epi32(stored2, stored_12); + let j3 = _mm_sub_epi32(stored3, stored_13); + + let stored_20 = + _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_21 = + _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_22 = + _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_23 = + _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_2) as *const _); + + let k0 = _mm_mul_by_3_epi32(j0); + let k1 = _mm_mul_by_3_epi32(j1); + let k2 = _mm_mul_by_3_epi32(j2); + let k3 = _mm_mul_by_3_epi32(j3); + + let new_diff0 = _mm_sub_epi32(k0, stored_20); + let new_diff1 = _mm_sub_epi32(k1, stored_21); + let new_diff2 = _mm_sub_epi32(k2, stored_22); + let new_diff3 = _mm_sub_epi32(k3, stored_23); + + diffs0 = _mm_add_epi32(diffs0, new_diff0); + diffs1 = _mm_add_epi32(diffs1, new_diff1); + diffs2 = _mm_add_epi32(diffs2, new_diff2); + diffs3 = _mm_add_epi32(diffs3, new_diff3); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + + let stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + let stored_10 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_11 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_12 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_13 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + + let new_diff0 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored0, stored_10)); + let new_diff1 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored1, stored_11)); + let new_diff2 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored2, stored_12)); + let new_diff3 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored3, stored_13)); + + diffs0 = _mm_add_epi32(diffs0, new_diff0); + diffs1 = _mm_add_epi32(diffs1, new_diff1); + diffs2 = _mm_add_epi32(diffs2, new_diff2); + diffs3 = _mm_add_epi32(diffs3, new_diff3); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + + let stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm_sub_epi32(diffs0, _mm_mul_by_3_epi32(stored0)); + diffs1 = _mm_sub_epi32(diffs1, _mm_mul_by_3_epi32(stored1)); + diffs2 = _mm_sub_epi32(diffs2, _mm_mul_by_3_epi32(stored2)); + diffs3 = _mm_sub_epi32(diffs3, _mm_mul_by_3_epi32(stored3)); + } + + let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u16; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u16; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u16; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u16; + + let pixel_color0 = load_u16_s32_fast::(s_ptr0); + let pixel_color1 = load_u16_s32_fast::(s_ptr1); + let pixel_color2 = load_u16_s32_fast::(s_ptr2); + let pixel_color3 = load_u16_s32_fast::(s_ptr3); + + let arr_index = ((y + 2 * radius_64) & 1023) as usize; + + let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr3 = buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr(); + + diffs0 = _mm_add_epi32(diffs0, pixel_color0); + diffs1 = _mm_add_epi32(diffs1, pixel_color1); + diffs2 = _mm_add_epi32(diffs2, pixel_color2); + diffs3 = _mm_add_epi32(diffs3, pixel_color3); + + _mm_store_si128(buf_ptr0 as *mut _, pixel_color0); + _mm_store_si128(buf_ptr1 as *mut _, pixel_color1); + _mm_store_si128(buf_ptr2 as *mut _, pixel_color2); + _mm_store_si128(buf_ptr3 as *mut _, pixel_color3); + + ders0 = _mm_add_epi32(ders0, diffs0); + ders1 = _mm_add_epi32(ders1, diffs1); + ders2 = _mm_add_epi32(ders2, diffs2); + ders3 = _mm_add_epi32(ders3, diffs3); + + summs0 = _mm_add_epi32(summs0, ders0); + summs1 = _mm_add_epi32(summs1, ders1); + summs2 = _mm_add_epi32(summs2, ders2); + summs3 = _mm_add_epi32(summs3, ders3); } - - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u16; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u16; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u16; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u16; - - let pixel_color0 = load_u16_s32_fast::(s_ptr0); - let pixel_color1 = load_u16_s32_fast::(s_ptr1); - let pixel_color2 = load_u16_s32_fast::(s_ptr2); - let pixel_color3 = load_u16_s32_fast::(s_ptr3); - - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - - let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr3 = buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr(); - - diffs0 = _mm_add_epi32(diffs0, pixel_color0); - diffs1 = _mm_add_epi32(diffs1, pixel_color1); - diffs2 = _mm_add_epi32(diffs2, pixel_color2); - diffs3 = _mm_add_epi32(diffs3, pixel_color3); - - _mm_store_si128(buf_ptr0 as *mut _, pixel_color0); - _mm_store_si128(buf_ptr1 as *mut _, pixel_color1); - _mm_store_si128(buf_ptr2 as *mut _, pixel_color2); - _mm_store_si128(buf_ptr3 as *mut _, pixel_color3); - - ders0 = _mm_add_epi32(ders0, diffs0); - ders1 = _mm_add_epi32(ders1, diffs1); - ders2 = _mm_add_epi32(ders2, diffs2); - ders3 = _mm_add_epi32(ders3, diffs3); - - summs0 = _mm_add_epi32(summs0, ders0); - summs1 = _mm_add_epi32(summs1, ders1); - summs2 = _mm_add_epi32(summs2, ders2); - summs3 = _mm_add_epi32(summs3, ders3); + xx += 4; } - xx += 4; - } - for x in xx..width.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut ders = _mm_setzero_si128(); - let mut summs = _mm_setzero_si128(); + for x in xx..width.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut ders = _mm_setzero_si128(); + let mut summs = _mm_setzero_si128(); - let start_y = 0 - 3 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let current_px = (x * CN as u32) as usize; - let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight)); + let start_y = 0 - 3 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let current_px = (x * CN as u32) as usize; + let prepared_px_s32 = + _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight)); - let current_y = (y * (stride as i64)) as usize; + let current_y = (y * (stride as i64)) as usize; - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); - store_u16_u32::(dst_ptr, prepared_px_s32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); + store_u16_u32::(dst_ptr, prepared_px_s32); - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.as_mut_ptr().add(d_arr_index_1); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.as_mut_ptr().add(d_arr_index_1); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let buf_ptr_2 = buffer0.as_mut_ptr().add(d_arr_index_2); - let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); + let buf_ptr_2 = buffer0.as_mut_ptr().add(d_arr_index_2); + let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); - let new_diff = _mm_sub_epi32( - _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), - stored_2, - ); - diffs = _mm_add_epi32(diffs, new_diff); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let new_diff = _mm_sub_epi32( + _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), + stored_2, + ); + diffs = _mm_add_epi32(diffs, new_diff); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr(); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr(); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); + let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); - diffs = _mm_add_epi32(diffs, new_diff); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); - } + diffs = _mm_add_epi32(diffs, new_diff); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); + diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); + } - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u16; + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u16; - let pixel_color = load_u16_s32_fast::(s_ptr); + let pixel_color = load_u16_s32_fast::(s_ptr); - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((y + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); - ders = _mm_add_epi32(ders, diffs); - summs = _mm_add_epi32(summs, ders); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + diffs = _mm_add_epi32(diffs, pixel_color); + ders = _mm_add_epi32(ders, diffs); + summs = _mm_add_epi32(summs, ders); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } @@ -372,7 +391,7 @@ pub(crate) fn fgn_horizontal_pass_sse_u16( } #[target_feature(enable = "sse4.1")] -unsafe fn fgn_horizontal_pass_sse_u16_def( +fn fgn_horizontal_pass_sse_u16_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -388,7 +407,7 @@ unsafe fn fgn_horizontal_pass_sse_u16_def( } #[inline(always)] -unsafe fn fgn_horizontal_pass_sse_u16_impl( +fn fgn_horizontal_pass_sse_u16_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -398,273 +417,292 @@ unsafe fn fgn_horizontal_pass_sse_u16_impl( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([SseI32x4::default(); 1024 * 4]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, rem) = rem.split_at_mut(1024); - let (buffer2, buffer3) = rem.split_at_mut(1024); - - let width_wide = width as i64; - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let v_weight = _mm_set1_ps(weight); - - let mut yy = start; - - while yy + 4 < height.min(end) { - let mut diffs0 = _mm_setzero_si128(); - let mut diffs1 = _mm_setzero_si128(); - let mut diffs2 = _mm_setzero_si128(); - let mut diffs3 = _mm_setzero_si128(); - - let mut ders0 = _mm_setzero_si128(); - let mut ders1 = _mm_setzero_si128(); - let mut ders2 = _mm_setzero_si128(); - let mut ders3 = _mm_setzero_si128(); - - let mut summs0 = _mm_setzero_si128(); - let mut summs1 = _mm_setzero_si128(); - let mut summs2 = _mm_setzero_si128(); - let mut summs3 = _mm_setzero_si128(); - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; - let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; - let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; - - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; - - let ss0 = _mm_cvtepi32_ps(summs0); - let ss1 = _mm_cvtepi32_ps(summs1); - let ss2 = _mm_cvtepi32_ps(summs2); - let ss3 = _mm_cvtepi32_ps(summs3); - - let r0 = _mm_mul_ps(ss0, v_weight); - let r1 = _mm_mul_ps(ss1, v_weight); - let r2 = _mm_mul_ps(ss2, v_weight); - let r3 = _mm_mul_ps(ss3, v_weight); - - let prepared_px0 = _mm_cvtps_epi32(r0); - let prepared_px1 = _mm_cvtps_epi32(r1); - let prepared_px2 = _mm_cvtps_epi32(r2); - let prepared_px3 = _mm_cvtps_epi32(r3); - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y0 + current_px); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y1 + current_px); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y2 + current_px); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y3 + current_px); - - store_u16_u32::(dst_ptr0, prepared_px0); - store_u16_u32::(dst_ptr1, prepared_px1); - store_u16_u32::(dst_ptr2, prepared_px2); - store_u16_u32::(dst_ptr3, prepared_px3); - - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let stored0 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index) as *const _); - let stored1 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index) as *const _); - let stored2 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index) as *const _); - let stored3 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index) as *const _); - - let stored_10 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_11 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_12 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); - let stored_13 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_1) as *const _); - - let j0 = _mm_sub_epi32(stored0, stored_10); - let j1 = _mm_sub_epi32(stored1, stored_11); - let j2 = _mm_sub_epi32(stored2, stored_12); - let j3 = _mm_sub_epi32(stored3, stored_13); - - let stored_20 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_21 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_22 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); - let stored_23 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_2) as *const _); - - let k0 = _mm_mul_by_3_epi32(j0); - let k1 = _mm_mul_by_3_epi32(j1); - let k2 = _mm_mul_by_3_epi32(j2); - let k3 = _mm_mul_by_3_epi32(j3); - - let new_diff0 = _mm_sub_epi32(k0, stored_20); - let new_diff1 = _mm_sub_epi32(k1, stored_21); - let new_diff2 = _mm_sub_epi32(k2, stored_22); - let new_diff3 = _mm_sub_epi32(k3, stored_23); - - diffs0 = _mm_add_epi32(diffs0, new_diff0); - diffs1 = _mm_add_epi32(diffs1, new_diff1); - diffs2 = _mm_add_epi32(diffs2, new_diff2); - diffs3 = _mm_add_epi32(diffs3, new_diff3); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - - let stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - let stored_10 = _mm_load_si128( - buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_11 = _mm_load_si128( - buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_12 = _mm_load_si128( - buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - let stored_13 = _mm_load_si128( - buffer3.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, - ); - - let new_diff0 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored0, stored_10)); - let new_diff1 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored1, stored_11)); - let new_diff2 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored2, stored_12)); - let new_diff3 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored3, stored_13)); - - diffs0 = _mm_add_epi32(diffs0, new_diff0); - diffs1 = _mm_add_epi32(diffs1, new_diff1); - diffs2 = _mm_add_epi32(diffs2, new_diff2); - diffs3 = _mm_add_epi32(diffs3, new_diff3); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - diffs0 = _mm_sub_epi32(diffs0, _mm_mul_by_3_epi32(stored0)); - diffs1 = _mm_sub_epi32(diffs1, _mm_mul_by_3_epi32(stored1)); - diffs2 = _mm_sub_epi32(diffs2, _mm_mul_by_3_epi32(stored2)); - diffs3 = _mm_sub_epi32(diffs3, _mm_mul_by_3_epi32(stored3)); + unsafe { + let mut full_buffer = [SseI32x4::default(); 1024 * 4]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, rem) = rem.split_at_mut(1024); + let (buffer2, buffer3) = rem.split_at_mut(1024); + + let width_wide = width as i64; + + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let v_weight = _mm_set1_ps(weight); + + let mut yy = start; + + while yy + 4 <= height.min(end) { + let mut diffs0 = _mm_setzero_si128(); + let mut diffs1 = _mm_setzero_si128(); + let mut diffs2 = _mm_setzero_si128(); + let mut diffs3 = _mm_setzero_si128(); + + let mut ders0 = _mm_setzero_si128(); + let mut ders1 = _mm_setzero_si128(); + let mut ders2 = _mm_setzero_si128(); + let mut ders3 = _mm_setzero_si128(); + + let mut summs0 = _mm_setzero_si128(); + let mut summs1 = _mm_setzero_si128(); + let mut summs2 = _mm_setzero_si128(); + let mut summs3 = _mm_setzero_si128(); + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; + let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; + let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; + + let ss0 = _mm_cvtepi32_ps(summs0); + let ss1 = _mm_cvtepi32_ps(summs1); + let ss2 = _mm_cvtepi32_ps(summs2); + let ss3 = _mm_cvtepi32_ps(summs3); + + let r0 = _mm_mul_ps(ss0, v_weight); + let r1 = _mm_mul_ps(ss1, v_weight); + let r2 = _mm_mul_ps(ss2, v_weight); + let r3 = _mm_mul_ps(ss3, v_weight); + + let prepared_px0 = _mm_cvtps_epi32(r0); + let prepared_px1 = _mm_cvtps_epi32(r1); + let prepared_px2 = _mm_cvtps_epi32(r2); + let prepared_px3 = _mm_cvtps_epi32(r3); + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y0 + current_px); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y1 + current_px); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y2 + current_px); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y3 + current_px); + + store_u16_u32::(dst_ptr0, prepared_px0); + store_u16_u32::(dst_ptr1, prepared_px1); + store_u16_u32::(dst_ptr2, prepared_px2); + store_u16_u32::(dst_ptr3, prepared_px3); + + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let stored0 = _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index) as *const _); + let stored1 = _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index) as *const _); + let stored2 = _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index) as *const _); + let stored3 = _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index) as *const _); + + let stored_10 = + _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_11 = + _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_12 = + _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_1) as *const _); + let stored_13 = + _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_1) as *const _); + + let j0 = _mm_sub_epi32(stored0, stored_10); + let j1 = _mm_sub_epi32(stored1, stored_11); + let j2 = _mm_sub_epi32(stored2, stored_12); + let j3 = _mm_sub_epi32(stored3, stored_13); + + let stored_20 = + _mm_load_si128(buffer0.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_21 = + _mm_load_si128(buffer1.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_22 = + _mm_load_si128(buffer2.as_mut_ptr().add(d_arr_index_2) as *const _); + let stored_23 = + _mm_load_si128(buffer3.as_mut_ptr().add(d_arr_index_2) as *const _); + + let k0 = _mm_mul_by_3_epi32(j0); + let k1 = _mm_mul_by_3_epi32(j1); + let k2 = _mm_mul_by_3_epi32(j2); + let k3 = _mm_mul_by_3_epi32(j3); + + let new_diff0 = _mm_sub_epi32(k0, stored_20); + let new_diff1 = _mm_sub_epi32(k1, stored_21); + let new_diff2 = _mm_sub_epi32(k2, stored_22); + let new_diff3 = _mm_sub_epi32(k3, stored_23); + + diffs0 = _mm_add_epi32(diffs0, new_diff0); + diffs1 = _mm_add_epi32(diffs1, new_diff1); + diffs2 = _mm_add_epi32(diffs2, new_diff2); + diffs3 = _mm_add_epi32(diffs3, new_diff3); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + + let stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + let stored_10 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_11 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_12 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + let stored_13 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index_1).0.as_mut_ptr() as *const _, + ); + + let new_diff0 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored0, stored_10)); + let new_diff1 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored1, stored_11)); + let new_diff2 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored2, stored_12)); + let new_diff3 = _mm_mul_by_3_epi32(_mm_sub_epi32(stored3, stored_13)); + + diffs0 = _mm_add_epi32(diffs0, new_diff0); + diffs1 = _mm_add_epi32(diffs1, new_diff1); + diffs2 = _mm_add_epi32(diffs2, new_diff2); + diffs3 = _mm_add_epi32(diffs3, new_diff3); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm_sub_epi32(diffs0, _mm_mul_by_3_epi32(stored0)); + diffs1 = _mm_sub_epi32(diffs1, _mm_mul_by_3_epi32(stored1)); + diffs2 = _mm_sub_epi32(diffs2, _mm_mul_by_3_epi32(stored2)); + diffs3 = _mm_sub_epi32(diffs3, _mm_mul_by_3_epi32(stored3)); + } + + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u16; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u16; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u16; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u16; + + let pixel_color0 = load_u16_s32_fast::(s_ptr0); + let pixel_color1 = load_u16_s32_fast::(s_ptr1); + let pixel_color2 = load_u16_s32_fast::(s_ptr2); + let pixel_color3 = load_u16_s32_fast::(s_ptr3); + + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let buf_ptr3 = buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr(); + + diffs0 = _mm_add_epi32(diffs0, pixel_color0); + diffs1 = _mm_add_epi32(diffs1, pixel_color1); + diffs2 = _mm_add_epi32(diffs2, pixel_color2); + diffs3 = _mm_add_epi32(diffs3, pixel_color3); + + _mm_store_si128(buf_ptr0 as *mut _, pixel_color0); + _mm_store_si128(buf_ptr1 as *mut _, pixel_color1); + _mm_store_si128(buf_ptr2 as *mut _, pixel_color2); + _mm_store_si128(buf_ptr3 as *mut _, pixel_color3); + + ders0 = _mm_add_epi32(ders0, diffs0); + ders1 = _mm_add_epi32(ders1, diffs1); + ders2 = _mm_add_epi32(ders2, diffs2); + ders3 = _mm_add_epi32(ders3, diffs3); + + summs0 = _mm_add_epi32(summs0, ders0); + summs1 = _mm_add_epi32(summs1, ders1); + summs2 = _mm_add_epi32(summs2, ders2); + summs3 = _mm_add_epi32(summs3, ders3); } - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u16; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u16; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u16; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u16; - - let pixel_color0 = load_u16_s32_fast::(s_ptr0); - let pixel_color1 = load_u16_s32_fast::(s_ptr1); - let pixel_color2 = load_u16_s32_fast::(s_ptr2); - let pixel_color3 = load_u16_s32_fast::(s_ptr3); - - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr0 = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr1 = buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr2 = buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let buf_ptr3 = buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr(); - - diffs0 = _mm_add_epi32(diffs0, pixel_color0); - diffs1 = _mm_add_epi32(diffs1, pixel_color1); - diffs2 = _mm_add_epi32(diffs2, pixel_color2); - diffs3 = _mm_add_epi32(diffs3, pixel_color3); - - _mm_store_si128(buf_ptr0 as *mut _, pixel_color0); - _mm_store_si128(buf_ptr1 as *mut _, pixel_color1); - _mm_store_si128(buf_ptr2 as *mut _, pixel_color2); - _mm_store_si128(buf_ptr3 as *mut _, pixel_color3); - - ders0 = _mm_add_epi32(ders0, diffs0); - ders1 = _mm_add_epi32(ders1, diffs1); - ders2 = _mm_add_epi32(ders2, diffs2); - ders3 = _mm_add_epi32(ders3, diffs3); - - summs0 = _mm_add_epi32(summs0, ders0); - summs1 = _mm_add_epi32(summs1, ders1); - summs2 = _mm_add_epi32(summs2, ders2); - summs3 = _mm_add_epi32(summs3, ders3); + yy += 4; } - yy += 4; - } - - for y in yy..height.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut ders = _mm_setzero_si128(); - let mut summs = _mm_setzero_si128(); + for y in yy..height.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut ders = _mm_setzero_si128(); + let mut summs = _mm_setzero_si128(); - let current_y = ((y as i64) * (stride as i64)) as usize; + let current_y = ((y as i64) * (stride as i64)) as usize; - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; - let prepared_px_s32 = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight)); + let prepared_px_s32 = + _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight)); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); - store_u16_u32::(dst_ptr, prepared_px_s32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); + store_u16_u32::(dst_ptr, prepared_px_s32); - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.get_unchecked_mut(d_arr_index_1).0.as_mut_ptr(); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.get_unchecked_mut(d_arr_index_1).0.as_mut_ptr(); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let buf_ptr_2 = buffer0.get_unchecked_mut(d_arr_index_2).0.as_mut_ptr(); - let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); + let buf_ptr_2 = buffer0.get_unchecked_mut(d_arr_index_2).0.as_mut_ptr(); + let stored_2 = _mm_load_si128(buf_ptr_2 as *const __m128i); - let new_diff = _mm_sub_epi32( - _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), - stored_2, - ); - diffs = _mm_add_epi32(diffs, new_diff); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; - let stored = _mm_load_si128(buf_ptr as *const __m128i); + let new_diff = _mm_sub_epi32( + _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)), + stored_2, + ); + diffs = _mm_add_epi32(diffs, new_diff); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; + let stored = _mm_load_si128(buf_ptr as *const __m128i); - let buf_ptr_1 = buffer0.as_mut_ptr().add(arr_index_1); - let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); + let buf_ptr_1 = buffer0.as_mut_ptr().add(arr_index_1); + let stored_1 = _mm_load_si128(buf_ptr_1 as *const __m128i); - let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); + let new_diff = _mm_mul_by_3_epi32(_mm_sub_epi32(stored, stored_1)); - diffs = _mm_add_epi32(diffs, new_diff); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.as_mut_ptr().add(arr_index); - let stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); - } + diffs = _mm_add_epi32(diffs, new_diff); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.as_mut_ptr().add(arr_index); + let stored = _mm_load_si128(buf_ptr as *const __m128i); + diffs = _mm_sub_epi32(diffs, _mm_mul_by_3_epi32(stored)); + } - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u16; + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u16; - let pixel_color = load_u16_s32_fast::(s_ptr); + let pixel_color = load_u16_s32_fast::(s_ptr); - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); - ders = _mm_add_epi32(ders, diffs); - summs = _mm_add_epi32(summs, ders); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + diffs = _mm_add_epi32(diffs, pixel_color); + ders = _mm_add_epi32(ders, diffs); + summs = _mm_add_epi32(summs, ders); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } diff --git a/src/sse/fast_gaussian_u16.rs b/src/sse/fast_gaussian_u16.rs index 5329715c..80bf33e6 100644 --- a/src/sse/fast_gaussian_u16.rs +++ b/src/sse/fast_gaussian_u16.rs @@ -25,12 +25,12 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::sse::fast_gaussian::SseI32x4; use crate::sse::store_u16_u32; use crate::sse::utils::load_u16_s32_fast; use crate::unsafe_slice::UnsafeSlice; -use crate::EdgeMode; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] @@ -54,7 +54,7 @@ pub(crate) fn fg_horizontal_pass_sse_u16( } #[target_feature(enable = "sse4.1")] -unsafe fn fg_horizontal_pass_sse_u16_def( +fn fg_horizontal_pass_sse_u16_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -70,7 +70,7 @@ unsafe fn fg_horizontal_pass_sse_u16_def( } #[inline(always)] -unsafe fn fg_horizontal_pass_sse_u16_impl( +fn fg_horizontal_pass_sse_u16_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -80,221 +80,231 @@ unsafe fn fg_horizontal_pass_sse_u16_impl( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([SseI32x4::default(); 1024 * 4]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, rem) = rem.split_at_mut(1024); - let (buffer2, buffer3) = rem.split_at_mut(1024); - - let initial_sum = ((radius * radius) >> 1) as i32; - - let radius_64 = radius as i64; - let width_wide = width as i64; - - let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let mut yy = start; - - while yy + 4 < height.min(end) { - let mut diffs0 = _mm_setzero_si128(); - let mut diffs1 = _mm_setzero_si128(); - let mut diffs2 = _mm_setzero_si128(); - let mut diffs3 = _mm_setzero_si128(); - - let mut summs0 = _mm_set1_epi32(initial_sum); - let mut summs1 = _mm_set1_epi32(initial_sum); - let mut summs2 = _mm_set1_epi32(initial_sum); - let mut summs3 = _mm_set1_epi32(initial_sum); - - let current_y0 = ((yy as i64) * (stride as i64)) as usize; - let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; - let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; - let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; - - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = (x as u32 * CN as u32) as usize; - - let ss0 = _mm_cvtepi32_ps(summs0); - let ss1 = _mm_cvtepi32_ps(summs1); - let ss2 = _mm_cvtepi32_ps(summs2); - let ss3 = _mm_cvtepi32_ps(summs3); - - let r0 = _mm_mul_ps(ss0, v_weight); - let r1 = _mm_mul_ps(ss1, v_weight); - let r2 = _mm_mul_ps(ss2, v_weight); - let r3 = _mm_mul_ps(ss3, v_weight); - - let prepared_px0 = _mm_cvtps_epi32(r0); - let prepared_px1 = _mm_cvtps_epi32(r1); - let prepared_px2 = _mm_cvtps_epi32(r2); - let prepared_px3 = _mm_cvtps_epi32(r3); - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y0 + current_px); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y1 + current_px); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y2 + current_px); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y3 + current_px); - - store_u16_u32::(dst_ptr0, prepared_px0); - store_u16_u32::(dst_ptr1, prepared_px1); - store_u16_u32::(dst_ptr2, prepared_px2); - store_u16_u32::(dst_ptr3, prepared_px3); - - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let mut d_stored0 = _mm_load_si128( - buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + unsafe { + let mut full_buffer = [SseI32x4::default(); 1024 * 4]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, rem) = rem.split_at_mut(1024); + let (buffer2, buffer3) = rem.split_at_mut(1024); + + let initial_sum = ((radius * radius) >> 1) as i32; + + let radius_64 = radius as i64; + let width_wide = width as i64; + + let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let mut yy = start; + + while yy + 4 <= height.min(end) { + let mut diffs0 = _mm_setzero_si128(); + let mut diffs1 = _mm_setzero_si128(); + let mut diffs2 = _mm_setzero_si128(); + let mut diffs3 = _mm_setzero_si128(); + + let mut summs0 = _mm_set1_epi32(initial_sum); + let mut summs1 = _mm_set1_epi32(initial_sum); + let mut summs2 = _mm_set1_epi32(initial_sum); + let mut summs3 = _mm_set1_epi32(initial_sum); + + let current_y0 = ((yy as i64) * (stride as i64)) as usize; + let current_y1 = ((yy as i64 + 1) * (stride as i64)) as usize; + let current_y2 = ((yy as i64 + 2) * (stride as i64)) as usize; + let current_y3 = ((yy as i64 + 3) * (stride as i64)) as usize; + + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = (x as u32 * CN as u32) as usize; + + let ss0 = _mm_cvtepi32_ps(summs0); + let ss1 = _mm_cvtepi32_ps(summs1); + let ss2 = _mm_cvtepi32_ps(summs2); + let ss3 = _mm_cvtepi32_ps(summs3); + + let r0 = _mm_mul_ps(ss0, v_weight); + let r1 = _mm_mul_ps(ss1, v_weight); + let r2 = _mm_mul_ps(ss2, v_weight); + let r3 = _mm_mul_ps(ss3, v_weight); + + let prepared_px0 = _mm_cvtps_epi32(r0); + let prepared_px1 = _mm_cvtps_epi32(r1); + let prepared_px2 = _mm_cvtps_epi32(r2); + let prepared_px3 = _mm_cvtps_epi32(r3); + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y0 + current_px); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y1 + current_px); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y2 + current_px); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y3 + current_px); + + store_u16_u32::(dst_ptr0, prepared_px0); + store_u16_u32::(dst_ptr1, prepared_px1); + store_u16_u32::(dst_ptr2, prepared_px2); + store_u16_u32::(dst_ptr3, prepared_px3); + + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let mut d_stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + + d_stored0 = _mm_slli_epi32::<1>(d_stored0); + d_stored1 = _mm_slli_epi32::<1>(d_stored1); + d_stored2 = _mm_slli_epi32::<1>(d_stored2); + d_stored3 = _mm_slli_epi32::<1>(d_stored3); + + let a_stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm_add_epi32(diffs0, _mm_sub_epi32(a_stored0, d_stored0)); + diffs1 = _mm_add_epi32(diffs1, _mm_sub_epi32(a_stored1, d_stored1)); + diffs2 = _mm_add_epi32(diffs2, _mm_sub_epi32(a_stored2, d_stored2)); + diffs3 = _mm_add_epi32(diffs3, _mm_sub_epi32(a_stored3, d_stored3)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let mut stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let mut stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let mut stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let mut stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + stored0 = _mm_slli_epi32::<1>(stored0); + stored1 = _mm_slli_epi32::<1>(stored1); + stored2 = _mm_slli_epi32::<1>(stored2); + stored3 = _mm_slli_epi32::<1>(stored3); + + diffs0 = _mm_sub_epi32(diffs0, stored0); + diffs1 = _mm_sub_epi32(diffs1, stored1); + diffs2 = _mm_sub_epi32(diffs2, stored2); + diffs3 = _mm_sub_epi32(diffs3, stored3); + } + + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u16; + let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u16; + let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u16; + let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u16; + + let pixel_color0 = load_u16_s32_fast::(s_ptr0); + let pixel_color1 = load_u16_s32_fast::(s_ptr1); + let pixel_color2 = load_u16_s32_fast::(s_ptr2); + let pixel_color3 = load_u16_s32_fast::(s_ptr3); + + let arr_index = ((x + radius_64) & 1023) as usize; + + _mm_store_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color0, ); - let mut d_stored1 = _mm_load_si128( - buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color1, ); - let mut d_stored2 = _mm_load_si128( - buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color2, ); - let mut d_stored3 = _mm_load_si128( - buffer3.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color3, ); - d_stored0 = _mm_slli_epi32::<1>(d_stored0); - d_stored1 = _mm_slli_epi32::<1>(d_stored1); - d_stored2 = _mm_slli_epi32::<1>(d_stored2); - d_stored3 = _mm_slli_epi32::<1>(d_stored3); - - let a_stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - diffs0 = _mm_add_epi32(diffs0, _mm_sub_epi32(a_stored0, d_stored0)); - diffs1 = _mm_add_epi32(diffs1, _mm_sub_epi32(a_stored1, d_stored1)); - diffs2 = _mm_add_epi32(diffs2, _mm_sub_epi32(a_stored2, d_stored2)); - diffs3 = _mm_add_epi32(diffs3, _mm_sub_epi32(a_stored3, d_stored3)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let mut stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let mut stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let mut stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let mut stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - stored0 = _mm_slli_epi32::<1>(stored0); - stored1 = _mm_slli_epi32::<1>(stored1); - stored2 = _mm_slli_epi32::<1>(stored2); - stored3 = _mm_slli_epi32::<1>(stored3); - - diffs0 = _mm_sub_epi32(diffs0, stored0); - diffs1 = _mm_sub_epi32(diffs1, stored1); - diffs2 = _mm_sub_epi32(diffs2, stored2); - diffs3 = _mm_sub_epi32(diffs3, stored3); + diffs0 = _mm_add_epi32(diffs0, pixel_color0); + diffs1 = _mm_add_epi32(diffs1, pixel_color1); + diffs2 = _mm_add_epi32(diffs2, pixel_color2); + diffs3 = _mm_add_epi32(diffs3, pixel_color3); + + summs0 = _mm_add_epi32(summs0, diffs0); + summs1 = _mm_add_epi32(summs1, diffs1); + summs2 = _mm_add_epi32(summs2, diffs2); + summs3 = _mm_add_epi32(summs3, diffs3); } - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr0 = bytes.slice.as_ptr().add(current_y0 + next_row_px) as *mut u16; - let s_ptr1 = bytes.slice.as_ptr().add(current_y1 + next_row_px) as *mut u16; - let s_ptr2 = bytes.slice.as_ptr().add(current_y2 + next_row_px) as *mut u16; - let s_ptr3 = bytes.slice.as_ptr().add(current_y3 + next_row_px) as *mut u16; - - let pixel_color0 = load_u16_s32_fast::(s_ptr0); - let pixel_color1 = load_u16_s32_fast::(s_ptr1); - let pixel_color2 = load_u16_s32_fast::(s_ptr2); - let pixel_color3 = load_u16_s32_fast::(s_ptr3); - - let arr_index = ((x + radius_64) & 1023) as usize; - - _mm_store_si128( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color0, - ); - _mm_store_si128( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color1, - ); - _mm_store_si128( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color2, - ); - _mm_store_si128( - buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color3, - ); - - diffs0 = _mm_add_epi32(diffs0, pixel_color0); - diffs1 = _mm_add_epi32(diffs1, pixel_color1); - diffs2 = _mm_add_epi32(diffs2, pixel_color2); - diffs3 = _mm_add_epi32(diffs3, pixel_color3); - - summs0 = _mm_add_epi32(summs0, diffs0); - summs1 = _mm_add_epi32(summs1, diffs1); - summs2 = _mm_add_epi32(summs2, diffs2); - summs3 = _mm_add_epi32(summs3, diffs3); + yy += 4; } - yy += 4; - } - - for y in yy..height.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut summs = _mm_set1_epi32(initial_sum); + for y in yy..height.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut summs = _mm_set1_epi32(initial_sum); - let current_y = ((y as i64) * (stride as i64)) as usize; + let current_y = ((y as i64) * (stride as i64)) as usize; - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = (x as u32 * CN as u32) as usize; + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = (x as u32 * CN as u32) as usize; - let pixel_f32 = _mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight); - let pixel_u32 = _mm_cvtps_epi32(pixel_f32); + let pixel_f32 = _mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight); + let pixel_u32 = _mm_cvtps_epi32(pixel_f32); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); - store_u16_u32::(dst_ptr, pixel_u32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); + store_u16_u32::(dst_ptr, pixel_u32); - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let d_buf_ptr = buffer0.as_mut_ptr().add(d_arr_index) as *const i32; - let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); - d_stored = _mm_slli_epi32::<1>(d_stored); + let d_buf_ptr = buffer0.as_mut_ptr().add(d_arr_index) as *const i32; + let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); + d_stored = _mm_slli_epi32::<1>(d_stored); - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let a_stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let a_stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let mut stored = _mm_load_si128(buf_ptr as *const __m128i); - stored = _mm_slli_epi32::<1>(stored); - diffs = _mm_sub_epi32(diffs, stored); - } + diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let mut stored = _mm_load_si128(buf_ptr as *const __m128i); + stored = _mm_slli_epi32::<1>(stored); + diffs = _mm_sub_epi32(diffs, stored); + } - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u16; - let pixel_color = load_u16_s32_fast::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u16; + let pixel_color = load_u16_s32_fast::(s_ptr); - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); - summs = _mm_add_epi32(summs, diffs); + diffs = _mm_add_epi32(diffs, pixel_color); + summs = _mm_add_epi32(summs, diffs); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + } } } } @@ -317,7 +327,7 @@ pub(crate) fn fg_vertical_pass_sse_u16( } #[target_feature(enable = "sse4.1")] -unsafe fn fg_vertical_pass_sse_u16_def( +fn fg_vertical_pass_sse_u16_def( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -333,7 +343,7 @@ unsafe fn fg_vertical_pass_sse_u16_def( } #[inline(always)] -unsafe fn fg_vertical_pass_sse_u16_impl( +fn fg_vertical_pass_sse_u16_impl( bytes: &UnsafeSlice, stride: u32, width: u32, @@ -343,226 +353,236 @@ unsafe fn fg_vertical_pass_sse_u16_impl( end: u32, edge_mode: EdgeMode, ) { - let mut full_buffer = Box::new([SseI32x4::default(); 1024 * 4]); - - let (buffer0, rem) = full_buffer.split_at_mut(1024); - let (buffer1, rem) = rem.split_at_mut(1024); - let (buffer2, buffer3) = rem.split_at_mut(1024); - - let initial_sum = ((radius * radius) >> 1) as i32; - - let height_wide = height as i64; - - let radius_64 = radius as i64; - - let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); - - let mut xx = start; - - while xx + 4 < width.min(end) { - let mut diffs0 = _mm_setzero_si128(); - let mut diffs1 = _mm_setzero_si128(); - let mut diffs2 = _mm_setzero_si128(); - let mut diffs3 = _mm_setzero_si128(); - - let mut summs0 = _mm_set1_epi32(initial_sum); - let mut summs1 = _mm_set1_epi32(initial_sum); - let mut summs2 = _mm_set1_epi32(initial_sum); - let mut summs3 = _mm_set1_epi32(initial_sum); - - let start_y = 0 - 2 * radius as i64; - - let current_px0 = (xx * CN as u32) as usize; - let current_px1 = ((xx + 1) * CN as u32) as usize; - let current_px2 = ((xx + 2) * CN as u32) as usize; - let current_px3 = ((xx + 3) * CN as u32) as usize; - - for y in start_y..height_wide { - if y >= 0 { - let ss0 = _mm_cvtepi32_ps(summs0); - let ss1 = _mm_cvtepi32_ps(summs1); - let ss2 = _mm_cvtepi32_ps(summs2); - let ss3 = _mm_cvtepi32_ps(summs3); - - let r0 = _mm_mul_ps(ss0, v_weight); - let r1 = _mm_mul_ps(ss1, v_weight); - let r2 = _mm_mul_ps(ss2, v_weight); - let r3 = _mm_mul_ps(ss3, v_weight); - - let prepared_px0 = _mm_cvtps_epi32(r0); - let prepared_px1 = _mm_cvtps_epi32(r1); - let prepared_px2 = _mm_cvtps_epi32(r2); - let prepared_px3 = _mm_cvtps_epi32(r3); - - let current_y = (y * (stride as i64)) as usize; - - let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px0); - let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px1); - let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px2); - let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px3); - - store_u16_u32::(dst_ptr0, prepared_px0); - store_u16_u32::(dst_ptr1, prepared_px1); - store_u16_u32::(dst_ptr2, prepared_px2); - store_u16_u32::(dst_ptr3, prepared_px3); - - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let mut d_stored0 = _mm_load_si128( - buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + unsafe { + let mut full_buffer = [SseI32x4::default(); 1024 * 4]; + + let (buffer0, rem) = full_buffer.split_at_mut(1024); + let (buffer1, rem) = rem.split_at_mut(1024); + let (buffer2, buffer3) = rem.split_at_mut(1024); + + let initial_sum = ((radius * radius) >> 1) as i32; + + let height_wide = height as i64; + + let radius_64 = radius as i64; + + let v_weight = _mm_set1_ps(1f32 / (radius as f32 * radius as f32)); + + let mut xx = start; + + while xx + 4 <= width.min(end) { + let mut diffs0 = _mm_setzero_si128(); + let mut diffs1 = _mm_setzero_si128(); + let mut diffs2 = _mm_setzero_si128(); + let mut diffs3 = _mm_setzero_si128(); + + let mut summs0 = _mm_set1_epi32(initial_sum); + let mut summs1 = _mm_set1_epi32(initial_sum); + let mut summs2 = _mm_set1_epi32(initial_sum); + let mut summs3 = _mm_set1_epi32(initial_sum); + + let start_y = 0 - 2 * radius as i64; + + let current_px0 = (xx * CN as u32) as usize; + let current_px1 = ((xx + 1) * CN as u32) as usize; + let current_px2 = ((xx + 2) * CN as u32) as usize; + let current_px3 = ((xx + 3) * CN as u32) as usize; + + for y in start_y..height_wide { + if y >= 0 { + let ss0 = _mm_cvtepi32_ps(summs0); + let ss1 = _mm_cvtepi32_ps(summs1); + let ss2 = _mm_cvtepi32_ps(summs2); + let ss3 = _mm_cvtepi32_ps(summs3); + + let r0 = _mm_mul_ps(ss0, v_weight); + let r1 = _mm_mul_ps(ss1, v_weight); + let r2 = _mm_mul_ps(ss2, v_weight); + let r3 = _mm_mul_ps(ss3, v_weight); + + let prepared_px0 = _mm_cvtps_epi32(r0); + let prepared_px1 = _mm_cvtps_epi32(r1); + let prepared_px2 = _mm_cvtps_epi32(r2); + let prepared_px3 = _mm_cvtps_epi32(r3); + + let current_y = (y * (stride as i64)) as usize; + + let dst_ptr0 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px0); + let dst_ptr1 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px1); + let dst_ptr2 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px2); + let dst_ptr3 = (bytes.slice.as_ptr() as *mut u16).add(current_y + current_px3); + + store_u16_u32::(dst_ptr0, prepared_px0); + store_u16_u32::(dst_ptr1, prepared_px1); + store_u16_u32::(dst_ptr2, prepared_px2); + store_u16_u32::(dst_ptr3, prepared_px3); + + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; + + let mut d_stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + let mut d_stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + ); + + d_stored0 = _mm_slli_epi32::<1>(d_stored0); + d_stored1 = _mm_slli_epi32::<1>(d_stored1); + d_stored2 = _mm_slli_epi32::<1>(d_stored2); + d_stored3 = _mm_slli_epi32::<1>(d_stored3); + + let a_stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + let a_stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _, + ); + + diffs0 = _mm_add_epi32(diffs0, _mm_sub_epi32(a_stored0, d_stored0)); + diffs1 = _mm_add_epi32(diffs1, _mm_sub_epi32(a_stored1, d_stored1)); + diffs2 = _mm_add_epi32(diffs2, _mm_sub_epi32(a_stored2, d_stored2)); + diffs3 = _mm_add_epi32(diffs3, _mm_sub_epi32(a_stored3, d_stored3)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + + let mut stored0 = _mm_load_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + let mut stored1 = _mm_load_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + let mut stored2 = _mm_load_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + let mut stored3 = _mm_load_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + ); + + stored0 = _mm_slli_epi32::<1>(stored0); + stored1 = _mm_slli_epi32::<1>(stored1); + stored2 = _mm_slli_epi32::<1>(stored2); + stored3 = _mm_slli_epi32::<1>(stored3); + + diffs0 = _mm_sub_epi32(diffs0, stored0); + diffs1 = _mm_sub_epi32(diffs1, stored1); + diffs2 = _mm_sub_epi32(diffs2, stored2); + diffs3 = _mm_sub_epi32(diffs3, stored3); + } + + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + + let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u16; + let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u16; + let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u16; + let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u16; + + let pixel_color0 = load_u16_s32_fast::(s_ptr0); + let pixel_color1 = load_u16_s32_fast::(s_ptr1); + let pixel_color2 = load_u16_s32_fast::(s_ptr2); + let pixel_color3 = load_u16_s32_fast::(s_ptr3); + + let arr_index = ((y + radius_64) & 1023) as usize; + + diffs0 = _mm_add_epi32(diffs0, pixel_color0); + diffs1 = _mm_add_epi32(diffs1, pixel_color1); + diffs2 = _mm_add_epi32(diffs2, pixel_color2); + diffs3 = _mm_add_epi32(diffs3, pixel_color3); + + _mm_store_si128( + buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color0, ); - let mut d_stored1 = _mm_load_si128( - buffer1.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color1, ); - let mut d_stored2 = _mm_load_si128( - buffer2.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color2, ); - let mut d_stored3 = _mm_load_si128( - buffer3.get_unchecked_mut(d_arr_index).0.as_mut_ptr() as *const _, + _mm_store_si128( + buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, + pixel_color3, ); - d_stored0 = _mm_slli_epi32::<1>(d_stored0); - d_stored1 = _mm_slli_epi32::<1>(d_stored1); - d_stored2 = _mm_slli_epi32::<1>(d_stored2); - d_stored3 = _mm_slli_epi32::<1>(d_stored3); - - let a_stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - let a_stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *const _); - - diffs0 = _mm_add_epi32(diffs0, _mm_sub_epi32(a_stored0, d_stored0)); - diffs1 = _mm_add_epi32(diffs1, _mm_sub_epi32(a_stored1, d_stored1)); - diffs2 = _mm_add_epi32(diffs2, _mm_sub_epi32(a_stored2, d_stored2)); - diffs3 = _mm_add_epi32(diffs3, _mm_sub_epi32(a_stored3, d_stored3)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - - let mut stored0 = - _mm_load_si128(buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _); - let mut stored1 = - _mm_load_si128(buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _); - let mut stored2 = - _mm_load_si128(buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _); - let mut stored3 = - _mm_load_si128(buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _); - - stored0 = _mm_slli_epi32::<1>(stored0); - stored1 = _mm_slli_epi32::<1>(stored1); - stored2 = _mm_slli_epi32::<1>(stored2); - stored3 = _mm_slli_epi32::<1>(stored3); - - diffs0 = _mm_sub_epi32(diffs0, stored0); - diffs1 = _mm_sub_epi32(diffs1, stored1); - diffs2 = _mm_sub_epi32(diffs2, stored2); - diffs3 = _mm_sub_epi32(diffs3, stored3); + summs0 = _mm_add_epi32(summs0, diffs0); + summs1 = _mm_add_epi32(summs1, diffs1); + summs2 = _mm_add_epi32(summs2, diffs2); + summs3 = _mm_add_epi32(summs3, diffs3); } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - - let s_ptr0 = bytes.slice.as_ptr().add(next_row_y + current_px0) as *mut u16; - let s_ptr1 = bytes.slice.as_ptr().add(next_row_y + current_px1) as *mut u16; - let s_ptr2 = bytes.slice.as_ptr().add(next_row_y + current_px2) as *mut u16; - let s_ptr3 = bytes.slice.as_ptr().add(next_row_y + current_px3) as *mut u16; - - let pixel_color0 = load_u16_s32_fast::(s_ptr0); - let pixel_color1 = load_u16_s32_fast::(s_ptr1); - let pixel_color2 = load_u16_s32_fast::(s_ptr2); - let pixel_color3 = load_u16_s32_fast::(s_ptr3); - - let arr_index = ((y + radius_64) & 1023) as usize; - - diffs0 = _mm_add_epi32(diffs0, pixel_color0); - diffs1 = _mm_add_epi32(diffs1, pixel_color1); - diffs2 = _mm_add_epi32(diffs2, pixel_color2); - diffs3 = _mm_add_epi32(diffs3, pixel_color3); - - _mm_store_si128( - buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color0, - ); - _mm_store_si128( - buffer1.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color1, - ); - _mm_store_si128( - buffer2.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color2, - ); - _mm_store_si128( - buffer3.get_unchecked_mut(arr_index).0.as_mut_ptr() as *mut _, - pixel_color3, - ); - - summs0 = _mm_add_epi32(summs0, diffs0); - summs1 = _mm_add_epi32(summs1, diffs1); - summs2 = _mm_add_epi32(summs2, diffs2); - summs3 = _mm_add_epi32(summs3, diffs3); + xx += 4; } - xx += 4; - } - - for x in xx..width.min(end) { - let mut diffs = _mm_setzero_si128(); - let mut summs = _mm_set1_epi32(initial_sum); + for x in xx..width.min(end) { + let mut diffs = _mm_setzero_si128(); + let mut summs = _mm_set1_epi32(initial_sum); - let current_px = (x * CN as u32) as usize; + let current_px = (x * CN as u32) as usize; - let start_y = 0 - 2 * radius as i64; - for y in start_y..height_wide { - if y >= 0 { - let pixel_f32 = _mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight); + let start_y = 0 - 2 * radius as i64; + for y in start_y..height_wide { + if y >= 0 { + let pixel_f32 = _mm_mul_ps(_mm_cvtepi32_ps(summs), v_weight); - let current_y = (y * (stride as i64)) as usize; + let current_y = (y * (stride as i64)) as usize; - let pixel_u32 = _mm_cvtps_epi32(pixel_f32); + let pixel_u32 = _mm_cvtps_epi32(pixel_f32); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); - store_u16_u32::(dst_ptr, pixel_u32); + let dst_ptr = (bytes.slice.as_ptr() as *mut u16).add(bytes_offset); + store_u16_u32::(dst_ptr, pixel_u32); - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let d_buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); - let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); - d_stored = _mm_slli_epi32::<1>(d_stored); + let d_buf_ptr = buffer0.get_unchecked_mut(d_arr_index).0.as_mut_ptr(); + let mut d_stored = _mm_load_si128(d_buf_ptr as *const __m128i); + d_stored = _mm_slli_epi32::<1>(d_stored); - let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; - let a_stored = _mm_load_si128(buf_ptr as *const __m128i); + let buf_ptr = buffer0.as_mut_ptr().add(arr_index) as *mut i32; + let a_stored = _mm_load_si128(buf_ptr as *const __m128i); - diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - let mut stored = _mm_load_si128(buf_ptr as *const __m128i); - stored = _mm_slli_epi32::<1>(stored); - diffs = _mm_sub_epi32(diffs, stored); - } + diffs = _mm_add_epi32(diffs, _mm_sub_epi32(a_stored, d_stored)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let mut stored = _mm_load_si128(buf_ptr as *const __m128i); + stored = _mm_slli_epi32::<1>(stored); + diffs = _mm_sub_epi32(diffs, stored); + } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u16; - let pixel_color = load_u16_s32_fast::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u16; + let pixel_color = load_u16_s32_fast::(s_ptr); - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer0.get_unchecked_mut(arr_index).0.as_mut_ptr(); - diffs = _mm_add_epi32(diffs, pixel_color); + diffs = _mm_add_epi32(diffs, pixel_color); - _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); + _mm_store_si128(buf_ptr as *mut __m128i, pixel_color); - summs = _mm_add_epi32(summs, diffs); + summs = _mm_add_epi32(summs, diffs); + } } } } diff --git a/src/sse/packing.rs b/src/sse/packing.rs index 6deeab12..b5283fd0 100644 --- a/src/sse/packing.rs +++ b/src/sse/packing.rs @@ -34,187 +34,203 @@ use std::arch::x86::*; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn _mm_deinterleave_rgba( +pub(crate) fn _mm_deinterleave_rgba( rgba0: __m128i, rgba1: __m128i, rgba2: __m128i, rgba3: __m128i, ) -> (__m128i, __m128i, __m128i, __m128i) { - let t0 = _mm_unpacklo_epi8(rgba0, rgba1); // r1 R1 g1 G1 b1 B1 a1 A1 r2 R2 g2 G2 b2 B2 a2 A2 - let t1 = _mm_unpackhi_epi8(rgba0, rgba1); - let t2 = _mm_unpacklo_epi8(rgba2, rgba3); // r4 R4 g4 G4 b4 B4 a4 A4 r5 R5 g5 G5 b5 B5 a5 A5 - let t3 = _mm_unpackhi_epi8(rgba2, rgba3); - - let t4 = _mm_unpacklo_epi16(t0, t2); // r1 R1 r4 R4 g1 G1 G4 g4 G4 b1 B1 b4 B4 a1 A1 a4 A4 - let t5 = _mm_unpackhi_epi16(t0, t2); - let t6 = _mm_unpacklo_epi16(t1, t3); - let t7 = _mm_unpackhi_epi16(t1, t3); - - let l1 = _mm_unpacklo_epi32(t4, t6); // r1 R1 r4 R4 g1 G1 G4 g4 G4 b1 B1 b4 B4 a1 A1 a4 A4 - let l2 = _mm_unpackhi_epi32(t4, t6); - let l3 = _mm_unpacklo_epi32(t5, t7); - let l4 = _mm_unpackhi_epi32(t5, t7); - - #[rustfmt::skip] + unsafe { + let t0 = _mm_unpacklo_epi8(rgba0, rgba1); // r1 R1 g1 G1 b1 B1 a1 A1 r2 R2 g2 G2 b2 B2 a2 A2 + let t1 = _mm_unpackhi_epi8(rgba0, rgba1); + let t2 = _mm_unpacklo_epi8(rgba2, rgba3); // r4 R4 g4 G4 b4 B4 a4 A4 r5 R5 g5 G5 b5 B5 a5 A5 + let t3 = _mm_unpackhi_epi8(rgba2, rgba3); + + let t4 = _mm_unpacklo_epi16(t0, t2); // r1 R1 r4 R4 g1 G1 G4 g4 G4 b1 B1 b4 B4 a1 A1 a4 A4 + let t5 = _mm_unpackhi_epi16(t0, t2); + let t6 = _mm_unpacklo_epi16(t1, t3); + let t7 = _mm_unpackhi_epi16(t1, t3); + + let l1 = _mm_unpacklo_epi32(t4, t6); // r1 R1 r4 R4 g1 G1 G4 g4 G4 b1 B1 b4 B4 a1 A1 a4 A4 + let l2 = _mm_unpackhi_epi32(t4, t6); + let l3 = _mm_unpacklo_epi32(t5, t7); + let l4 = _mm_unpackhi_epi32(t5, t7); + + #[rustfmt::skip] let shuffle = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, ); - let r1 = _mm_shuffle_epi8(_mm_unpacklo_epi32(l1, l3), shuffle); - let r2 = _mm_shuffle_epi8(_mm_unpackhi_epi32(l1, l3), shuffle); - let r3 = _mm_shuffle_epi8(_mm_unpacklo_epi32(l2, l4), shuffle); - let r4 = _mm_shuffle_epi8(_mm_unpackhi_epi32(l2, l4), shuffle); + let r1 = _mm_shuffle_epi8(_mm_unpacklo_epi32(l1, l3), shuffle); + let r2 = _mm_shuffle_epi8(_mm_unpackhi_epi32(l1, l3), shuffle); + let r3 = _mm_shuffle_epi8(_mm_unpacklo_epi32(l2, l4), shuffle); + let r4 = _mm_shuffle_epi8(_mm_unpackhi_epi32(l2, l4), shuffle); - (r1, r2, r3, r4) + (r1, r2, r3, r4) + } } #[inline(always)] -pub(crate) unsafe fn _mm_deinterleave_rgb( +pub(crate) fn _mm_deinterleave_rgb( s0: __m128i, s1: __m128i, s2: __m128i, ) -> (__m128i, __m128i, __m128i) { - let m0 = _mm_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0); - let m1 = _mm_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0); - - let a0 = _mm_blendv_epi8(_mm_blendv_epi8(s0, s1, m0), s2, m1); - let b0 = _mm_blendv_epi8(_mm_blendv_epi8(s1, s2, m0), s0, m1); - let c0 = _mm_blendv_epi8(_mm_blendv_epi8(s2, s0, m0), s1, m1); - let sh_b = _mm_setr_epi8(0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, 1, 4, 7, 10, 13); - let sh_g = _mm_setr_epi8(1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14); - let sh_r = _mm_setr_epi8(2, 5, 8, 11, 14, 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15); - let a0 = _mm_shuffle_epi8(a0, sh_b); - let b0 = _mm_shuffle_epi8(b0, sh_g); - let c0 = _mm_shuffle_epi8(c0, sh_r); - - (a0, b0, c0) + unsafe { + let m0 = _mm_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0); + let m1 = _mm_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0); + + let a0 = _mm_blendv_epi8(_mm_blendv_epi8(s0, s1, m0), s2, m1); + let b0 = _mm_blendv_epi8(_mm_blendv_epi8(s1, s2, m0), s0, m1); + let c0 = _mm_blendv_epi8(_mm_blendv_epi8(s2, s0, m0), s1, m1); + let sh_b = _mm_setr_epi8(0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14, 1, 4, 7, 10, 13); + let sh_g = _mm_setr_epi8(1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15, 2, 5, 8, 11, 14); + let sh_r = _mm_setr_epi8(2, 5, 8, 11, 14, 1, 4, 7, 10, 13, 0, 3, 6, 9, 12, 15); + let a0 = _mm_shuffle_epi8(a0, sh_b); + let b0 = _mm_shuffle_epi8(b0, sh_g); + let c0 = _mm_shuffle_epi8(c0, sh_r); + + (a0, b0, c0) + } } #[inline(always)] -pub(crate) unsafe fn _mm_interleave_rgb( +pub(crate) fn _mm_interleave_rgb( r: __m128i, g: __m128i, b: __m128i, ) -> (__m128i, __m128i, __m128i) { - let sh_a = _mm_setr_epi8(0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5); - let sh_b = _mm_setr_epi8(5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10); - let sh_c = _mm_setr_epi8(10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15); - let a0 = _mm_shuffle_epi8(r, sh_a); - let b0 = _mm_shuffle_epi8(g, sh_b); - let c0 = _mm_shuffle_epi8(b, sh_c); - - let m0 = _mm_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0); - let m1 = _mm_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0); - let v0 = _mm_blendv_epi8(_mm_blendv_epi8(a0, b0, m1), c0, m0); - let v1 = _mm_blendv_epi8(_mm_blendv_epi8(b0, c0, m1), a0, m0); - let v2 = _mm_blendv_epi8(_mm_blendv_epi8(c0, a0, m1), b0, m0); - (v0, v1, v2) + unsafe { + let sh_a = _mm_setr_epi8(0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10, 5); + let sh_b = _mm_setr_epi8(5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15, 10); + let sh_c = _mm_setr_epi8(10, 5, 0, 11, 6, 1, 12, 7, 2, 13, 8, 3, 14, 9, 4, 15); + let a0 = _mm_shuffle_epi8(r, sh_a); + let b0 = _mm_shuffle_epi8(g, sh_b); + let c0 = _mm_shuffle_epi8(b, sh_c); + + let m0 = _mm_setr_epi8(0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0); + let m1 = _mm_setr_epi8(0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0, -1, 0, 0); + let v0 = _mm_blendv_epi8(_mm_blendv_epi8(a0, b0, m1), c0, m0); + let v1 = _mm_blendv_epi8(_mm_blendv_epi8(b0, c0, m1), a0, m0); + let v2 = _mm_blendv_epi8(_mm_blendv_epi8(c0, a0, m1), b0, m0); + (v0, v1, v2) + } } #[inline(always)] -pub(crate) unsafe fn _mm_interleave_rgba( +pub(crate) fn _mm_interleave_rgba( r: __m128i, g: __m128i, b: __m128i, a: __m128i, ) -> (__m128i, __m128i, __m128i, __m128i) { - let rg_lo = _mm_unpacklo_epi8(r, g); - let rg_hi = _mm_unpackhi_epi8(r, g); - let ba_lo = _mm_unpacklo_epi8(b, a); - let ba_hi = _mm_unpackhi_epi8(b, a); - - let rgba_0_lo = _mm_unpacklo_epi16(rg_lo, ba_lo); - let rgba_0_hi = _mm_unpackhi_epi16(rg_lo, ba_lo); - let rgba_1_lo = _mm_unpacklo_epi16(rg_hi, ba_hi); - let rgba_1_hi = _mm_unpackhi_epi16(rg_hi, ba_hi); - (rgba_0_lo, rgba_0_hi, rgba_1_lo, rgba_1_hi) + unsafe { + let rg_lo = _mm_unpacklo_epi8(r, g); + let rg_hi = _mm_unpackhi_epi8(r, g); + let ba_lo = _mm_unpacklo_epi8(b, a); + let ba_hi = _mm_unpackhi_epi8(b, a); + + let rgba_0_lo = _mm_unpacklo_epi16(rg_lo, ba_lo); + let rgba_0_hi = _mm_unpackhi_epi16(rg_lo, ba_lo); + let rgba_1_lo = _mm_unpacklo_epi16(rg_hi, ba_hi); + let rgba_1_hi = _mm_unpackhi_epi16(rg_hi, ba_hi); + (rgba_0_lo, rgba_0_hi, rgba_1_lo, rgba_1_hi) + } } #[inline(always)] -pub(crate) unsafe fn _mm_deinterleave_rgba_ps( +pub(crate) fn _mm_deinterleave_rgba_ps( t0: __m128, t1: __m128, t2: __m128, t3: __m128, ) -> (__m128, __m128, __m128, __m128) { - let t02lo = _mm_unpacklo_ps(t0, t2); - let t13lo = _mm_unpacklo_ps(t1, t3); - let t02hi = _mm_unpackhi_ps(t0, t2); - let t13hi = _mm_unpackhi_ps(t1, t3); - let v0 = _mm_unpacklo_ps(t02lo, t13lo); - let v1 = _mm_unpackhi_ps(t02lo, t13lo); - let v2 = _mm_unpacklo_ps(t02hi, t13hi); - let v3 = _mm_unpackhi_ps(t02hi, t13hi); - (v0, v1, v2, v3) + unsafe { + let t02lo = _mm_unpacklo_ps(t0, t2); + let t13lo = _mm_unpacklo_ps(t1, t3); + let t02hi = _mm_unpackhi_ps(t0, t2); + let t13hi = _mm_unpackhi_ps(t1, t3); + let v0 = _mm_unpacklo_ps(t02lo, t13lo); + let v1 = _mm_unpackhi_ps(t02lo, t13lo); + let v2 = _mm_unpacklo_ps(t02hi, t13hi); + let v3 = _mm_unpackhi_ps(t02hi, t13hi); + (v0, v1, v2, v3) + } } #[inline(always)] -pub(crate) unsafe fn _mm_deinterleave_rgb_ps( +pub(crate) fn _mm_deinterleave_rgb_ps( t0: __m128, t1: __m128, t2: __m128, ) -> (__m128, __m128, __m128) { - const FLAG_1: i32 = _shuffle(0, 1, 0, 2); - let at12 = _mm_shuffle_ps::(t1, t2); - const FLAG_2: i32 = _shuffle(2, 0, 3, 0); - let v0 = _mm_shuffle_ps::(t0, at12); - const FLAG_3: i32 = _shuffle(0, 0, 0, 1); - let bt01 = _mm_shuffle_ps::(t0, t1); - const FLAG_4: i32 = _shuffle(0, 2, 0, 3); - let bt12 = _mm_shuffle_ps::(t1, t2); - const FLAG_5: i32 = _shuffle(2, 0, 2, 0); - let v1 = _mm_shuffle_ps::(bt01, bt12); - - const FLAG_6: i32 = _shuffle(0, 1, 0, 2); - let ct01 = _mm_shuffle_ps::(t0, t1); - const FLAG_7: i32 = _shuffle(3, 0, 2, 0); - let v2 = _mm_shuffle_ps::(ct01, t2); - (v0, v1, v2) + unsafe { + const FLAG_1: i32 = _shuffle(0, 1, 0, 2); + let at12 = _mm_shuffle_ps::(t1, t2); + const FLAG_2: i32 = _shuffle(2, 0, 3, 0); + let v0 = _mm_shuffle_ps::(t0, at12); + const FLAG_3: i32 = _shuffle(0, 0, 0, 1); + let bt01 = _mm_shuffle_ps::(t0, t1); + const FLAG_4: i32 = _shuffle(0, 2, 0, 3); + let bt12 = _mm_shuffle_ps::(t1, t2); + const FLAG_5: i32 = _shuffle(2, 0, 2, 0); + let v1 = _mm_shuffle_ps::(bt01, bt12); + + const FLAG_6: i32 = _shuffle(0, 1, 0, 2); + let ct01 = _mm_shuffle_ps::(t0, t1); + const FLAG_7: i32 = _shuffle(3, 0, 2, 0); + let v2 = _mm_shuffle_ps::(ct01, t2); + (v0, v1, v2) + } } #[inline(always)] -pub(crate) unsafe fn _mm_interleave_rgb_ps( +pub(crate) fn _mm_interleave_rgb_ps( t0: __m128, t1: __m128, t2: __m128, ) -> (__m128, __m128, __m128) { - const FLAG_1: i32 = _shuffle(0, 0, 0, 0); - let u0 = _mm_shuffle_ps::(t0, t1); - const FLAG_2: i32 = _shuffle(1, 1, 0, 0); - let u1 = _mm_shuffle_ps::(t2, t0); - const FLAG_3: i32 = _shuffle(2, 0, 2, 0); - let v0 = _mm_shuffle_ps::(u0, u1); - const FLAG_4: i32 = _shuffle(1, 1, 1, 1); - let u2 = _mm_shuffle_ps::(t1, t2); - const FLAG_5: i32 = _shuffle(2, 2, 2, 2); - let u3 = _mm_shuffle_ps::(t0, t1); - const FLAG_6: i32 = _shuffle(2, 0, 2, 0); - let v1 = _mm_shuffle_ps::(u2, u3); - const FLAG_7: i32 = _shuffle(3, 3, 2, 2); - let u4 = _mm_shuffle_ps::(t2, t0); - const FLAG_8: i32 = _shuffle(3, 3, 3, 3); - let u5 = _mm_shuffle_ps::(t1, t2); - const FLAG_9: i32 = _shuffle(2, 0, 2, 0); - let v2 = _mm_shuffle_ps::(u4, u5); - (v0, v1, v2) + unsafe { + const FLAG_1: i32 = _shuffle(0, 0, 0, 0); + let u0 = _mm_shuffle_ps::(t0, t1); + const FLAG_2: i32 = _shuffle(1, 1, 0, 0); + let u1 = _mm_shuffle_ps::(t2, t0); + const FLAG_3: i32 = _shuffle(2, 0, 2, 0); + let v0 = _mm_shuffle_ps::(u0, u1); + const FLAG_4: i32 = _shuffle(1, 1, 1, 1); + let u2 = _mm_shuffle_ps::(t1, t2); + const FLAG_5: i32 = _shuffle(2, 2, 2, 2); + let u3 = _mm_shuffle_ps::(t0, t1); + const FLAG_6: i32 = _shuffle(2, 0, 2, 0); + let v1 = _mm_shuffle_ps::(u2, u3); + const FLAG_7: i32 = _shuffle(3, 3, 2, 2); + let u4 = _mm_shuffle_ps::(t2, t0); + const FLAG_8: i32 = _shuffle(3, 3, 3, 3); + let u5 = _mm_shuffle_ps::(t1, t2); + const FLAG_9: i32 = _shuffle(2, 0, 2, 0); + let v2 = _mm_shuffle_ps::(u4, u5); + (v0, v1, v2) + } } #[inline(always)] -pub(crate) unsafe fn _mm_interleave_rgba_ps( +pub(crate) fn _mm_interleave_rgba_ps( t0: __m128, t1: __m128, t2: __m128, t3: __m128, ) -> (__m128, __m128, __m128, __m128) { - let u0 = _mm_unpacklo_ps(t0, t2); - let u1 = _mm_unpacklo_ps(t1, t3); - let u2 = _mm_unpackhi_ps(t0, t2); - let u3 = _mm_unpackhi_ps(t1, t3); - let v0 = _mm_unpacklo_ps(u0, u1); - let v2 = _mm_unpacklo_ps(u2, u3); - let v1 = _mm_unpackhi_ps(u0, u1); - let v3 = _mm_unpackhi_ps(u2, u3); - (v0, v1, v2, v3) + unsafe { + let u0 = _mm_unpacklo_ps(t0, t2); + let u1 = _mm_unpacklo_ps(t1, t3); + let u2 = _mm_unpackhi_ps(t0, t2); + let u3 = _mm_unpackhi_ps(t1, t3); + let v0 = _mm_unpacklo_ps(u0, u1); + let v2 = _mm_unpacklo_ps(u2, u3); + let v1 = _mm_unpackhi_ps(u0, u1); + let v3 = _mm_unpackhi_ps(u2, u3); + (v0, v1, v2, v3) + } } diff --git a/src/sse/utils.rs b/src/sse/utils.rs index 62671a08..58b4f8a9 100644 --- a/src/sse/utils.rs +++ b/src/sse/utils.rs @@ -33,88 +33,100 @@ use std::arch::x86::*; use std::arch::x86_64::*; #[inline(always)] -pub(crate) unsafe fn load_f32(ptr: *const f32) -> __m128 { - if CN == 4 { - _mm_loadu_ps(ptr) - } else if CN == 3 { - let mut j = _mm_loadu_si64(ptr as *const _); - j = _mm_insert_epi32::<2>(j, ptr.add(2).read_unaligned().to_bits() as i32); - _mm_castsi128_ps(j) - } else if CN == 2 { - _mm_castsi128_ps(_mm_loadu_si64(ptr as *const _)) - } else { - _mm_load_ss(ptr as *const _) +pub(crate) fn load_f32(ptr: *const f32) -> __m128 { + unsafe { + if CN == 4 { + _mm_loadu_ps(ptr) + } else if CN == 3 { + let mut j = _mm_loadu_si64(ptr as *const _); + j = _mm_insert_epi32::<2>(j, ptr.add(2).read_unaligned().to_bits() as i32); + _mm_castsi128_ps(j) + } else if CN == 2 { + _mm_castsi128_ps(_mm_loadu_si64(ptr as *const _)) + } else { + _mm_load_ss(ptr as *const _) + } } } #[allow(dead_code)] #[inline(always)] pub(crate) unsafe fn load_u8_s16_fast(ptr: *const u8) -> __m128i { - let sh1 = _mm_setr_epi8(0, -1, 1, -1, 2, -1, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1); - if CN == 4 { - let v = _mm_loadu_si32(ptr as *const _); - _mm_shuffle_epi8(v, sh1) - } else if CN == 3 { - let mut v0 = _mm_loadu_si16(ptr); - v0 = _mm_insert_epi8::<2>(v0, ptr.add(2).read_unaligned() as i32); - _mm_shuffle_epi8(v0, sh1) - } else if CN == 2 { - let v0 = _mm_loadu_si16(ptr); - _mm_shuffle_epi8(v0, sh1) - } else { - _mm_setr_epi16(ptr.read_unaligned() as i16, 0, 0, 0, 0, 0, 0, 0) + unsafe { + let sh1 = _mm_setr_epi8(0, -1, 1, -1, 2, -1, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1); + if CN == 4 { + let v = _mm_loadu_si32(ptr as *const _); + _mm_shuffle_epi8(v, sh1) + } else if CN == 3 { + let mut v0 = _mm_loadu_si16(ptr); + v0 = _mm_insert_epi8::<2>(v0, ptr.add(2).read_unaligned() as i32); + _mm_shuffle_epi8(v0, sh1) + } else if CN == 2 { + let v0 = _mm_loadu_si16(ptr); + _mm_shuffle_epi8(v0, sh1) + } else { + _mm_setr_epi16(ptr.read_unaligned() as i16, 0, 0, 0, 0, 0, 0, 0) + } } } #[inline(always)] pub(crate) unsafe fn load_u16_s32_fast(ptr: *const u16) -> __m128i { - if CN == 4 { - let v = _mm_loadu_si64(ptr as *const _); - _mm_unpacklo_epi16(v, _mm_setzero_si128()) - } else if CN == 3 { - let mut v0 = _mm_loadu_si32(ptr as *const _); - v0 = _mm_insert_epi16::<2>(v0, ptr.add(2).read_unaligned() as i32); - _mm_unpacklo_epi16(v0, _mm_setzero_si128()) - } else if CN == 2 { - let v0 = _mm_loadu_si32(ptr as *const _); - _mm_unpacklo_epi16(v0, _mm_setzero_si128()) - } else { - _mm_unpacklo_epi16(_mm_loadu_si16(ptr as *const _), _mm_setzero_si128()) + unsafe { + if CN == 4 { + let v = _mm_loadu_si64(ptr as *const _); + _mm_unpacklo_epi16(v, _mm_setzero_si128()) + } else if CN == 3 { + let mut v0 = _mm_loadu_si32(ptr as *const _); + v0 = _mm_insert_epi16::<2>(v0, ptr.add(2).read_unaligned() as i32); + _mm_unpacklo_epi16(v0, _mm_setzero_si128()) + } else if CN == 2 { + let v0 = _mm_loadu_si32(ptr as *const _); + _mm_unpacklo_epi16(v0, _mm_setzero_si128()) + } else { + _mm_unpacklo_epi16(_mm_loadu_si16(ptr as *const _), _mm_setzero_si128()) + } } } #[inline(always)] pub(crate) unsafe fn load_u8_s32_fast(ptr: *const u8) -> __m128i { - let sh1 = _mm_setr_epi8(0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1); - if CN == 4 { - let v = _mm_loadu_si32(ptr as *const _); - _mm_shuffle_epi8(v, sh1) - } else if CN == 3 { - let mut v0 = _mm_loadu_si16(ptr); - v0 = _mm_insert_epi8::<2>(v0, ptr.add(2).read_unaligned() as i32); - _mm_shuffle_epi8(v0, sh1) - } else if CN == 2 { - let v0 = _mm_loadu_si16(ptr); - _mm_shuffle_epi8(v0, sh1) - } else { - _mm_setr_epi32(ptr.read_unaligned() as i32, 0, 0, 0) + unsafe { + let sh1 = _mm_setr_epi8(0, -1, -1, -1, 1, -1, -1, -1, 2, -1, -1, -1, 3, -1, -1, -1); + if CN == 4 { + let v = _mm_loadu_si32(ptr as *const _); + _mm_shuffle_epi8(v, sh1) + } else if CN == 3 { + let mut v0 = _mm_loadu_si16(ptr); + v0 = _mm_insert_epi8::<2>(v0, ptr.add(2).read_unaligned() as i32); + _mm_shuffle_epi8(v0, sh1) + } else if CN == 2 { + let v0 = _mm_loadu_si16(ptr); + _mm_shuffle_epi8(v0, sh1) + } else { + _mm_setr_epi32(ptr.read_unaligned() as i32, 0, 0, 0) + } } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_ps_epi32(ab: __m128i, cd: __m128) -> __m128i { - let cvt = _mm_cvtepi32_ps(ab); - let rs = _mm_mul_ps(cvt, cd); - _mm_cvtps_epi32(rs) +pub(crate) fn _mm_mul_ps_epi32(ab: __m128i, cd: __m128) -> __m128i { + unsafe { + let cvt = _mm_cvtepi32_ps(ab); + let rs = _mm_mul_ps(cvt, cd); + _mm_cvtps_epi32(rs) + } } #[inline(always)] -pub(crate) unsafe fn _mm_blendv_epi32x(xmm0: __m128i, xmm1: __m128i, mask: __m128i) -> __m128i { - _mm_castps_si128(_mm_blendv_ps( - _mm_castsi128_ps(xmm0), - _mm_castsi128_ps(xmm1), - _mm_castsi128_ps(mask), - )) +pub(crate) fn _mm_blendv_epi32x(xmm0: __m128i, xmm1: __m128i, mask: __m128i) -> __m128i { + unsafe { + _mm_castps_si128(_mm_blendv_ps( + _mm_castsi128_ps(xmm0), + _mm_castsi128_ps(xmm1), + _mm_castsi128_ps(mask), + )) + } } pub(crate) const fn _shuffle(z: u32, y: u32, x: u32, w: u32) -> i32 { @@ -123,26 +135,30 @@ pub(crate) const fn _shuffle(z: u32, y: u32, x: u32, w: u32) -> i32 { } #[inline(always)] -pub(crate) unsafe fn _mm_packus_epi64(a: __m128i, b: __m128i) -> __m128i { - const SHUFFLE_MASK: i32 = _shuffle(3, 1, 2, 0); - let a = _mm_shuffle_epi32::(a); - let b1 = _mm_shuffle_epi32::(b); - _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b1))) +pub(crate) fn _mm_packus_epi64(a: __m128i, b: __m128i) -> __m128i { + unsafe { + const SHUFFLE_MASK: i32 = _shuffle(3, 1, 2, 0); + let a = _mm_shuffle_epi32::(a); + let b1 = _mm_shuffle_epi32::(b); + _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(a), _mm_castsi128_ps(b1))) + } } #[inline(always)] -pub(crate) unsafe fn store_f32(dst_ptr: *mut f32, regi: __m128) { - if CN == 4 { - _mm_storeu_ps(dst_ptr, regi); - } else if CN == 3 { - _mm_storeu_si64(dst_ptr as *mut u8, _mm_castps_si128(regi)); - dst_ptr - .add(2) - .write_unaligned(f32::from_bits(_mm_extract_ps::<2>(regi) as u32)); - } else if CN == 2 { - _mm_storeu_si64(dst_ptr as *mut u8, _mm_castps_si128(regi)); - } else { - _mm_store_ss(dst_ptr as *mut _, regi); +pub(crate) fn store_f32(dst_ptr: *mut f32, regi: __m128) { + unsafe { + if CN == 4 { + _mm_storeu_ps(dst_ptr, regi); + } else if CN == 3 { + _mm_storeu_si64(dst_ptr as *mut u8, _mm_castps_si128(regi)); + dst_ptr + .add(2) + .write_unaligned(f32::from_bits(_mm_extract_ps::<2>(regi) as u32)); + } else if CN == 2 { + _mm_storeu_si64(dst_ptr as *mut u8, _mm_castps_si128(regi)); + } else { + _mm_store_ss(dst_ptr as *mut _, regi); + } } } @@ -150,40 +166,44 @@ pub(crate) unsafe fn store_f32(dst_ptr: *mut f32, regi: __m128) #[inline(always)] #[cfg(any(feature = "sse", feature = "avx"))] pub(crate) unsafe fn store_u8_s32(dst_ptr: *mut u8, regi: __m128i) { - let s16 = _mm_packs_epi32(regi, regi); - let v8 = _mm_packus_epi16(s16, s16); - if CN == 4 { - _mm_storeu_si32(dst_ptr as *mut _, v8); - } else if CN == 3 { - let pixel_3 = _mm_extract_epi8::<2>(v8); - _mm_storeu_si16(dst_ptr, v8); - dst_ptr.add(2).write_unaligned(pixel_3 as u8); - } else if CN == 2 { - _mm_storeu_si16(dst_ptr, v8); - } else { - let pixel_s32 = _mm_extract_epi32::<0>(v8); - let pixel_bytes = pixel_s32.to_le_bytes(); - dst_ptr.write_unaligned(pixel_bytes[0]); + unsafe { + let s16 = _mm_packs_epi32(regi, regi); + let v8 = _mm_packus_epi16(s16, s16); + if CN == 4 { + _mm_storeu_si32(dst_ptr as *mut _, v8); + } else if CN == 3 { + let pixel_3 = _mm_extract_epi8::<2>(v8); + _mm_storeu_si16(dst_ptr, v8); + dst_ptr.add(2).write_unaligned(pixel_3 as u8); + } else if CN == 2 { + _mm_storeu_si16(dst_ptr, v8); + } else { + let pixel_s32 = _mm_extract_epi32::<0>(v8); + let pixel_bytes = pixel_s32.to_le_bytes(); + dst_ptr.write_unaligned(pixel_bytes[0]); + } } } /// Stores u32 up to x4 as u8 up to x4 based on channels count #[inline(always)] pub(crate) unsafe fn store_u8_u32(dst_ptr: *mut u8, regi: __m128i) { - let s16 = _mm_packus_epi32(regi, regi); - let v8 = _mm_packus_epi16(s16, s16); - if CN == 4 { - _mm_storeu_si32(dst_ptr as *mut _, v8); - } else if CN == 3 { - let pixel_3 = _mm_extract_epi8::<2>(v8); - _mm_storeu_si16(dst_ptr, v8); - dst_ptr.add(2).write_unaligned(pixel_3 as u8); - } else if CN == 2 { - _mm_storeu_si16(dst_ptr, v8); - } else { - let pixel_s32 = _mm_extract_epi32::<0>(v8); - let pixel_bytes = pixel_s32.to_le_bytes(); - dst_ptr.write_unaligned(pixel_bytes[0]); + unsafe { + let s16 = _mm_packus_epi32(regi, regi); + let v8 = _mm_packus_epi16(s16, s16); + if CN == 4 { + _mm_storeu_si32(dst_ptr as *mut _, v8); + } else if CN == 3 { + let pixel_3 = _mm_extract_epi8::<2>(v8); + _mm_storeu_si16(dst_ptr, v8); + dst_ptr.add(2).write_unaligned(pixel_3 as u8); + } else if CN == 2 { + _mm_storeu_si16(dst_ptr, v8); + } else { + let pixel_s32 = _mm_extract_epi32::<0>(v8); + let pixel_bytes = pixel_s32.to_le_bytes(); + dst_ptr.write_unaligned(pixel_bytes[0]); + } } } @@ -191,63 +211,71 @@ pub(crate) unsafe fn store_u8_u32(dst_ptr: *mut u8, regi: __m12 #[inline(always)] #[allow(dead_code)] pub(crate) unsafe fn store_u8_s16(dst_ptr: *mut u8, regi: __m128i) { - let v8 = _mm_packus_epi16(regi, regi); - if CN == 4 { - _mm_storeu_si32(dst_ptr as *mut _, v8); - } else if CN == 3 { - let pixel_3 = _mm_extract_epi8::<2>(v8); - _mm_storeu_si16(dst_ptr, v8); - dst_ptr.add(2).write_unaligned(pixel_3 as u8); - } else if CN == 2 { - _mm_storeu_si16(dst_ptr, v8); - } else { - let pixel_s32 = _mm_extract_epi32::<0>(v8); - let pixel_bytes = pixel_s32.to_le_bytes(); - dst_ptr.write_unaligned(pixel_bytes[0]); + unsafe { + let v8 = _mm_packus_epi16(regi, regi); + if CN == 4 { + _mm_storeu_si32(dst_ptr as *mut _, v8); + } else if CN == 3 { + let pixel_3 = _mm_extract_epi8::<2>(v8); + _mm_storeu_si16(dst_ptr, v8); + dst_ptr.add(2).write_unaligned(pixel_3 as u8); + } else if CN == 2 { + _mm_storeu_si16(dst_ptr, v8); + } else { + let pixel_s32 = _mm_extract_epi32::<0>(v8); + let pixel_bytes = pixel_s32.to_le_bytes(); + dst_ptr.write_unaligned(pixel_bytes[0]); + } } } #[allow(dead_code)] #[inline(always)] pub(crate) unsafe fn store_u16_u32(dst_ptr: *mut u16, regi: __m128i) { - let v0 = _mm_packus_epi32(regi, regi); - if CN == 4 { - _mm_storeu_si64(dst_ptr as *mut _, v0); - } else if CN == 3 { - _mm_storeu_si32(dst_ptr as *mut _, v0); - let val = _mm_extract_epi16::<2>(v0); - dst_ptr.add(2).write_unaligned(val as u16); - } else if CN == 2 { - _mm_storeu_si32(dst_ptr as *mut _, v0); - } else { - _mm_storeu_si16(dst_ptr as *mut _, v0); + unsafe { + let v0 = _mm_packus_epi32(regi, regi); + if CN == 4 { + _mm_storeu_si64(dst_ptr as *mut _, v0); + } else if CN == 3 { + _mm_storeu_si32(dst_ptr as *mut _, v0); + let val = _mm_extract_epi16::<2>(v0); + dst_ptr.add(2).write_unaligned(val as u16); + } else if CN == 2 { + _mm_storeu_si32(dst_ptr as *mut _, v0); + } else { + _mm_storeu_si16(dst_ptr as *mut _, v0); + } } } #[inline(always)] #[cfg(any(feature = "sse", feature = "avx"))] pub(crate) unsafe fn write_u8(dst_ptr: *mut u8, v8: __m128i) { - if CN == 4 { - _mm_storeu_si32(dst_ptr as *mut _, v8); - } else if CN == 3 { - let pixel_3 = _mm_extract_epi8::<2>(v8); - _mm_storeu_si16(dst_ptr, v8); - dst_ptr.add(2).write_unaligned(pixel_3 as u8); - } else if CN == 2 { - _mm_storeu_si16(dst_ptr, v8); - } else { - let pixel_s32 = _mm_extract_epi8::<0>(v8); - dst_ptr.write_unaligned(pixel_s32 as u8); + unsafe { + if CN == 4 { + _mm_storeu_si32(dst_ptr as *mut _, v8); + } else if CN == 3 { + let pixel_3 = _mm_extract_epi8::<2>(v8); + _mm_storeu_si16(dst_ptr, v8); + dst_ptr.add(2).write_unaligned(pixel_3 as u8); + } else if CN == 2 { + _mm_storeu_si16(dst_ptr, v8); + } else { + let pixel_s32 = _mm_extract_epi8::<0>(v8); + dst_ptr.write_unaligned(pixel_s32 as u8); + } } } #[inline(always)] -pub(crate) unsafe fn _mm_hsum_ps(v: __m128) -> f32 { - let mut shuf = _mm_movehdup_ps(v); - let mut sums = _mm_add_ps(v, shuf); - shuf = _mm_movehl_ps(shuf, sums); - sums = _mm_add_ss(sums, shuf); - _mm_cvtss_f32(sums) +pub(crate) fn _mm_hsum_ps(v: __m128) -> f32 { + unsafe { + let mut shuf = _mm_movehdup_ps(v); + let mut sums = _mm_add_ps(v, shuf); + shuf = _mm_movehl_ps(shuf, sums); + sums = _mm_add_ss(sums, shuf); + _mm_cvtss_f32(sums) + } } // #[inline(always)] @@ -262,165 +290,191 @@ pub(crate) unsafe fn _mm_hsum_ps(v: __m128) -> f32 { // } #[inline(always)] -pub(crate) unsafe fn _mm_loadu_si128_x2(ptr: *const u8) -> (__m128i, __m128i) { - ( - _mm_loadu_si128(ptr as *const __m128i), - _mm_loadu_si128(ptr.add(16) as *const __m128i), - ) +pub(crate) fn _mm_loadu_si128_x2(ptr: *const u8) -> (__m128i, __m128i) { + unsafe { + ( + _mm_loadu_si128(ptr as *const __m128i), + _mm_loadu_si128(ptr.add(16) as *const __m128i), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_loadu_ps_x4(ptr: *const f32) -> (__m128, __m128, __m128, __m128) { - ( - _mm_loadu_ps(ptr), - _mm_loadu_ps(ptr.add(4)), - _mm_loadu_ps(ptr.add(8)), - _mm_loadu_ps(ptr.add(12)), - ) +pub(crate) fn _mm_loadu_ps_x4(ptr: *const f32) -> (__m128, __m128, __m128, __m128) { + unsafe { + ( + _mm_loadu_ps(ptr), + _mm_loadu_ps(ptr.add(4)), + _mm_loadu_ps(ptr.add(8)), + _mm_loadu_ps(ptr.add(12)), + ) + } } #[inline(always)] -pub(crate) unsafe fn _mm_loadu_ps_x2(ptr: *const f32) -> (__m128, __m128) { - (_mm_loadu_ps(ptr), _mm_loadu_ps(ptr.add(4))) +pub(crate) fn _mm_loadu_ps_x2(ptr: *const f32) -> (__m128, __m128) { + unsafe { (_mm_loadu_ps(ptr), _mm_loadu_ps(ptr.add(4))) } } #[inline(always)] -pub(crate) unsafe fn _mm_erase_last_ps(item: __m128) -> __m128 { - #[allow(overflowing_literals)] - let mask = _mm_castsi128_ps(_mm_setr_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0)); - _mm_and_ps(item, mask) +pub(crate) fn _mm_erase_last_ps(item: __m128) -> __m128 { + unsafe { + #[allow(overflowing_literals)] + let mask = _mm_castsi128_ps(_mm_setr_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0)); + _mm_and_ps(item, mask) + } } #[inline(always)] -pub(crate) unsafe fn _mm_split_rgb_5_ps( +pub(crate) fn _mm_split_rgb_5_ps( vals: (__m128, __m128, __m128, __m128), ) -> (__m128, __m128, __m128, __m128, __m128) { - let first = _mm_erase_last_ps(vals.0); - - let second = _mm_erase_last_ps(_mm_castsi128_ps(_mm_alignr_epi8::<12>( - _mm_castps_si128(vals.1), - _mm_castps_si128(vals.0), - ))); - let third = _mm_erase_last_ps(_mm_castsi128_ps(_mm_alignr_epi8::<8>( - _mm_castps_si128(vals.2), - _mm_castps_si128(vals.1), - ))); - let fourth = _mm_erase_last_ps(_mm_castsi128_ps(_mm_alignr_epi8::<4>( - _mm_castps_si128(vals.3), - _mm_castps_si128(vals.2), - ))); - let fifth = _mm_erase_last_ps(vals.3); - (first, second, third, fourth, fifth) + unsafe { + let first = _mm_erase_last_ps(vals.0); + + let second = _mm_erase_last_ps(_mm_castsi128_ps(_mm_alignr_epi8::<12>( + _mm_castps_si128(vals.1), + _mm_castps_si128(vals.0), + ))); + let third = _mm_erase_last_ps(_mm_castsi128_ps(_mm_alignr_epi8::<8>( + _mm_castps_si128(vals.2), + _mm_castps_si128(vals.1), + ))); + let fourth = _mm_erase_last_ps(_mm_castsi128_ps(_mm_alignr_epi8::<4>( + _mm_castps_si128(vals.3), + _mm_castps_si128(vals.2), + ))); + let fifth = _mm_erase_last_ps(vals.3); + (first, second, third, fourth, fifth) + } } #[inline(always)] -pub(crate) unsafe fn _mm_broadcast_first(item: __m128) -> __m128 { - const FLAG: i32 = _shuffle(0, 0, 0, 0); - _mm_shuffle_ps::(item, item) +pub(crate) fn _mm_broadcast_first(item: __m128) -> __m128 { + unsafe { + const FLAG: i32 = _shuffle(0, 0, 0, 0); + _mm_shuffle_ps::(item, item) + } } #[inline(always)] -pub(crate) unsafe fn _mm_broadcast_second(item: __m128) -> __m128 { - const FLAG: i32 = _shuffle(1, 1, 1, 1); - _mm_shuffle_ps::(item, item) +pub(crate) fn _mm_broadcast_second(item: __m128) -> __m128 { + unsafe { + const FLAG: i32 = _shuffle(1, 1, 1, 1); + _mm_shuffle_ps::(item, item) + } } #[inline(always)] -pub(crate) unsafe fn _mm_broadcast_third(item: __m128) -> __m128 { - const FLAG: i32 = _shuffle(2, 2, 2, 2); - _mm_shuffle_ps::(item, item) +pub(crate) fn _mm_broadcast_third(item: __m128) -> __m128 { + unsafe { + const FLAG: i32 = _shuffle(2, 2, 2, 2); + _mm_shuffle_ps::(item, item) + } } #[inline(always)] -pub(crate) unsafe fn _mm_broadcast_fourth(item: __m128) -> __m128 { - const FLAG: i32 = _shuffle(3, 3, 3, 3); - _mm_shuffle_ps::(item, item) +pub(crate) fn _mm_broadcast_fourth(item: __m128) -> __m128 { + unsafe { + const FLAG: i32 = _shuffle(3, 3, 3, 3); + _mm_shuffle_ps::(item, item) + } } #[cfg(feature = "nightly_f16")] #[inline(always)] -pub(crate) unsafe fn load_f32_f16(ptr: *const f16) -> __m128 { - if CN == 4 { - let in_regi = _mm_loadu_si64(ptr as *const u8); - return _mm_cvtph_ps(in_regi); - } else if CN == 3 { - let casted_ptr = ptr as *const i16; - let in_regi = _mm_setr_epi16( - casted_ptr.read_unaligned(), - casted_ptr.add(1).read_unaligned(), - casted_ptr.add(2).read_unaligned(), - 0, - 0, - 0, - 0, - 0, - ); - return _mm_cvtph_ps(in_regi); - } else if CN == 2 { +pub(crate) fn load_f32_f16(ptr: *const f16) -> __m128 { + unsafe { + if CN == 4 { + let in_regi = _mm_loadu_si64(ptr as *const u8); + return _mm_cvtph_ps(in_regi); + } else if CN == 3 { + let casted_ptr = ptr as *const i16; + let in_regi = _mm_setr_epi16( + casted_ptr.read_unaligned(), + casted_ptr.add(1).read_unaligned(), + casted_ptr.add(2).read_unaligned(), + 0, + 0, + 0, + 0, + 0, + ); + return _mm_cvtph_ps(in_regi); + } else if CN == 2 { + let casted_ptr = ptr as *const i16; + let in_regi = _mm_setr_epi16( + casted_ptr.read_unaligned(), + casted_ptr.add(1).read_unaligned(), + 0, + 0, + 0, + 0, + 0, + 0, + ); + return _mm_cvtph_ps(in_regi); + } let casted_ptr = ptr as *const i16; - let in_regi = _mm_setr_epi16( - casted_ptr.read_unaligned(), - casted_ptr.add(1).read_unaligned(), - 0, - 0, - 0, - 0, - 0, - 0, - ); - return _mm_cvtph_ps(in_regi); + let in_regi = _mm_setr_epi16(casted_ptr.read_unaligned(), 0, 0, 0, 0, 0, 0, 0); + _mm_cvtph_ps(in_regi) } - let casted_ptr = ptr as *const i16; - let in_regi = _mm_setr_epi16(casted_ptr.read_unaligned(), 0, 0, 0, 0, 0, 0, 0); - _mm_cvtph_ps(in_regi) } #[cfg(feature = "nightly_f16")] #[inline(always)] pub(crate) unsafe fn store_f32_f16(dst_ptr: *mut f16, in_regi: __m128) { - let out_regi = _mm_cvtps_ph::<_MM_FROUND_TO_NEAREST_INT>(in_regi); - if CN == 4 { - _mm_storeu_si64(dst_ptr as *mut u8, out_regi); - } else if CN == 3 { - let casted_ptr = dst_ptr as *mut i16; - let item1 = _mm_extract_epi16::<2>(out_regi) as i16; - _mm_storeu_si32(dst_ptr as *mut u8, out_regi); - casted_ptr.add(2).write_unaligned(item1); - } else if CN == 2 { - _mm_storeu_si32(dst_ptr as *mut u8, out_regi); - } else { - _mm_storeu_si16(dst_ptr as *mut u8, out_regi); + unsafe { + let out_regi = _mm_cvtps_ph::<_MM_FROUND_TO_NEAREST_INT>(in_regi); + if CN == 4 { + _mm_storeu_si64(dst_ptr as *mut u8, out_regi); + } else if CN == 3 { + let casted_ptr = dst_ptr as *mut i16; + let item1 = _mm_extract_epi16::<2>(out_regi) as i16; + _mm_storeu_si32(dst_ptr as *mut u8, out_regi); + casted_ptr.add(2).write_unaligned(item1); + } else if CN == 2 { + _mm_storeu_si32(dst_ptr as *mut u8, out_regi); + } else { + _mm_storeu_si16(dst_ptr as *mut u8, out_regi); + } } } #[inline(always)] -pub(crate) unsafe fn _mm_mul_by_3_epi32(v: __m128i) -> __m128i { - _mm_add_epi32(_mm_slli_epi32::<1>(v), v) +pub(crate) fn _mm_mul_by_3_epi32(v: __m128i) -> __m128i { + unsafe { _mm_add_epi32(_mm_slli_epi32::<1>(v), v) } } #[inline(always)] pub(crate) unsafe fn _mm_opt_fnmlaf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { - if FMA { - _mm_fnmadd_ps(b, c, a) - } else { - _mm_sub_ps(a, _mm_mul_ps(b, c)) + unsafe { + if FMA { + _mm_fnmadd_ps(b, c, a) + } else { + _mm_sub_ps(a, _mm_mul_ps(b, c)) + } } } #[inline(always)] -pub(crate) unsafe fn _mm_opt_fmlaf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { - if FMA { - _mm_fmadd_ps(b, c, a) - } else { - _mm_add_ps(a, _mm_mul_ps(b, c)) +pub(crate) fn _mm_opt_fmlaf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { + unsafe { + if FMA { + _mm_fmadd_ps(b, c, a) + } else { + _mm_add_ps(a, _mm_mul_ps(b, c)) + } } } #[inline(always)] -pub(crate) unsafe fn _mm_opt_fnmlsf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { - if FMA { - _mm_fmsub_ps(b, c, a) - } else { - _mm_sub_ps(_mm_mul_ps(b, c), a) +pub(crate) fn _mm_opt_fnmlsf_ps(a: __m128, b: __m128, c: __m128) -> __m128 { + unsafe { + if FMA { + _mm_fmsub_ps(b, c, a) + } else { + _mm_sub_ps(_mm_mul_ps(b, c), a) + } } } diff --git a/src/sse/v_load_store.rs b/src/sse/v_load_store.rs index e5eeab79..b5ccb197 100644 --- a/src/sse/v_load_store.rs +++ b/src/sse/v_load_store.rs @@ -37,104 +37,127 @@ use std::arch::x86_64::*; #[inline(always)] pub(crate) unsafe fn _mm_load_pack_x4(ptr: *const u8) -> (__m128i, __m128i, __m128i, __m128i) { - let row0 = _mm_loadu_si128(ptr as *const __m128i); - let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); - let row2 = _mm_loadu_si128(ptr.add(32) as *const __m128i); - let row3 = _mm_loadu_si128(ptr.add(48) as *const __m128i); - (row0, row1, row2, row3) + unsafe { + let row0 = _mm_loadu_si128(ptr as *const __m128i); + let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); + let row2 = _mm_loadu_si128(ptr.add(32) as *const __m128i); + let row3 = _mm_loadu_si128(ptr.add(48) as *const __m128i); + (row0, row1, row2, row3) + } } #[inline(always)] pub(crate) unsafe fn _mm_load_pack_ps_x4(ptr: *const f32) -> (__m128, __m128, __m128, __m128) { - let row0 = _mm_loadu_ps(ptr); - let row1 = _mm_loadu_ps(ptr.add(4)); - let row2 = _mm_loadu_ps(ptr.add(8)); - let row3 = _mm_loadu_ps(ptr.add(12)); - (row0, row1, row2, row3) + unsafe { + let row0 = _mm_loadu_ps(ptr); + let row1 = _mm_loadu_ps(ptr.add(4)); + let row2 = _mm_loadu_ps(ptr.add(8)); + let row3 = _mm_loadu_ps(ptr.add(12)); + (row0, row1, row2, row3) + } } #[inline(always)] -pub(crate) unsafe fn _mm_load_pack_ps_x2(ptr: *const f32) -> (__m128, __m128) { - let row0 = _mm_loadu_ps(ptr); - let row1 = _mm_loadu_ps(ptr.add(4)); - (row0, row1) +pub(crate) fn _mm_load_pack_ps_x2(ptr: *const f32) -> (__m128, __m128) { + unsafe { + let row0 = _mm_loadu_ps(ptr); + let row1 = _mm_loadu_ps(ptr.add(4)); + (row0, row1) + } } #[inline(always)] -pub(crate) unsafe fn _mm_store_pack_x4(ptr: *mut u8, v: (__m128i, __m128i, __m128i, __m128i)) { - _mm_storeu_si128(ptr as *mut __m128i, v.0); - _mm_storeu_si128(ptr.add(16) as *mut __m128i, v.1); - _mm_storeu_si128(ptr.add(32) as *mut __m128i, v.2); - _mm_storeu_si128(ptr.add(48) as *mut __m128i, v.3); +pub(crate) fn _mm_store_pack_x4(ptr: *mut u8, v: (__m128i, __m128i, __m128i, __m128i)) { + unsafe { + _mm_storeu_si128(ptr as *mut __m128i, v.0); + _mm_storeu_si128(ptr.add(16) as *mut __m128i, v.1); + _mm_storeu_si128(ptr.add(32) as *mut __m128i, v.2); + _mm_storeu_si128(ptr.add(48) as *mut __m128i, v.3); + } } #[inline(always)] -pub(crate) unsafe fn _mm_store_pack_ps_x4(ptr: *mut f32, v: (__m128, __m128, __m128, __m128)) { - _mm_storeu_ps(ptr, v.0); - _mm_storeu_ps(ptr.add(4), v.1); - _mm_storeu_ps(ptr.add(8), v.2); - _mm_storeu_ps(ptr.add(12), v.3); +pub(crate) fn _mm_store_pack_ps_x4(ptr: *mut f32, v: (__m128, __m128, __m128, __m128)) { + unsafe { + _mm_storeu_ps(ptr, v.0); + _mm_storeu_ps(ptr.add(4), v.1); + _mm_storeu_ps(ptr.add(8), v.2); + _mm_storeu_ps(ptr.add(12), v.3); + } } #[inline(always)] -pub(crate) unsafe fn _mm_store_pack_ps_x3(ptr: *mut f32, v: (__m128, __m128, __m128)) { - _mm_storeu_ps(ptr, v.0); - _mm_storeu_ps(ptr.add(4), v.1); - _mm_storeu_ps(ptr.add(8), v.2); +pub(crate) fn _mm_store_pack_ps_x3(ptr: *mut f32, v: (__m128, __m128, __m128)) { + unsafe { + _mm_storeu_ps(ptr, v.0); + _mm_storeu_ps(ptr.add(4), v.1); + _mm_storeu_ps(ptr.add(8), v.2); + } } #[inline(always)] pub(crate) unsafe fn _mm_store_pack_ps_x2(ptr: *mut f32, v: (__m128, __m128)) { - _mm_storeu_ps(ptr, v.0); - _mm_storeu_ps(ptr.add(4), v.1); + unsafe { + _mm_storeu_ps(ptr, v.0); + _mm_storeu_ps(ptr.add(4), v.1); + } } #[inline(always)] pub(crate) unsafe fn _mm_load_pack_x3(ptr: *const u8) -> (__m128i, __m128i, __m128i) { - let row0 = _mm_loadu_si128(ptr as *const __m128i); - let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); - let row2 = _mm_loadu_si128(ptr.add(32) as *const __m128i); - (row0, row1, row2) + unsafe { + let row0 = _mm_loadu_si128(ptr as *const __m128i); + let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); + let row2 = _mm_loadu_si128(ptr.add(32) as *const __m128i); + (row0, row1, row2) + } } #[inline(always)] pub(crate) unsafe fn _mm_load_pack_x2(ptr: *const u8) -> (__m128i, __m128i) { - let row0 = _mm_loadu_si128(ptr as *const __m128i); - let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); - (row0, row1) + unsafe { + let row0 = _mm_loadu_si128(ptr as *const __m128i); + let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); + (row0, row1) + } } #[inline(always)] pub(crate) unsafe fn _mm_store_pack_x3(ptr: *mut u8, v: (__m128i, __m128i, __m128i)) { - _mm_storeu_si128(ptr as *mut __m128i, v.0); - _mm_storeu_si128(ptr.add(16) as *mut __m128i, v.1); - _mm_storeu_si128(ptr.add(32) as *mut __m128i, v.2); + unsafe { + _mm_storeu_si128(ptr as *mut __m128i, v.0); + _mm_storeu_si128(ptr.add(16) as *mut __m128i, v.1); + _mm_storeu_si128(ptr.add(32) as *mut __m128i, v.2); + } } #[inline(always)] pub(crate) unsafe fn _mm_store_pack_x2(ptr: *mut u8, v: (__m128i, __m128i)) { - _mm_storeu_si128(ptr as *mut __m128i, v.0); - _mm_storeu_si128(ptr.add(16) as *mut __m128i, v.1); + unsafe { + _mm_storeu_si128(ptr as *mut __m128i, v.0); + _mm_storeu_si128(ptr.add(16) as *mut __m128i, v.1); + } } #[inline(always)] pub(crate) unsafe fn _mm_store_interleave_rgb(ptr: *mut u8, v: (__m128i, __m128i, __m128i)) { - let (v0, v1, v2) = _mm_interleave_rgb(v.0, v.1, v.2); - _mm_store_pack_x3(ptr, (v0, v1, v2)); + unsafe { + let (v0, v1, v2) = _mm_interleave_rgb(v.0, v.1, v.2); + _mm_store_pack_x3(ptr, (v0, v1, v2)); + } } #[inline(always)] -pub(crate) unsafe fn _mm_store_interleave_rgb_half(ptr: *mut u8, v: (__m128i, __m128i, __m128i)) { - let (v0, v1, _) = _mm_interleave_rgb(v.0, v.1, v.2); - _mm_storeu_si128(ptr as *mut __m128i, v0); - _mm_storeu_si64(ptr.add(16), v1); +pub(crate) fn _mm_store_interleave_rgb_half(ptr: *mut u8, v: (__m128i, __m128i, __m128i)) { + unsafe { + let (v0, v1, _) = _mm_interleave_rgb(v.0, v.1, v.2); + _mm_storeu_si128(ptr as *mut __m128i, v0); + _mm_storeu_si64(ptr.add(16), v1); + } } #[inline(always)] -pub(crate) unsafe fn _mm_store_interleave_rgba( - ptr: *mut u8, - v: (__m128i, __m128i, __m128i, __m128i), -) { +pub(crate) fn _mm_store_interleave_rgba(ptr: *mut u8, v: (__m128i, __m128i, __m128i, __m128i)) { let (v0, v1, v2, v3) = _mm_interleave_rgba(v.0, v.1, v.2, v.3); _mm_store_pack_x4(ptr, (v0, v1, v2, v3)); } @@ -144,63 +167,75 @@ pub(crate) unsafe fn _mm_store_interleave_rgba_half( ptr: *mut u8, v: (__m128i, __m128i, __m128i, __m128i), ) { - let (v0, v1, _, _) = _mm_interleave_rgba(v.0, v.1, v.2, v.3); - _mm_storeu_si128(ptr as *mut __m128i, v0); - _mm_storeu_si128(ptr.add(16) as *mut __m128i, v1); + unsafe { + let (v0, v1, _, _) = _mm_interleave_rgba(v.0, v.1, v.2, v.3); + _mm_storeu_si128(ptr as *mut __m128i, v0); + _mm_storeu_si128(ptr.add(16) as *mut __m128i, v1); + } } #[inline(always)] -pub(crate) unsafe fn _mm_load_deinterleave_rgb(ptr: *const u8) -> (__m128i, __m128i, __m128i) { - let row0 = _mm_loadu_si128(ptr as *const __m128i); - let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); - let row2 = _mm_loadu_si128(ptr.add(32) as *const __m128i); - _mm_deinterleave_rgb(row0, row1, row2) +pub(crate) fn _mm_load_deinterleave_rgb(ptr: *const u8) -> (__m128i, __m128i, __m128i) { + unsafe { + let row0 = _mm_loadu_si128(ptr as *const __m128i); + let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); + let row2 = _mm_loadu_si128(ptr.add(32) as *const __m128i); + _mm_deinterleave_rgb(row0, row1, row2) + } } #[inline(always)] pub(crate) unsafe fn _mm_load_deinterleave_rgb_half(ptr: *const u8) -> (__m128i, __m128i, __m128i) { - let row0 = _mm_loadu_si128(ptr as *const __m128i); - let row1 = _mm_loadu_si64(ptr.add(16)); - _mm_deinterleave_rgb(row0, row1, _mm_setzero_si128()) + unsafe { + let row0 = _mm_loadu_si128(ptr as *const __m128i); + let row1 = _mm_loadu_si64(ptr.add(16)); + _mm_deinterleave_rgb(row0, row1, _mm_setzero_si128()) + } } #[inline(always)] pub(crate) unsafe fn _mm_load_deinterleave_rgba( ptr: *const u8, ) -> (__m128i, __m128i, __m128i, __m128i) { - let row0 = _mm_loadu_si128(ptr as *const __m128i); - let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); - let row2 = _mm_loadu_si128(ptr.add(32) as *const __m128i); - let row3 = _mm_loadu_si128(ptr.add(48) as *const __m128i); - _mm_deinterleave_rgba(row0, row1, row2, row3) + unsafe { + let row0 = _mm_loadu_si128(ptr as *const __m128i); + let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); + let row2 = _mm_loadu_si128(ptr.add(32) as *const __m128i); + let row3 = _mm_loadu_si128(ptr.add(48) as *const __m128i); + _mm_deinterleave_rgba(row0, row1, row2, row3) + } } #[inline(always)] pub(crate) unsafe fn _mm_load_deinterleave_rgba_half( ptr: *const u8, ) -> (__m128i, __m128i, __m128i, __m128i) { - let row0 = _mm_loadu_si128(ptr as *const __m128i); - let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); - _mm_deinterleave_rgba(row0, row1, _mm_setzero_si128(), _mm_setzero_si128()) + unsafe { + let row0 = _mm_loadu_si128(ptr as *const __m128i); + let row1 = _mm_loadu_si128(ptr.add(16) as *const __m128i); + _mm_deinterleave_rgba(row0, row1, _mm_setzero_si128(), _mm_setzero_si128()) + } } #[inline(always)] -pub(crate) unsafe fn _mm_load_deinterleave_rgba_ps( - ptr: *const f32, -) -> (__m128, __m128, __m128, __m128) { - let row0 = _mm_loadu_ps(ptr); - let row1 = _mm_loadu_ps(ptr.add(4)); - let row2 = _mm_loadu_ps(ptr.add(8)); - let row3 = _mm_loadu_ps(ptr.add(12)); - _mm_deinterleave_rgba_ps(row0, row1, row2, row3) +pub(crate) fn _mm_load_deinterleave_rgba_ps(ptr: *const f32) -> (__m128, __m128, __m128, __m128) { + unsafe { + let row0 = _mm_loadu_ps(ptr); + let row1 = _mm_loadu_ps(ptr.add(4)); + let row2 = _mm_loadu_ps(ptr.add(8)); + let row3 = _mm_loadu_ps(ptr.add(12)); + _mm_deinterleave_rgba_ps(row0, row1, row2, row3) + } } #[inline(always)] -pub(crate) unsafe fn _mm_load_deinterleave_rgb_ps(ptr: *const f32) -> (__m128, __m128, __m128) { - let row0 = _mm_loadu_ps(ptr); - let row1 = _mm_loadu_ps(ptr.add(4)); - let row2 = _mm_loadu_ps(ptr.add(8)); - _mm_deinterleave_rgb_ps(row0, row1, row2) +pub(crate) fn _mm_load_deinterleave_rgb_ps(ptr: *const f32) -> (__m128, __m128, __m128) { + unsafe { + let row0 = _mm_loadu_ps(ptr); + let row1 = _mm_loadu_ps(ptr.add(4)); + let row2 = _mm_loadu_ps(ptr.add(8)); + _mm_deinterleave_rgb_ps(row0, row1, row2) + } } #[inline(always)] diff --git a/src/stack_blur_image.rs b/src/stack_blur_image.rs index 129c2811..da9ef3f1 100644 --- a/src/stack_blur_image.rs +++ b/src/stack_blur_image.rs @@ -27,8 +27,8 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::{ - stack_blur, stack_blur_f32, stack_blur_u16, AnisotropicRadius, BlurImageMut, FastBlurChannels, - ThreadingPolicy, + AnisotropicRadius, BlurImageMut, FastBlurChannels, ThreadingPolicy, stack_blur, stack_blur_f32, + stack_blur_u16, }; use image::{ DynamicImage, GrayAlphaImage, GrayImage, ImageBuffer, Luma, LumaA, Rgb, RgbImage, Rgba, diff --git a/src/stackblur/avx/horizontal.rs b/src/stackblur/avx/horizontal.rs index 5771909f..fe2f97f1 100644 --- a/src/stackblur/avx/horizontal.rs +++ b/src/stackblur/avx/horizontal.rs @@ -29,6 +29,7 @@ use crate::sse::{load_u8_s32_fast, store_u8_s32}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] @@ -44,10 +45,10 @@ impl Default for HorizontalAvxStackBlurPass { #[repr(C, align(32))] #[derive(Copy, Clone, Default, Debug)] -struct StackBlurCache([i32; 8]); +pub(crate) struct AvxI32x8([i32; 8]); #[target_feature(enable = "avx2")] -unsafe fn avx_horiz_pass_impl( +fn avx_horiz_pass_impl( pixels: &UnsafeSlice, stride: u32, width: u32, @@ -56,410 +57,418 @@ unsafe fn avx_horiz_pass_impl( thread: usize, total_threads: usize, ) { - let pixels: &UnsafeSlice = std::mem::transmute(pixels); - let div = ((radius * 2) + 1) as usize; - let mut xp; - let mut sp; - let mut stack_start; - let mut stacks = vec![StackBlurCache::default(); div * 2]; + unsafe { + let div = ((radius * 2) + 1) as usize; + let mut xp; + let mut sp; + let mut stack_start; + let mut scratch_buffer = ScratchBuffer::::new(div * 2); + let stacks = scratch_buffer.as_mut_slice(); - let v_mul_value = _mm256_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); + let v_mul_value = _mm256_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - let wm = width - 1; - let div = (radius * 2) + 1; + let wm = width - 1; + let div = (radius * 2) + 1; - let min_y = thread * height as usize / total_threads; - let max_y = (thread + 1) * height as usize / total_threads; + let min_y = thread * height as usize / total_threads; + let max_y = (thread + 1) * height as usize / total_threads; - let mut yy = min_y; + let mut yy = min_y; - while yy + 4 <= max_y { - let mut sums0 = _mm256_setzero_si256(); - let mut sums1 = _mm256_setzero_si256(); + while yy + 4 <= max_y { + let mut sums0 = _mm256_setzero_si256(); + let mut sums1 = _mm256_setzero_si256(); - let mut sum_in0 = _mm256_setzero_si256(); - let mut sum_in1 = _mm256_setzero_si256(); + let mut sum_in0 = _mm256_setzero_si256(); + let mut sum_in1 = _mm256_setzero_si256(); - let mut sum_out0 = _mm256_setzero_si256(); - let mut sum_out1 = _mm256_setzero_si256(); + let mut sum_out0 = _mm256_setzero_si256(); + let mut sum_out1 = _mm256_setzero_si256(); - let mut src_ptr0 = stride as usize * yy; - let mut src_ptr1 = stride as usize * (yy + 1); - let mut src_ptr2 = stride as usize * (yy + 2); - let mut src_ptr3 = stride as usize * (yy + 3); + let mut src_ptr0 = stride as usize * yy; + let mut src_ptr1 = stride as usize * (yy + 1); + let mut src_ptr2 = stride as usize * (yy + 2); + let mut src_ptr3 = stride as usize * (yy + 3); - let src_ld0 = pixels.slice.as_ptr().add(src_ptr0) as *const i32; - let src_ld1 = pixels.slice.as_ptr().add(src_ptr1) as *const i32; - let src_ld2 = pixels.slice.as_ptr().add(src_ptr2) as *const i32; - let src_ld3 = pixels.slice.as_ptr().add(src_ptr3) as *const i32; + let src_ld0 = pixels.slice.as_ptr().add(src_ptr0) as *const i32; + let src_ld1 = pixels.slice.as_ptr().add(src_ptr1) as *const i32; + let src_ld2 = pixels.slice.as_ptr().add(src_ptr2) as *const i32; + let src_ld3 = pixels.slice.as_ptr().add(src_ptr3) as *const i32; - let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); - let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); - let src_pixel2 = load_u8_s32_fast::(src_ld2 as *const u8); - let src_pixel3 = load_u8_s32_fast::(src_ld3 as *const u8); - - let px0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel0), src_pixel1); - let px1 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel2), src_pixel3); - - for i in 0..=radius { - let stack_value = stacks.as_mut_ptr().add(i as usize * 2); - _mm256_store_si256(stack_value as *mut _, px0); - _mm256_store_si256(stack_value.add(1) as *mut _, px1); - - let w = _mm256_set1_epi32(i as i32 + 1); - sums0 = _mm256_add_epi32(sums0, _mm256_madd_epi16(px0, w)); - sums1 = _mm256_add_epi32(sums1, _mm256_madd_epi16(px1, w)); - - sum_out0 = _mm256_add_epi32(sum_out0, px0); - sum_out1 = _mm256_add_epi32(sum_out1, px1); - } - - for i in 1..=radius { - if i <= wm { - src_ptr0 += CN; - src_ptr1 += CN; - } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 2); - - let src_pixel0 = - load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr0) as *const u8); - let src_pixel1 = - load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr1) as *const u8); - let src_pixel2 = - load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr2) as *const u8); - let src_pixel3 = - load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr3) as *const u8); + let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); + let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); + let src_pixel2 = load_u8_s32_fast::(src_ld2 as *const u8); + let src_pixel3 = load_u8_s32_fast::(src_ld3 as *const u8); let px0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel0), src_pixel1); let px1 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel2), src_pixel3); - _mm256_store_si256(stack_ptr as *mut _, px0); - _mm256_store_si256(stack_ptr.add(1) as *mut _, px1); - - let w = _mm256_set1_epi32(radius as i32 + 1 - i as i32); - - sums0 = _mm256_add_epi32(sums0, _mm256_madd_epi16(px0, w)); - sums1 = _mm256_add_epi32(sums1, _mm256_madd_epi16(px1, w)); - - sum_in0 = _mm256_add_epi32(sum_in0, px0); - sum_in1 = _mm256_add_epi32(sum_in1, px1); - } - - sp = radius; - xp = radius; - if xp > wm { - xp = wm; - } - - src_ptr0 = CN * xp as usize + yy * stride as usize; - src_ptr1 = CN * xp as usize + (yy + 1) * stride as usize; - src_ptr2 = CN * xp as usize + (yy + 2) * stride as usize; - src_ptr3 = CN * xp as usize + (yy + 3) * stride as usize; - - let mut dst_ptr0 = yy * stride as usize; - let mut dst_ptr1 = (yy + 1) * stride as usize; - let mut dst_ptr2 = (yy + 2) * stride as usize; - let mut dst_ptr3 = (yy + 3) * stride as usize; - - for _ in 0..width { - let store_ld0 = pixels.slice.as_ptr().add(dst_ptr0) as *mut u8; - let store_ld1 = pixels.slice.as_ptr().add(dst_ptr1) as *mut u8; - let store_ld2 = pixels.slice.as_ptr().add(dst_ptr2) as *mut u8; - let store_ld3 = pixels.slice.as_ptr().add(dst_ptr3) as *mut u8; + for i in 0..=radius { + let stack_value = stacks.as_mut_ptr().add(i as usize * 2); + _mm256_store_si256(stack_value as *mut _, px0); + _mm256_store_si256(stack_value.add(1) as *mut _, px1); - let o0 = _mm256_cvtepi32_ps(sums0); - let o1 = _mm256_cvtepi32_ps(sums1); + let w = _mm256_set1_epi32(i as i32 + 1); + sums0 = _mm256_add_epi32(sums0, _mm256_madd_epi16(px0, w)); + sums1 = _mm256_add_epi32(sums1, _mm256_madd_epi16(px1, w)); - let r0 = _mm256_mul_ps(o0, v_mul_value); - let r1 = _mm256_mul_ps(o1, v_mul_value); + sum_out0 = _mm256_add_epi32(sum_out0, px0); + sum_out1 = _mm256_add_epi32(sum_out1, px1); + } - let cv0 = _mm256_cvtps_epi32(r0); - let cv1 = _mm256_cvtps_epi32(r1); + for i in 1..=radius { + if i <= wm { + src_ptr0 += CN; + src_ptr1 += CN; + } + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 2); + + let src_pixel0 = + load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr0) as *const u8); + let src_pixel1 = + load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr1) as *const u8); + let src_pixel2 = + load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr2) as *const u8); + let src_pixel3 = + load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr3) as *const u8); + + let px0 = + _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel0), src_pixel1); + let px1 = + _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel2), src_pixel3); + + _mm256_store_si256(stack_ptr as *mut _, px0); + _mm256_store_si256(stack_ptr.add(1) as *mut _, px1); + + let w = _mm256_set1_epi32(radius as i32 + 1 - i as i32); + + sums0 = _mm256_add_epi32(sums0, _mm256_madd_epi16(px0, w)); + sums1 = _mm256_add_epi32(sums1, _mm256_madd_epi16(px1, w)); + + sum_in0 = _mm256_add_epi32(sum_in0, px0); + sum_in1 = _mm256_add_epi32(sum_in1, px1); + } - store_u8_s32::(store_ld0, _mm256_castsi256_si128(cv0)); - store_u8_s32::(store_ld1, _mm256_extracti128_si256::<1>(cv0)); - store_u8_s32::(store_ld2, _mm256_castsi256_si128(cv1)); - store_u8_s32::(store_ld3, _mm256_extracti128_si256::<1>(cv1)); + sp = radius; + xp = radius; + if xp > wm { + xp = wm; + } - dst_ptr0 += CN; - dst_ptr1 += CN; - dst_ptr2 += CN; - dst_ptr3 += CN; + src_ptr0 = CN * xp as usize + yy * stride as usize; + src_ptr1 = CN * xp as usize + (yy + 1) * stride as usize; + src_ptr2 = CN * xp as usize + (yy + 2) * stride as usize; + src_ptr3 = CN * xp as usize + (yy + 3) * stride as usize; + + let mut dst_ptr0 = yy * stride as usize; + let mut dst_ptr1 = (yy + 1) * stride as usize; + let mut dst_ptr2 = (yy + 2) * stride as usize; + let mut dst_ptr3 = (yy + 3) * stride as usize; + + for _ in 0..width { + let store_ld0 = pixels.slice.as_ptr().add(dst_ptr0) as *mut u8; + let store_ld1 = pixels.slice.as_ptr().add(dst_ptr1) as *mut u8; + let store_ld2 = pixels.slice.as_ptr().add(dst_ptr2) as *mut u8; + let store_ld3 = pixels.slice.as_ptr().add(dst_ptr3) as *mut u8; + + let o0 = _mm256_cvtepi32_ps(sums0); + let o1 = _mm256_cvtepi32_ps(sums1); + + let r0 = _mm256_mul_ps(o0, v_mul_value); + let r1 = _mm256_mul_ps(o1, v_mul_value); + + let cv0 = _mm256_cvtps_epi32(r0); + let cv1 = _mm256_cvtps_epi32(r1); + + store_u8_s32::(store_ld0, _mm256_castsi256_si128(cv0)); + store_u8_s32::(store_ld1, _mm256_extracti128_si256::<1>(cv0)); + store_u8_s32::(store_ld2, _mm256_castsi256_si128(cv1)); + store_u8_s32::(store_ld3, _mm256_extracti128_si256::<1>(cv1)); + + dst_ptr0 += CN; + dst_ptr1 += CN; + dst_ptr2 += CN; + dst_ptr3 += CN; + + sums0 = _mm256_sub_epi32(sums0, sum_out0); + sums1 = _mm256_sub_epi32(sums1, sum_out1); + + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } + let stack = stacks.as_mut_ptr().add(stack_start as usize * 2); + + let stack_val0 = _mm256_load_si256(stack as *const _); + let stack_val1 = _mm256_load_si256(stack.add(1) as *const _); + + sum_out0 = _mm256_sub_epi32(sum_out0, stack_val0); + sum_out1 = _mm256_sub_epi32(sum_out1, stack_val1); + + if xp < wm { + src_ptr0 += CN; + src_ptr1 += CN; + src_ptr2 += CN; + src_ptr3 += CN; + xp += 1; + } + + let src_ld0 = pixels.slice.as_ptr().add(src_ptr0); + let src_ld1 = pixels.slice.as_ptr().add(src_ptr1); + let src_ld2 = pixels.slice.as_ptr().add(src_ptr2); + let src_ld3 = pixels.slice.as_ptr().add(src_ptr3); + + let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); + let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); + let src_pixel2 = load_u8_s32_fast::(src_ld2 as *const u8); + let src_pixel3 = load_u8_s32_fast::(src_ld3 as *const u8); + + let px0 = + _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel0), src_pixel1); + let px1 = + _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel2), src_pixel3); + + _mm256_store_si256(stack as *mut _, px0); + _mm256_store_si256(stack.add(1) as *mut _, px1); + + sum_in0 = _mm256_add_epi32(sum_in0, px0); + sum_in1 = _mm256_add_epi32(sum_in1, px1); + + sums0 = _mm256_add_epi32(sums0, sum_in0); + sums1 = _mm256_add_epi32(sums1, sum_in1); + + sp += 1; + if sp >= div { + sp = 0; + } + let stack = stacks.as_mut_ptr().add(sp as usize * 2); + let stack_val0 = _mm256_loadu_si256(stack as *const __m256i); + let stack_val1 = _mm256_loadu_si256(stack.add(1) as *const __m256i); + + sum_out0 = _mm256_add_epi32(sum_out0, stack_val0); + sum_out1 = _mm256_add_epi32(sum_out1, stack_val1); + + sum_in0 = _mm256_sub_epi32(sum_in0, stack_val0); + sum_in1 = _mm256_sub_epi32(sum_in1, stack_val1); + } - sums0 = _mm256_sub_epi32(sums0, sum_out0); - sums1 = _mm256_sub_epi32(sums1, sum_out1); + yy += 4; + } - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } - let stack = stacks.as_mut_ptr().add(stack_start as usize * 2); + while yy + 2 <= max_y { + let mut sums0 = _mm256_setzero_si256(); - let stack_val0 = _mm256_load_si256(stack as *const _); - let stack_val1 = _mm256_load_si256(stack.add(1) as *const _); + let mut sum_in0 = _mm256_setzero_si256(); - sum_out0 = _mm256_sub_epi32(sum_out0, stack_val0); - sum_out1 = _mm256_sub_epi32(sum_out1, stack_val1); + let mut sum_out0 = _mm256_setzero_si256(); - if xp < wm { - src_ptr0 += CN; - src_ptr1 += CN; - src_ptr2 += CN; - src_ptr3 += CN; - xp += 1; - } + let mut src_ptr0 = stride as usize * yy; + let mut src_ptr1 = stride as usize * (yy + 1); - let src_ld0 = pixels.slice.as_ptr().add(src_ptr0); - let src_ld1 = pixels.slice.as_ptr().add(src_ptr1); - let src_ld2 = pixels.slice.as_ptr().add(src_ptr2); - let src_ld3 = pixels.slice.as_ptr().add(src_ptr3); + let src_ld0 = pixels.slice.as_ptr().add(src_ptr0) as *const i32; + let src_ld1 = pixels.slice.as_ptr().add(src_ptr1) as *const i32; let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); - let src_pixel2 = load_u8_s32_fast::(src_ld2 as *const u8); - let src_pixel3 = load_u8_s32_fast::(src_ld3 as *const u8); let px0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel0), src_pixel1); - let px1 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel2), src_pixel3); - - _mm256_store_si256(stack as *mut _, px0); - _mm256_store_si256(stack.add(1) as *mut _, px1); - sum_in0 = _mm256_add_epi32(sum_in0, px0); - sum_in1 = _mm256_add_epi32(sum_in1, px1); + for i in 0..=radius { + let stack_value = stacks.as_mut_ptr().add(i as usize * 2); + _mm256_store_si256(stack_value as *mut _, px0); - sums0 = _mm256_add_epi32(sums0, sum_in0); - sums1 = _mm256_add_epi32(sums1, sum_in1); + let w = _mm256_set1_epi32(i as i32 + 1); + sums0 = _mm256_add_epi32(sums0, _mm256_madd_epi16(px0, w)); - sp += 1; - if sp >= div { - sp = 0; + sum_out0 = _mm256_add_epi32(sum_out0, px0); } - let stack = stacks.as_mut_ptr().add(sp as usize * 2); - let stack_val0 = _mm256_loadu_si256(stack as *const __m256i); - let stack_val1 = _mm256_loadu_si256(stack.add(1) as *const __m256i); - - sum_out0 = _mm256_add_epi32(sum_out0, stack_val0); - sum_out1 = _mm256_add_epi32(sum_out1, stack_val1); - - sum_in0 = _mm256_sub_epi32(sum_in0, stack_val0); - sum_in1 = _mm256_sub_epi32(sum_in1, stack_val1); - } - - yy += 4; - } - - while yy + 2 <= max_y { - let mut sums0 = _mm256_setzero_si256(); - - let mut sum_in0 = _mm256_setzero_si256(); - - let mut sum_out0 = _mm256_setzero_si256(); - - let mut src_ptr0 = stride as usize * yy; - let mut src_ptr1 = stride as usize * (yy + 1); - let src_ld0 = pixels.slice.as_ptr().add(src_ptr0) as *const i32; - let src_ld1 = pixels.slice.as_ptr().add(src_ptr1) as *const i32; + for i in 1..=radius { + if i <= wm { + src_ptr0 += CN; + src_ptr1 += CN; + } + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 2); + let src_pixel0 = + load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr0) as *const u8); + let src_pixel1 = + load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr1) as *const u8); - let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); - let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); + let px0 = + _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel0), src_pixel1); - let px0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel0), src_pixel1); + _mm256_store_si256(stack_ptr as *mut _, px0); - for i in 0..=radius { - let stack_value = stacks.as_mut_ptr().add(i as usize * 2); - _mm256_store_si256(stack_value as *mut _, px0); + let w = _mm256_set1_epi32(radius as i32 + 1 - i as i32); - let w = _mm256_set1_epi32(i as i32 + 1); - sums0 = _mm256_add_epi32(sums0, _mm256_madd_epi16(px0, w)); + sums0 = _mm256_add_epi32(sums0, _mm256_madd_epi16(px0, w)); - sum_out0 = _mm256_add_epi32(sum_out0, px0); - } - - for i in 1..=radius { - if i <= wm { - src_ptr0 += CN; - src_ptr1 += CN; + sum_in0 = _mm256_add_epi32(sum_in0, px0); } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 2); - let src_pixel0 = - load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr0) as *const u8); - let src_pixel1 = - load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr1) as *const u8); - - let px0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel0), src_pixel1); - _mm256_store_si256(stack_ptr as *mut _, px0); + sp = radius; + xp = radius; + if xp > wm { + xp = wm; + } - let w = _mm256_set1_epi32(radius as i32 + 1 - i as i32); + src_ptr0 = CN * xp as usize + yy * stride as usize; + src_ptr1 = CN * xp as usize + (yy + 1) * stride as usize; - sums0 = _mm256_add_epi32(sums0, _mm256_madd_epi16(px0, w)); + let mut dst_ptr0 = yy * stride as usize; + let mut dst_ptr1 = (yy + 1) * stride as usize; - sum_in0 = _mm256_add_epi32(sum_in0, px0); - } + for _ in 0..width { + let store_ld0 = pixels.slice.as_ptr().add(dst_ptr0) as *mut u8; + let store_ld1 = pixels.slice.as_ptr().add(dst_ptr1) as *mut u8; - sp = radius; - xp = radius; - if xp > wm { - xp = wm; - } + let o0 = _mm256_cvtepi32_ps(sums0); - src_ptr0 = CN * xp as usize + yy * stride as usize; - src_ptr1 = CN * xp as usize + (yy + 1) * stride as usize; + let r0 = _mm256_mul_ps(o0, v_mul_value); - let mut dst_ptr0 = yy * stride as usize; - let mut dst_ptr1 = (yy + 1) * stride as usize; + let cv0 = _mm256_cvtps_epi32(r0); - for _ in 0..width { - let store_ld0 = pixels.slice.as_ptr().add(dst_ptr0) as *mut u8; - let store_ld1 = pixels.slice.as_ptr().add(dst_ptr1) as *mut u8; + store_u8_s32::(store_ld0, _mm256_castsi256_si128(cv0)); + store_u8_s32::(store_ld1, _mm256_extracti128_si256::<1>(cv0)); - let o0 = _mm256_cvtepi32_ps(sums0); + dst_ptr0 += CN; + dst_ptr1 += CN; - let r0 = _mm256_mul_ps(o0, v_mul_value); + sums0 = _mm256_sub_epi32(sums0, sum_out0); - let cv0 = _mm256_cvtps_epi32(r0); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } + let stack = stacks.as_mut_ptr().add(stack_start as usize * 2); - store_u8_s32::(store_ld0, _mm256_castsi256_si128(cv0)); - store_u8_s32::(store_ld1, _mm256_extracti128_si256::<1>(cv0)); + let stack_val0 = _mm256_load_si256(stack as *const _); - dst_ptr0 += CN; - dst_ptr1 += CN; + sum_out0 = _mm256_sub_epi32(sum_out0, stack_val0); - sums0 = _mm256_sub_epi32(sums0, sum_out0); + if xp < wm { + src_ptr0 += CN; + src_ptr1 += CN; + xp += 1; + } - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } - let stack = stacks.as_mut_ptr().add(stack_start as usize * 2); + let src_ld0 = pixels.slice.as_ptr().add(src_ptr0); + let src_ld1 = pixels.slice.as_ptr().add(src_ptr1); - let stack_val0 = _mm256_load_si256(stack as *const _); + let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); + let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); - sum_out0 = _mm256_sub_epi32(sum_out0, stack_val0); + let px0 = + _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel0), src_pixel1); - if xp < wm { - src_ptr0 += CN; - src_ptr1 += CN; - xp += 1; - } - - let src_ld0 = pixels.slice.as_ptr().add(src_ptr0); - let src_ld1 = pixels.slice.as_ptr().add(src_ptr1); - - let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); - let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); + _mm256_store_si256(stack as *mut _, px0); - let px0 = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(src_pixel0), src_pixel1); + sum_in0 = _mm256_add_epi32(sum_in0, px0); - _mm256_store_si256(stack as *mut _, px0); + sums0 = _mm256_add_epi32(sums0, sum_in0); - sum_in0 = _mm256_add_epi32(sum_in0, px0); + sp += 1; + if sp >= div { + sp = 0; + } + let stack = stacks.as_mut_ptr().add(sp as usize * 2); + let stack_val0 = _mm256_load_si256(stack as *const _); - sums0 = _mm256_add_epi32(sums0, sum_in0); + sum_out0 = _mm256_add_epi32(sum_out0, stack_val0); - sp += 1; - if sp >= div { - sp = 0; + sum_in0 = _mm256_sub_epi32(sum_in0, stack_val0); } - let stack = stacks.as_mut_ptr().add(sp as usize * 2); - let stack_val0 = _mm256_load_si256(stack as *const _); - - sum_out0 = _mm256_add_epi32(sum_out0, stack_val0); - sum_in0 = _mm256_sub_epi32(sum_in0, stack_val0); + yy += 2; } - yy += 2; - } - - for y in yy..max_y { - let mut sums = _mm_setzero_si128(); - let mut sum_in = _mm_setzero_si128(); - let mut sum_out = _mm_setzero_si128(); - - let mut src_ptr = stride as usize * y; + for y in yy..max_y { + let mut sums = _mm_setzero_si128(); + let mut sum_in = _mm_setzero_si128(); + let mut sum_out = _mm_setzero_si128(); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + let mut src_ptr = stride as usize * y; - for i in 0..=radius { - let stack_value = stacks.as_mut_ptr().add(i as usize * 2); - _mm_storeu_si128(stack_value as *mut __m128i, src_pixel); - sums = _mm_add_epi32( - sums, - _mm_madd_epi16(src_pixel, _mm_set1_epi32(i as i32 + 1)), - ); - sum_out = _mm_add_epi32(sum_out, src_pixel); - } - - for i in 1..=radius { - if i <= wm { - src_ptr += CN; - } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 2); let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); - sums = _mm_add_epi32( - sums, - _mm_madd_epi16(src_pixel, _mm_set1_epi32(radius as i32 + 1 - i as i32)), - ); - - sum_in = _mm_add_epi32(sum_in, src_pixel); - } - - sp = radius; - xp = radius; - if xp > wm { - xp = wm; - } - src_ptr = CN * xp as usize + y * stride as usize; - let mut dst_ptr = y * stride as usize; - for _ in 0..width { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - let result = _mm_cvtps_epi32(_mm_mul_ps( - _mm_cvtepi32_ps(sums), - _mm256_castps256_ps128(v_mul_value), - )); - store_u8_s32::(store_ld, result); - dst_ptr += CN; - - sums = _mm_sub_epi32(sums, sum_out); - - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; + for i in 0..=radius { + let stack_value = stacks.as_mut_ptr().add(i as usize * 2); + _mm_store_si128(stack_value as *mut __m128i, src_pixel); + sums = _mm_add_epi32( + sums, + _mm_madd_epi16(src_pixel, _mm_set1_epi32(i as i32 + 1)), + ); + sum_out = _mm_add_epi32(sum_out, src_pixel); } - let stack = stacks.as_mut_ptr().add(stack_start as usize * 2); - - let stack_val = _mm_loadu_si128(stack as *const __m128i); - - sum_out = _mm_sub_epi32(sum_out, stack_val); - if xp < wm { - src_ptr += CN; - xp += 1; + for i in 1..=radius { + if i <= wm { + src_ptr += CN; + } + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 2); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_store_si128(stack_ptr as *mut __m128i, src_pixel); + sums = _mm_add_epi32( + sums, + _mm_madd_epi16(src_pixel, _mm_set1_epi32(radius as i32 + 1 - i as i32)), + ); + + sum_in = _mm_add_epi32(sum_in, src_pixel); } - let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack as *mut __m128i, src_pixel); - - sum_in = _mm_add_epi32(sum_in, src_pixel); - sums = _mm_add_epi32(sums, sum_in); - - sp += 1; - if sp >= div { - sp = 0; + sp = radius; + xp = radius; + if xp > wm { + xp = wm; } - let stack = stacks.as_mut_ptr().add(sp as usize * 2); - let stack_val = _mm_loadu_si128(stack as *const __m128i); - sum_out = _mm_add_epi32(sum_out, stack_val); - sum_in = _mm_sub_epi32(sum_in, stack_val); + src_ptr = CN * xp as usize + y * stride as usize; + let mut dst_ptr = y * stride as usize; + for _ in 0..width { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + let result = _mm_cvtps_epi32(_mm_mul_ps( + _mm_cvtepi32_ps(sums), + _mm256_castps256_ps128(v_mul_value), + )); + store_u8_s32::(store_ld, result); + dst_ptr += CN; + + sums = _mm_sub_epi32(sums, sum_out); + + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } + let stack = stacks.as_mut_ptr().add(stack_start as usize * 2); + + let stack_val = _mm_load_si128(stack as *const __m128i); + + sum_out = _mm_sub_epi32(sum_out, stack_val); + + if xp < wm { + src_ptr += CN; + xp += 1; + } + + let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_store_si128(stack as *mut __m128i, src_pixel); + + sum_in = _mm_add_epi32(sum_in, src_pixel); + sums = _mm_add_epi32(sums, sum_in); + + sp += 1; + if sp >= div { + sp = 0; + } + let stack = stacks.as_mut_ptr().add(sp as usize * 2); + let stack_val = _mm_load_si128(stack as *const __m128i); + + sum_out = _mm_add_epi32(sum_out, stack_val); + sum_in = _mm_sub_epi32(sum_in, stack_val); + } } } } diff --git a/src/stackblur/avx/horizontal_f32.rs b/src/stackblur/avx/horizontal_f32.rs index 158df805..106b1aa8 100644 --- a/src/stackblur/avx/horizontal_f32.rs +++ b/src/stackblur/avx/horizontal_f32.rs @@ -30,6 +30,7 @@ use crate::sse::{load_f32, store_f32}; use crate::stackblur::avx::vertical_f32::Avx2F32x8; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::arch::x86_64::*; pub(crate) struct HorizontalAvxStackBlurPassFloat32 {} @@ -41,7 +42,7 @@ impl Default for HorizontalAvxStackBlurPassFloat32 { } #[target_feature(enable = "avx2", enable = "fma")] -unsafe fn horiz_f32_pass_stack_impl( +fn horiz_f32_pass_stack_impl( pixels: &UnsafeSlice, stride: u32, width: u32, @@ -50,137 +51,37 @@ unsafe fn horiz_f32_pass_stack_impl( thread: usize, total_threads: usize, ) { - let div = ((radius * 2) + 1) as usize; - let v_mul_value = _mm256_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - let mut xp; - let mut sp; - let mut stack_start; - let mut stacks = vec![Avx2F32x8::default(); div * 2]; - - let wm = width - 1; - let div = (radius * 2) + 1; - - let min_y = thread * height as usize / total_threads; - let max_y = (thread + 1) * height as usize / total_threads; - - let mut cy = min_y; - - while cy + 4 <= max_y { - let mut sums0 = _mm256_setzero_ps(); - let mut sums1 = _mm256_setzero_ps(); - - let mut sum_in0 = _mm256_setzero_ps(); - let mut sum_in1 = _mm256_setzero_ps(); - - let mut sum_out0 = _mm256_setzero_ps(); - let mut sum_out1 = _mm256_setzero_ps(); - - let mut src_ptr0 = stride as usize * cy; - let mut src_ptr1 = stride as usize * (cy + 1); - let mut src_ptr2 = stride as usize * (cy + 2); - let mut src_ptr3 = stride as usize * (cy + 3); - - let src_pixel0 = load_f32::(pixels.slice.as_ptr().add(src_ptr0) as *const f32); - let src_pixel1 = load_f32::(pixels.slice.as_ptr().add(src_ptr1) as *const f32); - let src_pixel2 = load_f32::(pixels.slice.as_ptr().add(src_ptr2) as *const f32); - let src_pixel3 = load_f32::(pixels.slice.as_ptr().add(src_ptr3) as *const f32); - - let x0 = _mm256_setr_m128(src_pixel0, src_pixel1); - let x1 = _mm256_setr_m128(src_pixel2, src_pixel3); - - for i in 0..=radius { - let stack_value = stacks.as_mut_ptr().add(i as usize * 2); - _mm256_store_ps(stack_value.cast(), x0); - _mm256_store_ps(stack_value.add(1).cast(), x1); - let w = _mm256_set1_ps((i + 1) as f32); - sums0 = _mm256_fmadd_ps(x0, w, sums0); - sums1 = _mm256_fmadd_ps(x1, w, sums1); - sum_out0 = _mm256_add_ps(sum_out0, x0); - sum_out1 = _mm256_add_ps(sum_out1, x1); - } - - for i in 1..=radius { - if i <= wm { - src_ptr0 += CN; - src_ptr1 += CN; - src_ptr2 += CN; - src_ptr3 += CN; - } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 2); - let src_pixel0 = load_f32::(pixels.slice.as_ptr().add(src_ptr0) as *const f32); - let src_pixel1 = load_f32::(pixels.slice.as_ptr().add(src_ptr1) as *const f32); - let src_pixel2 = load_f32::(pixels.slice.as_ptr().add(src_ptr2) as *const f32); - let src_pixel3 = load_f32::(pixels.slice.as_ptr().add(src_ptr3) as *const f32); - - let x0 = _mm256_setr_m128(src_pixel0, src_pixel1); - let x1 = _mm256_setr_m128(src_pixel2, src_pixel3); + unsafe { + let div = ((radius * 2) + 1) as usize; + let v_mul_value = _mm256_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); + let mut xp; + let mut sp; + let mut stack_start; + let mut scratch_buffer = ScratchBuffer::::new(div * 2); + let stacks = scratch_buffer.as_mut_slice(); - _mm256_store_ps(stack_ptr.cast(), x0); - _mm256_store_ps(stack_ptr.add(1).cast(), x1); + let wm = width - 1; + let div = (radius * 2) + 1; - let w = _mm256_set1_ps((radius + 1 - i) as f32); - sums0 = _mm256_fmadd_ps(x0, w, sums0); - sums1 = _mm256_fmadd_ps(x1, w, sums1); + let min_y = thread * height as usize / total_threads; + let max_y = (thread + 1) * height as usize / total_threads; - sum_in0 = _mm256_add_ps(sum_in0, x0); - sum_in1 = _mm256_add_ps(sum_in1, x1); - } + let mut cy = min_y; - sp = radius; - xp = radius; - if xp > wm { - xp = wm; - } + while cy + 4 <= max_y { + let mut sums0 = _mm256_setzero_ps(); + let mut sums1 = _mm256_setzero_ps(); - src_ptr0 = CN * xp as usize + cy * stride as usize; - src_ptr1 = CN * xp as usize + (cy + 1) * stride as usize; - src_ptr2 = CN * xp as usize + (cy + 2) * stride as usize; - src_ptr3 = CN * xp as usize + (cy + 3) * stride as usize; - let mut dst_ptr0 = cy * stride as usize; - let mut dst_ptr1 = (cy + 1) * stride as usize; - let mut dst_ptr2 = (cy + 2) * stride as usize; - let mut dst_ptr3 = (cy + 3) * stride as usize; - for _ in 0..width { - let store_ld0 = pixels.slice.as_ptr().add(dst_ptr0) as *mut f32; - let store_ld1 = pixels.slice.as_ptr().add(dst_ptr1) as *mut f32; - let store_ld2 = pixels.slice.as_ptr().add(dst_ptr2) as *mut f32; - let store_ld3 = pixels.slice.as_ptr().add(dst_ptr3) as *mut f32; - - let bx0 = _mm256_mul_ps(sums0, v_mul_value); - let bx1 = _mm256_mul_ps(sums1, v_mul_value); - - store_f32::(store_ld0, _mm256_castps256_ps128(bx0)); - store_f32::(store_ld1, _mm256_extractf128_ps::<1>(bx0)); - store_f32::(store_ld2, _mm256_castps256_ps128(bx1)); - store_f32::(store_ld3, _mm256_extractf128_ps::<1>(bx1)); - - dst_ptr0 += CN; - dst_ptr1 += CN; - dst_ptr2 += CN; - dst_ptr3 += CN; - - sums0 = _mm256_sub_ps(sums0, sum_out0); - sums1 = _mm256_sub_ps(sums1, sum_out1); - - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } - let stack = stacks.as_mut_ptr().add(stack_start as usize * 2); + let mut sum_in0 = _mm256_setzero_ps(); + let mut sum_in1 = _mm256_setzero_ps(); - let stack_val0 = _mm256_load_ps(stack.cast()); - let stack_val1 = _mm256_load_ps(stack.add(1).cast()); + let mut sum_out0 = _mm256_setzero_ps(); + let mut sum_out1 = _mm256_setzero_ps(); - sum_out0 = _mm256_sub_ps(sum_out0, stack_val0); - sum_out1 = _mm256_sub_ps(sum_out1, stack_val1); - - if xp < wm { - src_ptr0 += CN; - src_ptr1 += CN; - src_ptr2 += CN; - src_ptr3 += CN; - xp += 1; - } + let mut src_ptr0 = stride as usize * cy; + let mut src_ptr1 = stride as usize * (cy + 1); + let mut src_ptr2 = stride as usize * (cy + 2); + let mut src_ptr3 = stride as usize * (cy + 3); let src_pixel0 = load_f32::(pixels.slice.as_ptr().add(src_ptr0) as *const f32); let src_pixel1 = load_f32::(pixels.slice.as_ptr().add(src_ptr1) as *const f32); @@ -190,114 +91,217 @@ unsafe fn horiz_f32_pass_stack_impl( let x0 = _mm256_setr_m128(src_pixel0, src_pixel1); let x1 = _mm256_setr_m128(src_pixel2, src_pixel3); - _mm256_store_ps(stack.cast(), x0); - _mm256_store_ps(stack.add(1).cast(), x1); + for i in 0..=radius { + let stack_value = stacks.as_mut_ptr().add(i as usize * 2); + _mm256_store_ps(stack_value.cast(), x0); + _mm256_store_ps(stack_value.add(1).cast(), x1); + let w = _mm256_set1_ps((i + 1) as f32); + sums0 = _mm256_fmadd_ps(x0, w, sums0); + sums1 = _mm256_fmadd_ps(x1, w, sums1); + sum_out0 = _mm256_add_ps(sum_out0, x0); + sum_out1 = _mm256_add_ps(sum_out1, x1); + } - sum_in0 = _mm256_add_ps(sum_in0, x0); - sum_in1 = _mm256_add_ps(sum_in1, x1); + for i in 1..=radius { + if i <= wm { + src_ptr0 += CN; + src_ptr1 += CN; + src_ptr2 += CN; + src_ptr3 += CN; + } + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 2); + let src_pixel0 = load_f32::(pixels.slice.as_ptr().add(src_ptr0) as *const f32); + let src_pixel1 = load_f32::(pixels.slice.as_ptr().add(src_ptr1) as *const f32); + let src_pixel2 = load_f32::(pixels.slice.as_ptr().add(src_ptr2) as *const f32); + let src_pixel3 = load_f32::(pixels.slice.as_ptr().add(src_ptr3) as *const f32); + + let x0 = _mm256_setr_m128(src_pixel0, src_pixel1); + let x1 = _mm256_setr_m128(src_pixel2, src_pixel3); + + _mm256_store_ps(stack_ptr.cast(), x0); + _mm256_store_ps(stack_ptr.add(1).cast(), x1); + + let w = _mm256_set1_ps((radius + 1 - i) as f32); + sums0 = _mm256_fmadd_ps(x0, w, sums0); + sums1 = _mm256_fmadd_ps(x1, w, sums1); + + sum_in0 = _mm256_add_ps(sum_in0, x0); + sum_in1 = _mm256_add_ps(sum_in1, x1); + } - sums0 = _mm256_add_ps(sums0, sum_in0); - sums1 = _mm256_add_ps(sums1, sum_in1); + sp = radius; + xp = radius; + if xp > wm { + xp = wm; + } - sp += 1; - if sp >= div { - sp = 0; + src_ptr0 = CN * xp as usize + cy * stride as usize; + src_ptr1 = CN * xp as usize + (cy + 1) * stride as usize; + src_ptr2 = CN * xp as usize + (cy + 2) * stride as usize; + src_ptr3 = CN * xp as usize + (cy + 3) * stride as usize; + let mut dst_ptr0 = cy * stride as usize; + let mut dst_ptr1 = (cy + 1) * stride as usize; + let mut dst_ptr2 = (cy + 2) * stride as usize; + let mut dst_ptr3 = (cy + 3) * stride as usize; + for _ in 0..width { + let store_ld0 = pixels.slice.as_ptr().add(dst_ptr0) as *mut f32; + let store_ld1 = pixels.slice.as_ptr().add(dst_ptr1) as *mut f32; + let store_ld2 = pixels.slice.as_ptr().add(dst_ptr2) as *mut f32; + let store_ld3 = pixels.slice.as_ptr().add(dst_ptr3) as *mut f32; + + let bx0 = _mm256_mul_ps(sums0, v_mul_value); + let bx1 = _mm256_mul_ps(sums1, v_mul_value); + + store_f32::(store_ld0, _mm256_castps256_ps128(bx0)); + store_f32::(store_ld1, _mm256_extractf128_ps::<1>(bx0)); + store_f32::(store_ld2, _mm256_castps256_ps128(bx1)); + store_f32::(store_ld3, _mm256_extractf128_ps::<1>(bx1)); + + dst_ptr0 += CN; + dst_ptr1 += CN; + dst_ptr2 += CN; + dst_ptr3 += CN; + + sums0 = _mm256_sub_ps(sums0, sum_out0); + sums1 = _mm256_sub_ps(sums1, sum_out1); + + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } + let stack = stacks.as_mut_ptr().add(stack_start as usize * 2); + + let stack_val0 = _mm256_load_ps(stack.cast()); + let stack_val1 = _mm256_load_ps(stack.add(1).cast()); + + sum_out0 = _mm256_sub_ps(sum_out0, stack_val0); + sum_out1 = _mm256_sub_ps(sum_out1, stack_val1); + + if xp < wm { + src_ptr0 += CN; + src_ptr1 += CN; + src_ptr2 += CN; + src_ptr3 += CN; + xp += 1; + } + + let src_pixel0 = load_f32::(pixels.slice.as_ptr().add(src_ptr0) as *const f32); + let src_pixel1 = load_f32::(pixels.slice.as_ptr().add(src_ptr1) as *const f32); + let src_pixel2 = load_f32::(pixels.slice.as_ptr().add(src_ptr2) as *const f32); + let src_pixel3 = load_f32::(pixels.slice.as_ptr().add(src_ptr3) as *const f32); + + let x0 = _mm256_setr_m128(src_pixel0, src_pixel1); + let x1 = _mm256_setr_m128(src_pixel2, src_pixel3); + + _mm256_store_ps(stack.cast(), x0); + _mm256_store_ps(stack.add(1).cast(), x1); + + sum_in0 = _mm256_add_ps(sum_in0, x0); + sum_in1 = _mm256_add_ps(sum_in1, x1); + + sums0 = _mm256_add_ps(sums0, sum_in0); + sums1 = _mm256_add_ps(sums1, sum_in1); + + sp += 1; + if sp >= div { + sp = 0; + } + let stack = stacks.as_mut_ptr().add(sp as usize * 2); + let stack_val0 = _mm256_load_ps(stack.cast()); + let stack_val1 = _mm256_load_ps(stack.add(1).cast()); + + sum_out0 = _mm256_add_ps(sum_out0, stack_val0); + sum_out1 = _mm256_add_ps(sum_out1, stack_val1); + sum_in0 = _mm256_sub_ps(sum_in0, stack_val0); + sum_in1 = _mm256_sub_ps(sum_in1, stack_val1); } - let stack = stacks.as_mut_ptr().add(sp as usize * 2); - let stack_val0 = _mm256_load_ps(stack.cast()); - let stack_val1 = _mm256_load_ps(stack.add(1).cast()); - - sum_out0 = _mm256_add_ps(sum_out0, stack_val0); - sum_out1 = _mm256_add_ps(sum_out1, stack_val1); - sum_in0 = _mm256_sub_ps(sum_in0, stack_val0); - sum_in1 = _mm256_sub_ps(sum_in1, stack_val1); - } - cy += 4; - } + cy += 4; + } - let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); + let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - let mut src_ptr; - let mut dst_ptr; + let mut src_ptr; + let mut dst_ptr; - for y in cy..max_y { - let mut sums = _mm_setzero_ps(); - let mut sum_in = _mm_setzero_ps(); - let mut sum_out = _mm_setzero_ps(); + for y in cy..max_y { + let mut sums = _mm_setzero_ps(); + let mut sum_in = _mm_setzero_ps(); + let mut sum_out = _mm_setzero_ps(); - src_ptr = stride as usize * y; + src_ptr = stride as usize * y; - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; - let src_pixel = load_f32::(src_ld); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; + let src_pixel = load_f32::(src_ld); - for i in 0..=radius { - let stack_value = stacks.as_mut_ptr().add(i as usize); - _mm_store_ps(stack_value.cast(), src_pixel); - sums = _mm_fmadd_ps(src_pixel, _mm_set1_ps((i + 1) as f32), sums); - sum_out = _mm_add_ps(sum_out, src_pixel); - } + for i in 0..=radius { + let stack_value = stacks.as_mut_ptr().add(i as usize); + _mm_store_ps(stack_value.cast(), src_pixel); + sums = _mm_fmadd_ps(src_pixel, _mm_set1_ps((i + 1) as f32), sums); + sum_out = _mm_add_ps(sum_out, src_pixel); + } - for i in 1..=radius { - if i <= wm { - src_ptr += CN; + for i in 1..=radius { + if i <= wm { + src_ptr += CN; + } + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; + let src_pixel = load_f32::(src_ld); + _mm_store_ps(stack_ptr.cast(), src_pixel); + sums = _mm_fmadd_ps(src_pixel, _mm_set1_ps((radius + 1 - i) as f32), sums); + + sum_in = _mm_add_ps(sum_in, src_pixel); } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; - let src_pixel = load_f32::(src_ld); - _mm_store_ps(stack_ptr.cast(), src_pixel); - sums = _mm_fmadd_ps(src_pixel, _mm_set1_ps((radius + 1 - i) as f32), sums); - sum_in = _mm_add_ps(sum_in, src_pixel); - } + sp = radius; + xp = radius; + if xp > wm { + xp = wm; + } - sp = radius; - xp = radius; - if xp > wm { - xp = wm; - } + src_ptr = CN * xp as usize + y * stride as usize; + dst_ptr = y * stride as usize; + for _ in 0..width { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut f32; + let blurred = _mm_mul_ps(sums, v_mul_value); + store_f32::(store_ld, blurred); + dst_ptr += CN; - src_ptr = CN * xp as usize + y * stride as usize; - dst_ptr = y * stride as usize; - for _ in 0..width { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut f32; - let blurred = _mm_mul_ps(sums, v_mul_value); - store_f32::(store_ld, blurred); - dst_ptr += CN; + sums = _mm_sub_ps(sums, sum_out); - sums = _mm_sub_ps(sums, sum_out); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } + let stack = stacks.as_mut_ptr().add(stack_start as usize); - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } - let stack = stacks.as_mut_ptr().add(stack_start as usize); + let stack_val = _mm_load_ps(stack.cast()); - let stack_val = _mm_load_ps(stack.cast()); + sum_out = _mm_sub_ps(sum_out, stack_val); - sum_out = _mm_sub_ps(sum_out, stack_val); + if xp < wm { + src_ptr += CN; + xp += 1; + } - if xp < wm { - src_ptr += CN; - xp += 1; - } + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; + let src_pixel = load_f32::(src_ld); + _mm_store_ps(stack.cast(), src_pixel); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; - let src_pixel = load_f32::(src_ld); - _mm_store_ps(stack.cast(), src_pixel); + sum_in = _mm_add_ps(sum_in, src_pixel); + sums = _mm_add_ps(sums, sum_in); - sum_in = _mm_add_ps(sum_in, src_pixel); - sums = _mm_add_ps(sums, sum_in); + sp += 1; + if sp >= div { + sp = 0; + } + let stack = stacks.as_mut_ptr().add(sp as usize); + let stack_val = _mm_load_ps(stack.cast()); - sp += 1; - if sp >= div { - sp = 0; + sum_out = _mm_add_ps(sum_out, stack_val); + sum_in = _mm_sub_ps(sum_in, stack_val); } - let stack = stacks.as_mut_ptr().add(sp as usize); - let stack_val = _mm_load_ps(stack.cast()); - - sum_out = _mm_add_ps(sum_out, stack_val); - sum_in = _mm_sub_ps(sum_in, stack_val); } } } diff --git a/src/stackblur/avx/vertical.rs b/src/stackblur/avx/vertical.rs index 0b436a8e..774fb770 100644 --- a/src/stackblur/avx/vertical.rs +++ b/src/stackblur/avx/vertical.rs @@ -28,8 +28,10 @@ */ #![allow(clippy::incompatible_msrv)] use crate::sse::{load_u8_s32_fast, store_u8_s32}; +use crate::stackblur::avx::horizontal::AvxI32x8; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::arch::x86_64::*; pub(crate) struct VerticalAvxStackBlurPass {} @@ -41,7 +43,7 @@ impl Default for VerticalAvxStackBlurPass { } #[target_feature(enable = "avx2")] -unsafe fn stack_blur_avx_vertical_def( +fn stack_blur_avx_vertical_def( pixels: &UnsafeSlice, stride: u32, width: u32, @@ -62,7 +64,7 @@ unsafe fn stack_blur_avx_vertical_def( } #[inline(always)] -unsafe fn stack_blur_avx_vertical( +fn stack_blur_avx_vertical( pixels: &UnsafeSlice, stride: u32, width: u32, @@ -71,98 +73,46 @@ unsafe fn stack_blur_avx_vertical( thread: usize, total_threads: usize, ) { - let div = ((radius * 2) + 1) as usize; - let mut yp; - let mut sp; - let mut stack_start; - let mut stacks = vec![0i32; 4 * div * 8]; + unsafe { + let div = ((radius * 2) + 1) as usize; + let mut yp; + let mut sp; + let mut stack_start; + let mut scratch_buffer = ScratchBuffer::::new(div * 4); + let stacks = scratch_buffer.as_mut_slice(); - let hm = height - 1; - let div = (radius * 2) + 1; + let hm = height - 1; + let div = (radius * 2) + 1; - let mut src_ptr; - let mut dst_ptr; + let mut src_ptr; + let mut dst_ptr; - let min_x = (thread * width as usize / total_threads) * CN; - let max_x = ((thread + 1) * width as usize / total_threads) * CN; + let min_x = (thread * width as usize / total_threads) * CN; + let max_x = ((thread + 1) * width as usize / total_threads) * CN; - let mut cx = min_x; + let mut cx = min_x; - let v_mul_value = _mm256_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); + let v_mul_value = _mm256_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - while cx + 32 < max_x { - let mut sums0 = _mm256_setzero_si256(); - let mut sums1 = _mm256_setzero_si256(); - let mut sums2 = _mm256_setzero_si256(); - let mut sums3 = _mm256_setzero_si256(); + while cx + 32 <= max_x { + let mut sums0 = _mm256_setzero_si256(); + let mut sums1 = _mm256_setzero_si256(); + let mut sums2 = _mm256_setzero_si256(); + let mut sums3 = _mm256_setzero_si256(); - let mut sum_in0 = _mm256_setzero_si256(); - let mut sum_in1 = _mm256_setzero_si256(); - let mut sum_in2 = _mm256_setzero_si256(); - let mut sum_in3 = _mm256_setzero_si256(); + let mut sum_in0 = _mm256_setzero_si256(); + let mut sum_in1 = _mm256_setzero_si256(); + let mut sum_in2 = _mm256_setzero_si256(); + let mut sum_in3 = _mm256_setzero_si256(); - let mut sum_out0 = _mm256_setzero_si256(); - let mut sum_out1 = _mm256_setzero_si256(); - let mut sum_out2 = _mm256_setzero_si256(); - let mut sum_out3 = _mm256_setzero_si256(); + let mut sum_out0 = _mm256_setzero_si256(); + let mut sum_out1 = _mm256_setzero_si256(); + let mut sum_out2 = _mm256_setzero_si256(); + let mut sum_out3 = _mm256_setzero_si256(); - let mut src_ptr = cx; // x,0 + let mut src_ptr = cx; // x,0 - { - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel0 = _mm256_loadu_si256(src_ld as *const _); - let lo0 = _mm256_unpacklo_epi8(src_pixel0, _mm256_setzero_si256()); - let hi1 = _mm256_unpackhi_epi8(src_pixel0, _mm256_setzero_si256()); - - let i16_l0 = _mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()); - let i16_h0 = _mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()); - let i16_l1 = _mm256_unpacklo_epi16(hi1, _mm256_setzero_si256()); - let i16_h1 = _mm256_unpackhi_epi16(hi1, _mm256_setzero_si256()); - - for i in 0..=radius { - let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4 * 8); - - _mm256_storeu_si256(stack_ptr as *mut _, i16_l0); - _mm256_storeu_si256(stack_ptr.add(8) as *mut _, i16_h0); - _mm256_storeu_si256(stack_ptr.add(16) as *mut _, i16_l1); - _mm256_storeu_si256(stack_ptr.add(24) as *mut _, i16_h1); - - let w = _mm256_set1_epi32(i as i32 + 1); - - if VNNI { - #[cfg(feature = "nightly_avx512")] - { - sums0 = _mm256_dpwssd_avx_epi32(sums0, i16_l0, w); - sums1 = _mm256_dpwssd_avx_epi32(sums1, i16_h0, w); - sums2 = _mm256_dpwssd_avx_epi32(sums2, i16_l1, w); - sums3 = _mm256_dpwssd_avx_epi32(sums3, i16_h1, w); - } - } else { - let j0 = _mm256_madd_epi16(i16_l0, w); - let j1 = _mm256_madd_epi16(i16_h0, w); - let j2 = _mm256_madd_epi16(i16_l1, w); - let j3 = _mm256_madd_epi16(i16_h1, w); - - sums0 = _mm256_add_epi32(sums0, j0); - sums1 = _mm256_add_epi32(sums1, j1); - sums2 = _mm256_add_epi32(sums2, j2); - sums3 = _mm256_add_epi32(sums3, j3); - } - - sum_out0 = _mm256_add_epi32(sum_out0, i16_l0); - sum_out1 = _mm256_add_epi32(sum_out1, i16_h0); - sum_out2 = _mm256_add_epi32(sum_out2, i16_l1); - sum_out3 = _mm256_add_epi32(sum_out3, i16_h1); - } - } - - { - for i in 1..=radius { - if i <= hm { - src_ptr += stride as usize; - } - - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4 * 8); + { let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; let src_pixel0 = _mm256_loadu_si256(src_ld as *const _); let lo0 = _mm256_unpacklo_epi8(src_pixel0, _mm256_setzero_si256()); @@ -173,530 +123,582 @@ unsafe fn stack_blur_avx_vertical( let i16_l1 = _mm256_unpacklo_epi16(hi1, _mm256_setzero_si256()); let i16_h1 = _mm256_unpackhi_epi16(hi1, _mm256_setzero_si256()); - _mm256_storeu_si256(stack_ptr as *mut _, i16_l0); - _mm256_storeu_si256(stack_ptr.add(8) as *mut _, i16_h0); - _mm256_storeu_si256(stack_ptr.add(16) as *mut _, i16_l1); - _mm256_storeu_si256(stack_ptr.add(24) as *mut _, i16_h1); + for i in 0..=radius { + let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); + + _mm256_store_si256(stack_ptr as *mut _, i16_l0); + _mm256_store_si256(stack_ptr.add(1) as *mut _, i16_h0); + _mm256_store_si256(stack_ptr.add(2) as *mut _, i16_l1); + _mm256_store_si256(stack_ptr.add(3) as *mut _, i16_h1); + + let w = _mm256_set1_epi32(i as i32 + 1); + + if VNNI { + #[cfg(feature = "avx512")] + { + sums0 = _mm256_dpwssd_avx_epi32(sums0, i16_l0, w); + sums1 = _mm256_dpwssd_avx_epi32(sums1, i16_h0, w); + sums2 = _mm256_dpwssd_avx_epi32(sums2, i16_l1, w); + sums3 = _mm256_dpwssd_avx_epi32(sums3, i16_h1, w); + } + } else { + let j0 = _mm256_madd_epi16(i16_l0, w); + let j1 = _mm256_madd_epi16(i16_h0, w); + let j2 = _mm256_madd_epi16(i16_l1, w); + let j3 = _mm256_madd_epi16(i16_h1, w); + + sums0 = _mm256_add_epi32(sums0, j0); + sums1 = _mm256_add_epi32(sums1, j1); + sums2 = _mm256_add_epi32(sums2, j2); + sums3 = _mm256_add_epi32(sums3, j3); + } - let w = _mm256_set1_epi32(radius as i32 + 1 - i as i32); + sum_out0 = _mm256_add_epi32(sum_out0, i16_l0); + sum_out1 = _mm256_add_epi32(sum_out1, i16_h0); + sum_out2 = _mm256_add_epi32(sum_out2, i16_l1); + sum_out3 = _mm256_add_epi32(sum_out3, i16_h1); + } + } - if VNNI { - #[cfg(feature = "nightly_avx512")] - { - sums0 = _mm256_dpwssd_avx_epi32(sums0, i16_l0, w); - sums1 = _mm256_dpwssd_avx_epi32(sums1, i16_h0, w); - sums2 = _mm256_dpwssd_avx_epi32(sums2, i16_l1, w); - sums3 = _mm256_dpwssd_avx_epi32(sums3, i16_h1, w); + { + for i in 1..=radius { + if i <= hm { + src_ptr += stride as usize; } - } else { - let j0 = _mm256_madd_epi16(i16_l0, w); - let j1 = _mm256_madd_epi16(i16_h0, w); - let j2 = _mm256_madd_epi16(i16_l1, w); - let j3 = _mm256_madd_epi16(i16_h1, w); - - sums0 = _mm256_add_epi32(sums0, j0); - sums1 = _mm256_add_epi32(sums1, j1); - sums2 = _mm256_add_epi32(sums2, j2); - sums3 = _mm256_add_epi32(sums3, j3); - } - sum_in0 = _mm256_add_epi32(sum_in0, i16_l0); - sum_in1 = _mm256_add_epi32(sum_in1, i16_h0); - sum_in2 = _mm256_add_epi32(sum_in2, i16_l1); - sum_in3 = _mm256_add_epi32(sum_in3, i16_h1); + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel0 = _mm256_loadu_si256(src_ld as *const _); + let lo0 = _mm256_unpacklo_epi8(src_pixel0, _mm256_setzero_si256()); + let hi1 = _mm256_unpackhi_epi8(src_pixel0, _mm256_setzero_si256()); + + let i16_l0 = _mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()); + let i16_h0 = _mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()); + let i16_l1 = _mm256_unpacklo_epi16(hi1, _mm256_setzero_si256()); + let i16_h1 = _mm256_unpackhi_epi16(hi1, _mm256_setzero_si256()); + + _mm256_store_si256(stack_ptr as *mut _, i16_l0); + _mm256_store_si256(stack_ptr.add(1) as *mut _, i16_h0); + _mm256_store_si256(stack_ptr.add(2) as *mut _, i16_l1); + _mm256_store_si256(stack_ptr.add(3) as *mut _, i16_h1); + + let w = _mm256_set1_epi32(radius as i32 + 1 - i as i32); + + if VNNI { + #[cfg(feature = "avx512")] + { + sums0 = _mm256_dpwssd_avx_epi32(sums0, i16_l0, w); + sums1 = _mm256_dpwssd_avx_epi32(sums1, i16_h0, w); + sums2 = _mm256_dpwssd_avx_epi32(sums2, i16_l1, w); + sums3 = _mm256_dpwssd_avx_epi32(sums3, i16_h1, w); + } + } else { + let j0 = _mm256_madd_epi16(i16_l0, w); + let j1 = _mm256_madd_epi16(i16_h0, w); + let j2 = _mm256_madd_epi16(i16_l1, w); + let j3 = _mm256_madd_epi16(i16_h1, w); + + sums0 = _mm256_add_epi32(sums0, j0); + sums1 = _mm256_add_epi32(sums1, j1); + sums2 = _mm256_add_epi32(sums2, j2); + sums3 = _mm256_add_epi32(sums3, j3); + } + + sum_in0 = _mm256_add_epi32(sum_in0, i16_l0); + sum_in1 = _mm256_add_epi32(sum_in1, i16_h0); + sum_in2 = _mm256_add_epi32(sum_in2, i16_l1); + sum_in3 = _mm256_add_epi32(sum_in3, i16_h1); + } } - } - sp = radius; - yp = radius; - if yp > hm { - yp = hm; - } - src_ptr = cx + yp as usize * stride as usize; - let mut dst_ptr = cx; - for _ in 0..height { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + sp = radius; + yp = radius; + if yp > hm { + yp = hm; + } + src_ptr = cx + yp as usize * stride as usize; + let mut dst_ptr = cx; + for _ in 0..height { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - let casted_sum0 = _mm256_cvtepi32_ps(sums0); - let casted_sum1 = _mm256_cvtepi32_ps(sums1); - let casted_sum2 = _mm256_cvtepi32_ps(sums2); - let casted_sum3 = _mm256_cvtepi32_ps(sums3); + let casted_sum0 = _mm256_cvtepi32_ps(sums0); + let casted_sum1 = _mm256_cvtepi32_ps(sums1); + let casted_sum2 = _mm256_cvtepi32_ps(sums2); + let casted_sum3 = _mm256_cvtepi32_ps(sums3); - let a0 = _mm256_mul_ps(casted_sum0, v_mul_value); - let a1 = _mm256_mul_ps(casted_sum1, v_mul_value); - let a2 = _mm256_mul_ps(casted_sum2, v_mul_value); - let a3 = _mm256_mul_ps(casted_sum3, v_mul_value); + let a0 = _mm256_mul_ps(casted_sum0, v_mul_value); + let a1 = _mm256_mul_ps(casted_sum1, v_mul_value); + let a2 = _mm256_mul_ps(casted_sum2, v_mul_value); + let a3 = _mm256_mul_ps(casted_sum3, v_mul_value); - let scaled_val0 = _mm256_cvtps_epi32(a0); - let scaled_val1 = _mm256_cvtps_epi32(a1); - let scaled_val2 = _mm256_cvtps_epi32(a2); - let scaled_val3 = _mm256_cvtps_epi32(a3); + let scaled_val0 = _mm256_cvtps_epi32(a0); + let scaled_val1 = _mm256_cvtps_epi32(a1); + let scaled_val2 = _mm256_cvtps_epi32(a2); + let scaled_val3 = _mm256_cvtps_epi32(a3); - let jv0 = _mm256_packus_epi32(scaled_val0, scaled_val1); - let jv1 = _mm256_packus_epi32(scaled_val2, scaled_val3); + let jv0 = _mm256_packus_epi32(scaled_val0, scaled_val1); + let jv1 = _mm256_packus_epi32(scaled_val2, scaled_val3); - _mm256_storeu_si256(store_ld as *mut _, _mm256_packus_epi16(jv0, jv1)); + _mm256_storeu_si256(store_ld as *mut _, _mm256_packus_epi16(jv0, jv1)); - dst_ptr += stride as usize; + dst_ptr += stride as usize; - sums0 = _mm256_sub_epi32(sums0, sum_out0); - sums1 = _mm256_sub_epi32(sums1, sum_out1); - sums2 = _mm256_sub_epi32(sums2, sum_out2); - sums3 = _mm256_sub_epi32(sums3, sum_out3); + sums0 = _mm256_sub_epi32(sums0, sum_out0); + sums1 = _mm256_sub_epi32(sums1, sum_out1); + sums2 = _mm256_sub_epi32(sums2, sum_out2); + sums3 = _mm256_sub_epi32(sums3, sum_out3); - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } - let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4 * 8); + let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); - let stack_val0 = _mm256_loadu_si256(stack_ptr as *const _); - let stack_val1 = _mm256_loadu_si256(stack_ptr.add(8) as *const _); - let stack_val2 = _mm256_loadu_si256(stack_ptr.add(16) as *const _); - let stack_val3 = _mm256_loadu_si256(stack_ptr.add(24) as *const _); + let stack_val0 = _mm256_load_si256(stack_ptr as *const _); + let stack_val1 = _mm256_load_si256(stack_ptr.add(1) as *const _); + let stack_val2 = _mm256_load_si256(stack_ptr.add(2) as *const _); + let stack_val3 = _mm256_load_si256(stack_ptr.add(3) as *const _); - sum_out0 = _mm256_sub_epi32(sum_out0, stack_val0); - sum_out1 = _mm256_sub_epi32(sum_out1, stack_val1); - sum_out2 = _mm256_sub_epi32(sum_out2, stack_val2); - sum_out3 = _mm256_sub_epi32(sum_out3, stack_val3); + sum_out0 = _mm256_sub_epi32(sum_out0, stack_val0); + sum_out1 = _mm256_sub_epi32(sum_out1, stack_val1); + sum_out2 = _mm256_sub_epi32(sum_out2, stack_val2); + sum_out3 = _mm256_sub_epi32(sum_out3, stack_val3); - if yp < hm { - src_ptr += stride as usize; // stride - yp += 1; - } + if yp < hm { + src_ptr += stride as usize; // stride + yp += 1; + } - let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_pixel0 = _mm256_loadu_si256(src_ld as *const _); - let lo0 = _mm256_unpacklo_epi8(src_pixel0, _mm256_setzero_si256()); - let hi1 = _mm256_unpackhi_epi8(src_pixel0, _mm256_setzero_si256()); + let src_pixel0 = _mm256_loadu_si256(src_ld as *const _); + let lo0 = _mm256_unpacklo_epi8(src_pixel0, _mm256_setzero_si256()); + let hi1 = _mm256_unpackhi_epi8(src_pixel0, _mm256_setzero_si256()); - let i16_l0 = _mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()); - let i16_h0 = _mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()); - let i16_l1 = _mm256_unpacklo_epi16(hi1, _mm256_setzero_si256()); - let i16_h1 = _mm256_unpackhi_epi16(hi1, _mm256_setzero_si256()); + let i16_l0 = _mm256_unpacklo_epi16(lo0, _mm256_setzero_si256()); + let i16_h0 = _mm256_unpackhi_epi16(lo0, _mm256_setzero_si256()); + let i16_l1 = _mm256_unpacklo_epi16(hi1, _mm256_setzero_si256()); + let i16_h1 = _mm256_unpackhi_epi16(hi1, _mm256_setzero_si256()); - _mm256_storeu_si256(stack_ptr as *mut _, i16_l0); - _mm256_storeu_si256(stack_ptr.add(8) as *mut _, i16_h0); - _mm256_storeu_si256(stack_ptr.add(16) as *mut _, i16_l1); - _mm256_storeu_si256(stack_ptr.add(24) as *mut _, i16_h1); + _mm256_store_si256(stack_ptr as *mut _, i16_l0); + _mm256_store_si256(stack_ptr.add(1) as *mut _, i16_h0); + _mm256_store_si256(stack_ptr.add(2) as *mut _, i16_l1); + _mm256_store_si256(stack_ptr.add(3) as *mut _, i16_h1); - sum_in0 = _mm256_add_epi32(sum_in0, i16_l0); - sum_in1 = _mm256_add_epi32(sum_in1, i16_h0); - sum_in2 = _mm256_add_epi32(sum_in2, i16_l1); - sum_in3 = _mm256_add_epi32(sum_in3, i16_h1); + sum_in0 = _mm256_add_epi32(sum_in0, i16_l0); + sum_in1 = _mm256_add_epi32(sum_in1, i16_h0); + sum_in2 = _mm256_add_epi32(sum_in2, i16_l1); + sum_in3 = _mm256_add_epi32(sum_in3, i16_h1); - sums0 = _mm256_add_epi32(sums0, sum_in0); - sums1 = _mm256_add_epi32(sums1, sum_in1); - sums2 = _mm256_add_epi32(sums2, sum_in2); - sums3 = _mm256_add_epi32(sums3, sum_in3); + sums0 = _mm256_add_epi32(sums0, sum_in0); + sums1 = _mm256_add_epi32(sums1, sum_in1); + sums2 = _mm256_add_epi32(sums2, sum_in2); + sums3 = _mm256_add_epi32(sums3, sum_in3); - sp += 1; + sp += 1; - if sp >= div { - sp = 0; + if sp >= div { + sp = 0; + } + let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); + + let stack_val0 = _mm256_load_si256(stack_ptr as *const _); + let stack_val1 = _mm256_load_si256(stack_ptr.add(1) as *const _); + let stack_val2 = _mm256_load_si256(stack_ptr.add(2) as *const _); + let stack_val3 = _mm256_load_si256(stack_ptr.add(3) as *const _); + + sum_out0 = _mm256_add_epi32(sum_out0, stack_val0); + sum_out1 = _mm256_add_epi32(sum_out1, stack_val1); + sum_out2 = _mm256_add_epi32(sum_out2, stack_val2); + sum_out3 = _mm256_add_epi32(sum_out3, stack_val3); + + sum_in0 = _mm256_sub_epi32(sum_in0, stack_val0); + sum_in1 = _mm256_sub_epi32(sum_in1, stack_val1); + sum_in2 = _mm256_sub_epi32(sum_in2, stack_val2); + sum_in3 = _mm256_sub_epi32(sum_in3, stack_val3); } - let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4 * 8); - - let stack_val0 = _mm256_loadu_si256(stack_ptr as *const _); - let stack_val1 = _mm256_loadu_si256(stack_ptr.add(8) as *const _); - let stack_val2 = _mm256_loadu_si256(stack_ptr.add(16) as *const _); - let stack_val3 = _mm256_loadu_si256(stack_ptr.add(24) as *const _); - - sum_out0 = _mm256_add_epi32(sum_out0, stack_val0); - sum_out1 = _mm256_add_epi32(sum_out1, stack_val1); - sum_out2 = _mm256_add_epi32(sum_out2, stack_val2); - sum_out3 = _mm256_add_epi32(sum_out3, stack_val3); - - sum_in0 = _mm256_sub_epi32(sum_in0, stack_val0); - sum_in1 = _mm256_sub_epi32(sum_in1, stack_val1); - sum_in2 = _mm256_sub_epi32(sum_in2, stack_val2); - sum_in3 = _mm256_sub_epi32(sum_in3, stack_val3); + + cx += 32; } - cx += 32; - } + let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); + while cx + 8 <= max_x { + let mut sums0 = _mm_setzero_si128(); + let mut sums1 = _mm_setzero_si128(); - while cx + 8 < max_x { - let mut sums0 = _mm_setzero_si128(); - let mut sums1 = _mm_setzero_si128(); + let mut sum_in0 = _mm_setzero_si128(); + let mut sum_in1 = _mm_setzero_si128(); - let mut sum_in0 = _mm_setzero_si128(); - let mut sum_in1 = _mm_setzero_si128(); + let mut sum_out0 = _mm_setzero_si128(); + let mut sum_out1 = _mm_setzero_si128(); - let mut sum_out0 = _mm_setzero_si128(); - let mut sum_out1 = _mm_setzero_si128(); + let mut src_ptr = cx; // x,0 - let mut src_ptr = cx; // x,0 + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + { + let src_pixel0 = _mm_loadu_si64(src_ld as *const _); + let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); - { - let src_pixel0 = _mm_loadu_si64(src_ld as *const _); - let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); + let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); + for i in 0..=radius { + let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); - for i in 0..=radius { - let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4 * 4); + _mm256_store_si256(stack_ptr as *mut _, _mm256_setr_m128i(i16_l0, i16_h0)); - _mm_storeu_si128(stack_ptr as *mut _, i16_l0); - _mm_storeu_si128(stack_ptr.add(4) as *mut _, i16_h0); + let w = _mm_set1_epi32(i as i32 + 1); - let w = _mm_set1_epi32(i as i32 + 1); + if VNNI { + #[cfg(feature = "avx512")] + { + sums0 = _mm_dpwssd_avx_epi32(sums0, i16_l0, w); + sums1 = _mm_dpwssd_avx_epi32(sums1, i16_h0, w); + } + } else { + let j0 = _mm_madd_epi16(i16_l0, w); + let j1 = _mm_madd_epi16(i16_h0, w); - if VNNI { - #[cfg(feature = "nightly_avx512")] - { - sums0 = _mm_dpwssd_avx_epi32(sums0, i16_l0, w); - sums1 = _mm_dpwssd_avx_epi32(sums1, i16_h0, w); + sums0 = _mm_add_epi32(sums0, j0); + sums1 = _mm_add_epi32(sums1, j1); } - } else { - let j0 = _mm_madd_epi16(i16_l0, w); - let j1 = _mm_madd_epi16(i16_h0, w); - sums0 = _mm_add_epi32(sums0, j0); - sums1 = _mm_add_epi32(sums1, j1); + sum_out0 = _mm_add_epi32(sum_out0, i16_l0); + sum_out1 = _mm_add_epi32(sum_out1, i16_h0); } - - sum_out0 = _mm_add_epi32(sum_out0, i16_l0); - sum_out1 = _mm_add_epi32(sum_out1, i16_h0); } - } - { - for i in 1..=radius { - if i <= hm { - src_ptr += stride as usize; - } + { + for i in 1..=radius { + if i <= hm { + src_ptr += stride as usize; + } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4 * 4); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel0 = _mm_loadu_si64(src_ld as *const u8); - let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel0 = _mm_loadu_si64(src_ld as *const u8); + let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); - let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); + let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - _mm_storeu_si128(stack_ptr as *mut _, i16_l0); - _mm_storeu_si128(stack_ptr.add(4) as *mut _, i16_h0); + _mm256_store_si256(stack_ptr as *mut _, _mm256_setr_m128i(i16_l0, i16_h0)); - let w = _mm_set1_epi32(radius as i32 + 1 - i as i32); + let w = _mm_set1_epi32(radius as i32 + 1 - i as i32); - if VNNI { - #[cfg(feature = "nightly_avx512")] - { - sums0 = _mm_dpwssd_avx_epi32(sums0, i16_l0, w); - sums1 = _mm_dpwssd_avx_epi32(sums1, i16_h0, w); + if VNNI { + #[cfg(feature = "avx512")] + { + sums0 = _mm_dpwssd_avx_epi32(sums0, i16_l0, w); + sums1 = _mm_dpwssd_avx_epi32(sums1, i16_h0, w); + } + } else { + let j0 = _mm_madd_epi16(i16_l0, w); + let j1 = _mm_madd_epi16(i16_h0, w); + + sums0 = _mm_add_epi32(sums0, j0); + sums1 = _mm_add_epi32(sums1, j1); } - } else { - let j0 = _mm_madd_epi16(i16_l0, w); - let j1 = _mm_madd_epi16(i16_h0, w); - sums0 = _mm_add_epi32(sums0, j0); - sums1 = _mm_add_epi32(sums1, j1); + sum_in0 = _mm_add_epi32(sum_in0, i16_l0); + sum_in1 = _mm_add_epi32(sum_in1, i16_h0); } + } - sum_in0 = _mm_add_epi32(sum_in0, i16_l0); - sum_in1 = _mm_add_epi32(sum_in1, i16_h0); + sp = radius; + yp = radius; + if yp > hm { + yp = hm; } - } + src_ptr = cx + yp as usize * stride as usize; + let mut dst_ptr = cx; + for _ in 0..height { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - sp = radius; - yp = radius; - if yp > hm { - yp = hm; - } - src_ptr = cx + yp as usize * stride as usize; - let mut dst_ptr = cx; - for _ in 0..height { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + let casted_sum0 = _mm_cvtepi32_ps(sums0); + let casted_sum1 = _mm_cvtepi32_ps(sums1); - let casted_sum0 = _mm_cvtepi32_ps(sums0); - let casted_sum1 = _mm_cvtepi32_ps(sums1); + let a0 = _mm_mul_ps(casted_sum0, v_mul_value); + let a1 = _mm_mul_ps(casted_sum1, v_mul_value); - let a0 = _mm_mul_ps(casted_sum0, v_mul_value); - let a1 = _mm_mul_ps(casted_sum1, v_mul_value); + let scaled_val0 = _mm_cvtps_epi32(a0); + let scaled_val1 = _mm_cvtps_epi32(a1); - let scaled_val0 = _mm_cvtps_epi32(a0); - let scaled_val1 = _mm_cvtps_epi32(a1); + let jv0 = _mm_packus_epi32(scaled_val0, scaled_val1); - let jv0 = _mm_packus_epi32(scaled_val0, scaled_val1); + _mm_storeu_si64(store_ld, _mm_packus_epi16(jv0, _mm_setzero_si128())); - _mm_storeu_si64(store_ld, _mm_packus_epi16(jv0, _mm_setzero_si128())); + dst_ptr += stride as usize; - dst_ptr += stride as usize; + sums0 = _mm_sub_epi32(sums0, sum_out0); + sums1 = _mm_sub_epi32(sums1, sum_out1); - sums0 = _mm_sub_epi32(sums0, sum_out0); - sums1 = _mm_sub_epi32(sums1, sum_out1); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } + let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); - let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4 * 4); + let stack_vals = _mm256_load_si256(stack_ptr as *const _); - let stack_val0 = _mm_loadu_si128(stack_ptr as *const _); - let stack_val1 = _mm_loadu_si128(stack_ptr.add(4) as *const _); + sum_out0 = _mm_sub_epi32(sum_out0, _mm256_castsi256_si128(stack_vals)); + sum_out1 = _mm_sub_epi32(sum_out1, _mm256_extractf128_si256::<1>(stack_vals)); - sum_out0 = _mm_sub_epi32(sum_out0, stack_val0); - sum_out1 = _mm_sub_epi32(sum_out1, stack_val1); + if yp < hm { + src_ptr += stride as usize; // stride + yp += 1; + } - if yp < hm { - src_ptr += stride as usize; // stride - yp += 1; - } + let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_pixel0 = _mm_loadu_si64(src_ld as *const u8); + let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); - let src_pixel0 = _mm_loadu_si64(src_ld as *const u8); - let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); + let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); + _mm256_store_si256(stack_ptr as *mut _, _mm256_setr_m128i(i16_l0, i16_h0)); - _mm_storeu_si128(stack_ptr as *mut _, i16_l0); - _mm_storeu_si128(stack_ptr.add(4) as *mut _, i16_h0); + sum_in0 = _mm_add_epi32(sum_in0, i16_l0); + sum_in1 = _mm_add_epi32(sum_in1, i16_h0); - sum_in0 = _mm_add_epi32(sum_in0, i16_l0); - sum_in1 = _mm_add_epi32(sum_in1, i16_h0); + sums0 = _mm_add_epi32(sums0, sum_in0); + sums1 = _mm_add_epi32(sums1, sum_in1); - sums0 = _mm_add_epi32(sums0, sum_in0); - sums1 = _mm_add_epi32(sums1, sum_in1); + sp += 1; - sp += 1; + if sp >= div { + sp = 0; + } + let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); - if sp >= div { - sp = 0; - } - let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4 * 4); + let stack_vals = _mm256_load_si256(stack_ptr as *const _); + let stack_val0 = _mm256_castsi256_si128(stack_vals); + let stack_val1 = _mm256_extracti128_si256::<1>(stack_vals); - let stack_val0 = _mm_loadu_si128(stack_ptr as *const _); - let stack_val1 = _mm_loadu_si128(stack_ptr.add(4) as *const _); + sum_out0 = _mm_add_epi32(sum_out0, stack_val0); + sum_out1 = _mm_add_epi32(sum_out1, stack_val1); - sum_out0 = _mm_add_epi32(sum_out0, stack_val0); - sum_out1 = _mm_add_epi32(sum_out1, stack_val1); + sum_in0 = _mm_sub_epi32(sum_in0, stack_val0); + sum_in1 = _mm_sub_epi32(sum_in1, stack_val1); + } - sum_in0 = _mm_sub_epi32(sum_in0, stack_val0); - sum_in1 = _mm_sub_epi32(sum_in1, stack_val1); + cx += 8; } - cx += 8; - } - - while cx + CN < max_x { - let mut sums = _mm_setzero_si128(); - let mut sum_in = _mm_setzero_si128(); - let mut sum_out = _mm_setzero_si128(); + while cx + CN < max_x { + let mut sums = _mm_setzero_si128(); + let mut sum_in = _mm_setzero_si128(); + let mut sum_out = _mm_setzero_si128(); - src_ptr = cx; // x,0 + src_ptr = cx; // x,0 - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - for i in 0..=radius { - let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + for i in 0..=radius { + let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); + _mm_store_si128(stack_ptr as *mut __m128i, src_pixel); - let w = _mm_set1_epi32(i as i32 + 1); + let w = _mm_set1_epi32(i as i32 + 1); - if VNNI { - #[cfg(feature = "nightly_avx512")] - { - sums = _mm_dpwssd_avx_epi32(sums, src_pixel, w); + if VNNI { + #[cfg(feature = "avx512")] + { + sums = _mm_dpwssd_avx_epi32(sums, src_pixel, w); + } + } else { + sums = _mm_add_epi32(sums, _mm_madd_epi16(src_pixel, w)); } - } else { - sums = _mm_add_epi32(sums, _mm_madd_epi16(src_pixel, w)); + sum_out = _mm_add_epi32(sum_out, src_pixel); } - sum_out = _mm_add_epi32(sum_out, src_pixel); - } - for i in 1..=radius { - if i <= hm { - src_ptr += stride as usize; - } + for i in 1..=radius { + if i <= hm { + src_ptr += stride as usize; + } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_store_si128(stack_ptr as *mut __m128i, src_pixel); - let w = _mm_set1_epi32(radius as i32 + 1 - i as i32); + let w = _mm_set1_epi32(radius as i32 + 1 - i as i32); - if VNNI { - #[cfg(feature = "nightly_avx512")] - { - sums = _mm_dpwssd_avx_epi32(sums, src_pixel, w); + if VNNI { + #[cfg(feature = "avx512")] + { + sums = _mm_dpwssd_avx_epi32(sums, src_pixel, w); + } + } else { + sums = _mm_add_epi32(sums, _mm_madd_epi16(src_pixel, w)); } - } else { - sums = _mm_add_epi32(sums, _mm_madd_epi16(src_pixel, w)); + + sum_in = _mm_add_epi32(sum_in, src_pixel); } - sum_in = _mm_add_epi32(sum_in, src_pixel); - } + sp = radius; + yp = radius; + if yp > hm { + yp = hm; + } + src_ptr = cx + yp as usize * stride as usize; + dst_ptr = cx; + for _ in 0..height { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + let result = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(sums), v_mul_value)); + store_u8_s32::(store_ld, result); - sp = radius; - yp = radius; - if yp > hm { - yp = hm; - } - src_ptr = cx + yp as usize * stride as usize; - dst_ptr = cx; - for _ in 0..height { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - let result = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(sums), v_mul_value)); - store_u8_s32::(store_ld, result); + dst_ptr += stride as usize; - dst_ptr += stride as usize; + sums = _mm_sub_epi32(sums, sum_out); - sums = _mm_sub_epi32(sums, sum_out); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } + let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); + let stack_val = _mm_load_si128(stack_ptr as *const __m128i); + sum_out = _mm_sub_epi32(sum_out, stack_val); - let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); - let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); - sum_out = _mm_sub_epi32(sum_out, stack_val); + if yp < hm { + src_ptr += stride as usize; // stride + yp += 1; + } - if yp < hm { - src_ptr += stride as usize; // stride - yp += 1; - } + let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_store_si128(stack_ptr as *mut __m128i, src_pixel); - let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + sum_in = _mm_add_epi32(sum_in, src_pixel); + sums = _mm_add_epi32(sums, sum_in); - sum_in = _mm_add_epi32(sum_in, src_pixel); - sums = _mm_add_epi32(sums, sum_in); + sp += 1; - sp += 1; + if sp >= div { + sp = 0; + } + let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); + let stack_val = _mm_load_si128(stack_ptr as *const __m128i); - if sp >= div { - sp = 0; + sum_out = _mm_add_epi32(sum_out, stack_val); + sum_in = _mm_sub_epi32(sum_in, stack_val); } - let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); - let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); - sum_out = _mm_add_epi32(sum_out, stack_val); - sum_in = _mm_sub_epi32(sum_in, stack_val); + cx += CN; } - cx += CN; - } - - const TAIL: usize = 1; + const TAIL: usize = 1; - while cx < max_x { - let mut sums = _mm_setzero_si128(); - let mut sum_in = _mm_setzero_si128(); - let mut sum_out = _mm_setzero_si128(); + while cx < max_x { + let mut sums = _mm_setzero_si128(); + let mut sum_in = _mm_setzero_si128(); + let mut sum_out = _mm_setzero_si128(); - src_ptr = cx; // x,0 + src_ptr = cx; // x,0 - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - for i in 0..=radius { - let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + for i in 0..=radius { + let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); + _mm_store_si128(stack_ptr as *mut __m128i, src_pixel); - let w = _mm_set1_epi32(i as i32 + 1); + let w = _mm_set1_epi32(i as i32 + 1); - if VNNI { - #[cfg(feature = "nightly_avx512")] - { - sums = _mm_dpwssd_avx_epi32(sums, src_pixel, w); + if VNNI { + #[cfg(feature = "avx512")] + { + sums = _mm_dpwssd_avx_epi32(sums, src_pixel, w); + } + } else { + sums = _mm_add_epi32(sums, _mm_madd_epi16(src_pixel, w)); } - } else { - sums = _mm_add_epi32(sums, _mm_madd_epi16(src_pixel, w)); + sum_out = _mm_add_epi32(sum_out, src_pixel); } - sum_out = _mm_add_epi32(sum_out, src_pixel); - } - for i in 1..=radius { - if i <= hm { - src_ptr += stride as usize; - } + for i in 1..=radius { + if i <= hm { + src_ptr += stride as usize; + } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_store_si128(stack_ptr as *mut __m128i, src_pixel); - let w = _mm_set1_epi32(radius as i32 + 1 - i as i32); + let w = _mm_set1_epi32(radius as i32 + 1 - i as i32); - if VNNI { - #[cfg(feature = "nightly_avx512")] - { - sums = _mm_dpwssd_avx_epi32(sums, src_pixel, w); + if VNNI { + #[cfg(feature = "avx512")] + { + sums = _mm_dpwssd_avx_epi32(sums, src_pixel, w); + } + } else { + sums = _mm_add_epi32(sums, _mm_madd_epi16(src_pixel, w)); } - } else { - sums = _mm_add_epi32(sums, _mm_madd_epi16(src_pixel, w)); + + sum_in = _mm_add_epi32(sum_in, src_pixel); } - sum_in = _mm_add_epi32(sum_in, src_pixel); - } + sp = radius; + yp = radius; + if yp > hm { + yp = hm; + } + src_ptr = cx + yp as usize * stride as usize; + dst_ptr = cx; + for _ in 0..height { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + let result = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(sums), v_mul_value)); + store_u8_s32::(store_ld, result); - sp = radius; - yp = radius; - if yp > hm { - yp = hm; - } - src_ptr = cx + yp as usize * stride as usize; - dst_ptr = cx; - for _ in 0..height { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - let result = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(sums), v_mul_value)); - store_u8_s32::(store_ld, result); + dst_ptr += stride as usize; - dst_ptr += stride as usize; + sums = _mm_sub_epi32(sums, sum_out); - sums = _mm_sub_epi32(sums, sum_out); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } + let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); + let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); + sum_out = _mm_sub_epi32(sum_out, stack_val); - let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); - let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); - sum_out = _mm_sub_epi32(sum_out, stack_val); + if yp < hm { + src_ptr += stride as usize; // stride + yp += 1; + } - if yp < hm { - src_ptr += stride as usize; // stride - yp += 1; - } + let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_store_si128(stack_ptr as *mut __m128i, src_pixel); - let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + sum_in = _mm_add_epi32(sum_in, src_pixel); + sums = _mm_add_epi32(sums, sum_in); - sum_in = _mm_add_epi32(sum_in, src_pixel); - sums = _mm_add_epi32(sums, sum_in); + sp += 1; - sp += 1; + if sp >= div { + sp = 0; + } + let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); + let stack_val = _mm_load_si128(stack_ptr as *const __m128i); - if sp >= div { - sp = 0; + sum_out = _mm_add_epi32(sum_out, stack_val); + sum_in = _mm_sub_epi32(sum_in, stack_val); } - let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); - let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); - sum_out = _mm_add_epi32(sum_out, stack_val); - sum_in = _mm_sub_epi32(sum_in, stack_val); + cx += TAIL; } - - cx += TAIL; } } diff --git a/src/stackblur/avx/vertical_f32.rs b/src/stackblur/avx/vertical_f32.rs index f55ad5ad..3e76e5cc 100644 --- a/src/stackblur/avx/vertical_f32.rs +++ b/src/stackblur/avx/vertical_f32.rs @@ -28,6 +28,7 @@ */ use crate::stackblur::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::arch::x86_64::*; #[repr(C, align(32))] @@ -56,7 +57,8 @@ unsafe fn stack_blur_pass_vert_avx( let mut yp; let mut sp; let mut stack_start; - let mut stacks = vec![Avx2F32x8::default(); div * 2]; + let mut scratch_buffer = ScratchBuffer::::new(div * 2); + let stacks = scratch_buffer.as_mut_slice(); let hm = height - 1; let div = (radius * 2) + 1; diff --git a/src/stackblur/horizontal.rs b/src/stackblur/horizontal.rs index e5a8ba5a..0fea036c 100644 --- a/src/stackblur/horizontal.rs +++ b/src/stackblur/horizontal.rs @@ -30,6 +30,7 @@ use crate::primitives::PrimitiveCast; use crate::stackblur::sliding_window::SlidingWindow; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::marker::PhantomData; use std::ops::{AddAssign, Mul, Sub, SubAssign}; @@ -70,8 +71,7 @@ where usize: PrimitiveCast, f32: PrimitiveCast, { - #[inline] - unsafe fn pass_impl( + fn pass_impl( &self, pixels: &UnsafeSlice, stride: u32, @@ -85,7 +85,8 @@ where let mut xp; let mut sp; let mut stack_start; - let mut stacks0 = vec![SlidingWindow::::new(); div]; + let mut stack_buffer = ScratchBuffer::, 1024>::new(div); + let stacks0 = stack_buffer.as_mut_slice(); let rad_p_1 = radius as f32 + 1.; let scale_filter_value = (1. / (rad_p_1 * rad_p_1)).cast_(); @@ -205,8 +206,6 @@ where thread: usize, total_threads: usize, ) { - unsafe { - self.pass_impl(pixels, stride, width, height, radius, thread, total_threads); - } + self.pass_impl(pixels, stride, width, height, radius, thread, total_threads); } } diff --git a/src/stackblur/neon/horizontal.rs b/src/stackblur/neon/horizontal.rs index bc68ef2a..0ecf8c8f 100644 --- a/src/stackblur/neon/horizontal.rs +++ b/src/stackblur/neon/horizontal.rs @@ -29,6 +29,7 @@ use crate::neon::{load_u8_s32_fast, store_u8_s32, store_u8_s32_x4}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::arch::aarch64::*; pub(crate) struct HorizontalNeonStackBlurPass {} @@ -58,7 +59,8 @@ impl StackBlurWorkingPass for HorizontalNeonStackBlurPa let mut _xp; let mut sp; let mut stack_start; - let mut stacks0 = vec![0i32; 4 * div * 4]; + let mut scratch_buffer = ScratchBuffer::::new(4 * div * 4); + let stacks0 = scratch_buffer.as_mut_slice(); let mul_value = vdupq_n_f32(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); diff --git a/src/stackblur/neon/horizontal_f16.rs b/src/stackblur/neon/horizontal_f16.rs index 424717b5..c87b164e 100644 --- a/src/stackblur/neon/horizontal_f16.rs +++ b/src/stackblur/neon/horizontal_f16.rs @@ -27,50 +27,27 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::neon::{load_f32_f16, p_vfmaq_f32, store_f32_f16}; -use crate::primitives::PrimitiveCast; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; #[cfg(feature = "nightly_f16")] use core::f16; use std::arch::aarch64::*; -use std::marker::PhantomData; -use std::ops::{AddAssign, Mul, Sub, SubAssign}; -pub(crate) struct HorizontalNeonStackBlurPassFloat16 { - _phantom_t: PhantomData, - _phantom_j: PhantomData, -} +pub(crate) struct HorizontalNeonStackBlurPassFloat16 {} -impl Default for HorizontalNeonStackBlurPassFloat16 { +impl Default for HorizontalNeonStackBlurPassFloat16 { fn default() -> Self { - HorizontalNeonStackBlurPassFloat16:: { - _phantom_t: Default::default(), - _phantom_j: Default::default(), - } + HorizontalNeonStackBlurPassFloat16:: {} } } -impl HorizontalNeonStackBlurPassFloat16 -where - J: Copy - + 'static - + AddAssign - + Mul - + Sub - + PrimitiveCast - + SubAssign - + PrimitiveCast - + Default, - T: Copy + PrimitiveCast + Default, - i32: PrimitiveCast, - u32: PrimitiveCast, - f32: PrimitiveCast, - usize: PrimitiveCast, -{ +impl HorizontalNeonStackBlurPassFloat16 { #[inline] - unsafe fn pass_impl( + #[target_feature(enable = "neon")] + fn pass_impl( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -79,12 +56,12 @@ where total_threads: usize, ) { unsafe { - let pixels: &UnsafeSlice = std::mem::transmute(pixels); let div = ((radius * 2) + 1) as usize; let mut xp; let mut sp; let mut stack_start; - let mut stacks = vec![0f32; 4 * div]; + let mut scratch_buffer = ScratchBuffer::::new(4 * div); + let stacks = scratch_buffer.as_mut_slice(); let wm = width - 1; let div = (radius * 2) + 1; @@ -183,27 +160,12 @@ where } } -impl StackBlurWorkingPass - for HorizontalNeonStackBlurPassFloat16 -where - J: Copy - + 'static - + AddAssign - + Mul - + Sub - + PrimitiveCast - + SubAssign - + PrimitiveCast - + Default, - T: Copy + PrimitiveCast + Default, - i32: PrimitiveCast, - u32: PrimitiveCast, - f32: PrimitiveCast, - usize: PrimitiveCast, +impl StackBlurWorkingPass + for HorizontalNeonStackBlurPassFloat16 { fn pass( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, diff --git a/src/stackblur/neon/horizontal_f32.rs b/src/stackblur/neon/horizontal_f32.rs index 02fae5b6..1b0587be 100644 --- a/src/stackblur/neon/horizontal_f32.rs +++ b/src/stackblur/neon/horizontal_f32.rs @@ -29,6 +29,7 @@ use crate::neon::{load_f32_fast, p_vfmaq_f32, store_f32}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::arch::aarch64::*; pub(crate) struct HorizontalNeonStackBlurPassFloat32 {} @@ -57,7 +58,8 @@ impl HorizontalNeonStackBlurPassFloat32 { let mut xp; let mut sp; let mut stack_start; - let mut stacks = vec![0f32; 4 * div]; + let mut scratch_buffer = ScratchBuffer::::new(4 * div); + let stacks = scratch_buffer.as_mut_slice(); let wm = width - 1; let div = (radius * 2) + 1; diff --git a/src/stackblur/neon/horizontal_q0_31.rs b/src/stackblur/neon/horizontal_q0_31.rs index 86a58e9f..564b2c30 100644 --- a/src/stackblur/neon/horizontal_q0_31.rs +++ b/src/stackblur/neon/horizontal_q0_31.rs @@ -29,6 +29,7 @@ use crate::neon::{load_u8_s32_fast, store_u8_s32, store_u8_s32_x4}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::arch::aarch64::*; pub(crate) struct HorizontalNeonStackBlurPassQ0_31 {} @@ -56,7 +57,7 @@ impl StackBlurWorkingPass for HorizontalNeonStackBlurPa impl HorizontalNeonStackBlurPassQ0_31 { #[target_feature(enable = "rdm")] - unsafe fn pass_impl( + fn pass_impl( &self, pixels: &UnsafeSlice, stride: u32, @@ -74,7 +75,8 @@ impl HorizontalNeonStackBlurPassQ0_31 { let mut _xp; let mut sp; let mut stack_start; - let mut stacks0 = vec![0i32; 4 * div * 4]; + let mut scratch_buffer = ScratchBuffer::::new(4 * div * 4); + let stacks0 = scratch_buffer.as_mut_slice(); const Q: f64 = ((1i64 << 31i64) - 1) as f64; let recip_scale = Q / ((radius as f64 + 1.) * (radius as f64 + 1.)); diff --git a/src/stackblur/neon/vertical.rs b/src/stackblur/neon/vertical.rs index b203180f..f314c611 100644 --- a/src/stackblur/neon/vertical.rs +++ b/src/stackblur/neon/vertical.rs @@ -29,6 +29,7 @@ use crate::neon::{load_u8_s32_fast, store_u8_s32}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::arch::aarch64::*; pub(crate) struct VerticalNeonStackBlurPass {} @@ -55,7 +56,8 @@ impl StackBlurWorkingPass for VerticalNeonStackBlurPass let mut yp; let mut sp; let mut stack_start; - let mut stacks0 = vec![0i32; 4 * div * 4]; + let mut scratch_buffer = ScratchBuffer::::new(4 * div * 4); + let stacks0 = scratch_buffer.as_mut_slice(); let hm = height - 1; let div = (radius * 2) + 1; diff --git a/src/stackblur/neon/vertical_f16.rs b/src/stackblur/neon/vertical_f16.rs index 2966de9b..2734b33b 100644 --- a/src/stackblur/neon/vertical_f16.rs +++ b/src/stackblur/neon/vertical_f16.rs @@ -27,48 +27,25 @@ * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ use crate::neon::{load_f32_f16, p_vfmaq_f32, store_f32_f16}; -use crate::primitives::PrimitiveCast; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use core::f16; use std::arch::aarch64::*; -use std::marker::PhantomData; -use std::ops::{AddAssign, Mul, Sub, SubAssign}; -pub(crate) struct VerticalNeonStackBlurPassFloat16 { - _phantom_t: PhantomData, - _phantom_j: PhantomData, -} +pub(crate) struct VerticalNeonStackBlurPassFloat16 {} -impl Default for VerticalNeonStackBlurPassFloat16 { +impl Default for VerticalNeonStackBlurPassFloat16 { fn default() -> Self { - VerticalNeonStackBlurPassFloat16:: { - _phantom_t: Default::default(), - _phantom_j: Default::default(), - } + VerticalNeonStackBlurPassFloat16:: {} } } -impl VerticalNeonStackBlurPassFloat16 -where - J: Copy - + 'static - + AddAssign - + Mul - + Sub - + PrimitiveCast - + SubAssign - + PrimitiveCast - + Default, - T: Copy + PrimitiveCast + Default, - i32: PrimitiveCast, - u32: PrimitiveCast, - f32: PrimitiveCast, - usize: PrimitiveCast, -{ - unsafe fn pass_impl( +impl VerticalNeonStackBlurPassFloat16 { + #[target_feature(enable = "neon")] + fn pass_impl( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -77,12 +54,12 @@ where total_threads: usize, ) { unsafe { - let pixels: &UnsafeSlice = std::mem::transmute(pixels); let div = ((radius * 2) + 1) as usize; let mut yp; let mut sp; let mut stack_start; - let mut stacks = vec![0.; 4 * div * total_threads]; + let mut scratch_buffer = ScratchBuffer::::new(4 * div); + let stacks = scratch_buffer.as_mut_slice(); let v_mul_value = vdupq_n_f32(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); @@ -183,27 +160,10 @@ where } } -impl StackBlurWorkingPass - for VerticalNeonStackBlurPassFloat16 -where - J: Copy - + 'static - + AddAssign - + Mul - + Sub - + PrimitiveCast - + SubAssign - + PrimitiveCast - + Default, - T: Copy + PrimitiveCast + Default, - i32: PrimitiveCast, - u32: PrimitiveCast, - f32: PrimitiveCast, - usize: PrimitiveCast, -{ +impl StackBlurWorkingPass for VerticalNeonStackBlurPassFloat16 { fn pass( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, diff --git a/src/stackblur/neon/vertical_f32.rs b/src/stackblur/neon/vertical_f32.rs index 064f471b..91de5b12 100644 --- a/src/stackblur/neon/vertical_f32.rs +++ b/src/stackblur/neon/vertical_f32.rs @@ -29,6 +29,7 @@ use crate::neon::{load_f32_fast, p_vfmaq_f32, store_f32}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::arch::aarch64::*; pub(crate) struct VerticalNeonStackBlurPassFloat32 {} @@ -40,8 +41,8 @@ impl Default for VerticalNeonStackBlurPassFloat32 { } impl VerticalNeonStackBlurPassFloat32 { - #[inline] - unsafe fn pass_impl( + #[target_feature(enable = "neon")] + fn pass_impl( &self, pixels: &UnsafeSlice, stride: u32, @@ -52,12 +53,12 @@ impl VerticalNeonStackBlurPassFloat32 { total_threads: usize, ) { unsafe { - let pixels: &UnsafeSlice = std::mem::transmute(pixels); let div = ((radius * 2) + 1) as usize; let mut yp; let mut sp; let mut stack_start; - let mut stacks = vec![0.; 4 * div * total_threads]; + let mut scratch_buffer = ScratchBuffer::::new(4 * div); + let stacks = scratch_buffer.as_mut_slice(); let v_mul_value = vdupq_n_f32(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); diff --git a/src/stackblur/neon/vertical_q0_31.rs b/src/stackblur/neon/vertical_q0_31.rs index bd9e7fac..f5971a99 100644 --- a/src/stackblur/neon/vertical_q0_31.rs +++ b/src/stackblur/neon/vertical_q0_31.rs @@ -29,6 +29,7 @@ use crate::neon::{load_u8_s32_fast, store_u8_s32}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use std::arch::aarch64::*; pub(crate) struct VerticalNeonStackBlurPassQ0_31 {} @@ -70,7 +71,8 @@ impl VerticalNeonStackBlurPassQ0_31 { let mut yp; let mut sp; let mut stack_start; - let mut stacks0 = vec![0i32; 4 * div * 4]; + let mut scratch_buffer = ScratchBuffer::::new(4 * div * 4); + let stacks0 = scratch_buffer.as_mut_slice(); let hm = height - 1; let div = (radius * 2) + 1; diff --git a/src/stackblur/sse/horizontal.rs b/src/stackblur/sse/horizontal.rs index 2b65e32e..35d223ea 100644 --- a/src/stackblur/sse/horizontal.rs +++ b/src/stackblur/sse/horizontal.rs @@ -29,6 +29,7 @@ use crate::sse::{load_u8_s32_fast, store_u8_s32}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] @@ -42,8 +43,9 @@ impl Default for HorizontalSseStackBlurPass { } } -#[inline(always)] -unsafe fn sse_horiz_pass_impl( +#[inline] +#[target_feature(enable = "sse4.1")] +fn sse_horiz_pass_impl( pixels: &UnsafeSlice, stride: u32, width: u32, @@ -52,260 +54,249 @@ unsafe fn sse_horiz_pass_impl( thread: usize, total_threads: usize, ) { - let pixels: &UnsafeSlice = std::mem::transmute(pixels); - let div = ((radius * 2) + 1) as usize; - let mut xp; - let mut sp; - let mut stack_start; - let mut stacks = vec![0i32; 4 * div * 2]; - - let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); + unsafe { + let div = ((radius * 2) + 1) as usize; + let mut xp; + let mut sp; + let mut stack_start; + let mut scratch_buffer = ScratchBuffer::::new(4 * div * 2); + let stacks = scratch_buffer.as_mut_slice(); - let wm = width - 1; - let div = (radius * 2) + 1; + let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - let min_y = thread * height as usize / total_threads; - let max_y = (thread + 1) * height as usize / total_threads; + let wm = width - 1; + let div = (radius * 2) + 1; - let mut yy = min_y; + let min_y = thread * height as usize / total_threads; + let max_y = (thread + 1) * height as usize / total_threads; - while yy + 2 <= max_y { - let mut sums0 = _mm_setzero_si128(); - let mut sums1 = _mm_setzero_si128(); + let mut yy = min_y; - let mut sum_in0 = _mm_setzero_si128(); - let mut sum_in1 = _mm_setzero_si128(); + while yy + 2 <= max_y { + let mut sums0 = _mm_setzero_si128(); + let mut sums1 = _mm_setzero_si128(); - let mut sum_out0 = _mm_setzero_si128(); - let mut sum_out1 = _mm_setzero_si128(); + let mut sum_in0 = _mm_setzero_si128(); + let mut sum_in1 = _mm_setzero_si128(); - let mut src_ptr0 = stride as usize * yy; - let mut src_ptr1 = stride as usize * (yy + 1); + let mut sum_out0 = _mm_setzero_si128(); + let mut sum_out1 = _mm_setzero_si128(); - let src_ld0 = pixels.slice.as_ptr().add(src_ptr0) as *const i32; - let src_ld1 = pixels.slice.as_ptr().add(src_ptr1) as *const i32; + let mut src_ptr0 = stride as usize * yy; + let mut src_ptr1 = stride as usize * (yy + 1); - let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); - let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); + let src_ld0 = pixels.slice.as_ptr().add(src_ptr0) as *const i32; + let src_ld1 = pixels.slice.as_ptr().add(src_ptr1) as *const i32; - for i in 0..=radius { - let stack_value = stacks.as_mut_ptr().add(i as usize * 4 * 2); - _mm_storeu_si128(stack_value as *mut __m128i, src_pixel0); - _mm_storeu_si128(stack_value.add(4) as *mut __m128i, src_pixel1); + let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); + let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); - let w = _mm_set1_epi32(i as i32 + 1); - sums0 = _mm_add_epi32(sums0, _mm_madd_epi16(src_pixel0, w)); - sums1 = _mm_add_epi32(sums1, _mm_madd_epi16(src_pixel1, w)); + for i in 0..=radius { + let stack_value = stacks.as_mut_ptr().add(i as usize * 4 * 2); + _mm_storeu_si128(stack_value as *mut __m128i, src_pixel0); + _mm_storeu_si128(stack_value.add(4) as *mut __m128i, src_pixel1); - sum_out0 = _mm_add_epi32(sum_out0, src_pixel0); - sum_out1 = _mm_add_epi32(sum_out1, src_pixel1); - } + let w = _mm_set1_epi32(i as i32 + 1); + sums0 = _mm_add_epi32(sums0, _mm_madd_epi16(src_pixel0, w)); + sums1 = _mm_add_epi32(sums1, _mm_madd_epi16(src_pixel1, w)); - for i in 1..=radius { - if i <= wm { - src_ptr0 += CN; - src_ptr1 += CN; + sum_out0 = _mm_add_epi32(sum_out0, src_pixel0); + sum_out1 = _mm_add_epi32(sum_out1, src_pixel1); } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4 * 2); - let src_pixel0 = - load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr0) as *const u8); - let src_pixel1 = - load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr1) as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel0); - _mm_storeu_si128(stack_ptr.add(4) as *mut __m128i, src_pixel1); + for i in 1..=radius { + if i <= wm { + src_ptr0 += CN; + src_ptr1 += CN; + } + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4 * 2); + let src_pixel0 = + load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr0) as *const u8); + let src_pixel1 = + load_u8_s32_fast::(pixels.slice.as_ptr().add(src_ptr1) as *const u8); - let w = _mm_set1_epi32(radius as i32 + 1 - i as i32); + _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel0); + _mm_storeu_si128(stack_ptr.add(4) as *mut __m128i, src_pixel1); - sums0 = _mm_add_epi32(sums0, _mm_madd_epi16(src_pixel0, w)); - sums1 = _mm_add_epi32(sums1, _mm_madd_epi16(src_pixel1, w)); + let w = _mm_set1_epi32(radius as i32 + 1 - i as i32); - sum_in0 = _mm_add_epi32(sum_in0, src_pixel0); - sum_in1 = _mm_add_epi32(sum_in1, src_pixel1); - } + sums0 = _mm_add_epi32(sums0, _mm_madd_epi16(src_pixel0, w)); + sums1 = _mm_add_epi32(sums1, _mm_madd_epi16(src_pixel1, w)); - sp = radius; - xp = radius; - if xp > wm { - xp = wm; - } - - src_ptr0 = CN * xp as usize + yy * stride as usize; - src_ptr1 = CN * xp as usize + (yy + 1) * stride as usize; + sum_in0 = _mm_add_epi32(sum_in0, src_pixel0); + sum_in1 = _mm_add_epi32(sum_in1, src_pixel1); + } - let mut dst_ptr0 = yy * stride as usize; - let mut dst_ptr1 = (yy + 1) * stride as usize; + sp = radius; + xp = radius; + if xp > wm { + xp = wm; + } - for _ in 0..width { - let store_ld0 = pixels.slice.as_ptr().add(dst_ptr0) as *mut u8; - let store_ld1 = pixels.slice.as_ptr().add(dst_ptr1) as *mut u8; + src_ptr0 = CN * xp as usize + yy * stride as usize; + src_ptr1 = CN * xp as usize + (yy + 1) * stride as usize; - let o0 = _mm_cvtepi32_ps(sums0); - let o1 = _mm_cvtepi32_ps(sums1); + let mut dst_ptr0 = yy * stride as usize; + let mut dst_ptr1 = (yy + 1) * stride as usize; - let r0 = _mm_mul_ps(o0, v_mul_value); - let r1 = _mm_mul_ps(o1, v_mul_value); + for _ in 0..width { + let store_ld0 = pixels.slice.as_ptr().add(dst_ptr0) as *mut u8; + let store_ld1 = pixels.slice.as_ptr().add(dst_ptr1) as *mut u8; - store_u8_s32::(store_ld0, _mm_cvtps_epi32(r0)); - store_u8_s32::(store_ld1, _mm_cvtps_epi32(r1)); + let o0 = _mm_cvtepi32_ps(sums0); + let o1 = _mm_cvtepi32_ps(sums1); - dst_ptr0 += CN; - dst_ptr1 += CN; + let r0 = _mm_mul_ps(o0, v_mul_value); + let r1 = _mm_mul_ps(o1, v_mul_value); - sums0 = _mm_sub_epi32(sums0, sum_out0); - sums1 = _mm_sub_epi32(sums1, sum_out1); + store_u8_s32::(store_ld0, _mm_cvtps_epi32(r0)); + store_u8_s32::(store_ld1, _mm_cvtps_epi32(r1)); - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } - let stack = stacks.as_mut_ptr().add(stack_start as usize * 4 * 2); + dst_ptr0 += CN; + dst_ptr1 += CN; - let stack_val0 = _mm_loadu_si128(stack as *const __m128i); - let stack_val1 = _mm_loadu_si128(stack.add(4) as *const __m128i); + sums0 = _mm_sub_epi32(sums0, sum_out0); + sums1 = _mm_sub_epi32(sums1, sum_out1); - sum_out0 = _mm_sub_epi32(sum_out0, stack_val0); - sum_out1 = _mm_sub_epi32(sum_out1, stack_val1); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } + let stack = stacks.as_mut_ptr().add(stack_start as usize * 4 * 2); - if xp < wm { - src_ptr0 += CN; - src_ptr1 += CN; - xp += 1; - } + let stack_val0 = _mm_loadu_si128(stack as *const __m128i); + let stack_val1 = _mm_loadu_si128(stack.add(4) as *const __m128i); - let src_ld0 = pixels.slice.as_ptr().add(src_ptr0); - let src_ld1 = pixels.slice.as_ptr().add(src_ptr1); + sum_out0 = _mm_sub_epi32(sum_out0, stack_val0); + sum_out1 = _mm_sub_epi32(sum_out1, stack_val1); - let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); - let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); + if xp < wm { + src_ptr0 += CN; + src_ptr1 += CN; + xp += 1; + } - _mm_storeu_si128(stack as *mut __m128i, src_pixel0); - _mm_storeu_si128(stack.add(4) as *mut __m128i, src_pixel1); + let src_ld0 = pixels.slice.as_ptr().add(src_ptr0); + let src_ld1 = pixels.slice.as_ptr().add(src_ptr1); - sum_in0 = _mm_add_epi32(sum_in0, src_pixel0); - sum_in1 = _mm_add_epi32(sum_in1, src_pixel1); + let src_pixel0 = load_u8_s32_fast::(src_ld0 as *const u8); + let src_pixel1 = load_u8_s32_fast::(src_ld1 as *const u8); - sums0 = _mm_add_epi32(sums0, sum_in0); - sums1 = _mm_add_epi32(sums1, sum_in1); + _mm_storeu_si128(stack as *mut __m128i, src_pixel0); + _mm_storeu_si128(stack.add(4) as *mut __m128i, src_pixel1); - sp += 1; - if sp >= div { - sp = 0; - } - let stack = stacks.as_mut_ptr().add(sp as usize * 4 * 2); - let stack_val0 = _mm_loadu_si128(stack as *const __m128i); - let stack_val1 = _mm_loadu_si128(stack.add(4) as *const __m128i); + sum_in0 = _mm_add_epi32(sum_in0, src_pixel0); + sum_in1 = _mm_add_epi32(sum_in1, src_pixel1); - sum_out0 = _mm_add_epi32(sum_out0, stack_val0); - sum_out1 = _mm_add_epi32(sum_out1, stack_val1); + sums0 = _mm_add_epi32(sums0, sum_in0); + sums1 = _mm_add_epi32(sums1, sum_in1); - sum_in0 = _mm_sub_epi32(sum_in0, stack_val0); - sum_in1 = _mm_sub_epi32(sum_in1, stack_val1); - } + sp += 1; + if sp >= div { + sp = 0; + } + let stack = stacks.as_mut_ptr().add(sp as usize * 4 * 2); + let stack_val0 = _mm_loadu_si128(stack as *const __m128i); + let stack_val1 = _mm_loadu_si128(stack.add(4) as *const __m128i); - yy += 2; - } + sum_out0 = _mm_add_epi32(sum_out0, stack_val0); + sum_out1 = _mm_add_epi32(sum_out1, stack_val1); - for y in yy..max_y { - let mut sums = _mm_setzero_si128(); - let mut sum_in = _mm_setzero_si128(); - let mut sum_out = _mm_setzero_si128(); + sum_in0 = _mm_sub_epi32(sum_in0, stack_val0); + sum_in1 = _mm_sub_epi32(sum_in1, stack_val1); + } - let mut src_ptr = stride as usize * y; + yy += 2; + } - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + for y in yy..max_y { + let mut sums = _mm_setzero_si128(); + let mut sum_in = _mm_setzero_si128(); + let mut sum_out = _mm_setzero_si128(); - for i in 0..=radius { - let stack_value = stacks.as_mut_ptr().add(i as usize * 4); - _mm_storeu_si128(stack_value as *mut __m128i, src_pixel); - sums = _mm_add_epi32( - sums, - _mm_madd_epi16(src_pixel, _mm_set1_epi32(i as i32 + 1)), - ); - sum_out = _mm_add_epi32(sum_out, src_pixel); - } + let mut src_ptr = stride as usize * y; - for i in 1..=radius { - if i <= wm { - src_ptr += CN; - } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); - sums = _mm_add_epi32( - sums, - _mm_madd_epi16(src_pixel, _mm_set1_epi32(radius as i32 + 1 - i as i32)), - ); - sum_in = _mm_add_epi32(sum_in, src_pixel); - } + for i in 0..=radius { + let stack_value = stacks.as_mut_ptr().add(i as usize * 4); + _mm_storeu_si128(stack_value as *mut __m128i, src_pixel); + sums = _mm_add_epi32( + sums, + _mm_madd_epi16(src_pixel, _mm_set1_epi32(i as i32 + 1)), + ); + sum_out = _mm_add_epi32(sum_out, src_pixel); + } - sp = radius; - xp = radius; - if xp > wm { - xp = wm; - } + for i in 1..=radius { + if i <= wm { + src_ptr += CN; + } + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + sums = _mm_add_epi32( + sums, + _mm_madd_epi16(src_pixel, _mm_set1_epi32(radius as i32 + 1 - i as i32)), + ); + + sum_in = _mm_add_epi32(sum_in, src_pixel); + } - src_ptr = CN * xp as usize + y * stride as usize; - let mut dst_ptr = y * stride as usize; - for _ in 0..width { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - let result = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(sums), v_mul_value)); - store_u8_s32::(store_ld, result); - dst_ptr += CN; + sp = radius; + xp = radius; + if xp > wm { + xp = wm; + } - sums = _mm_sub_epi32(sums, sum_out); + src_ptr = CN * xp as usize + y * stride as usize; + let mut dst_ptr = y * stride as usize; + for _ in 0..width { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + let result = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(sums), v_mul_value)); + store_u8_s32::(store_ld, result); + dst_ptr += CN; - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } - let stack = stacks.as_mut_ptr().add(stack_start as usize * 4); + sums = _mm_sub_epi32(sums, sum_out); - let stack_val = _mm_loadu_si128(stack as *const __m128i); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } + let stack = stacks.as_mut_ptr().add(stack_start as usize * 4); - sum_out = _mm_sub_epi32(sum_out, stack_val); + let stack_val = _mm_loadu_si128(stack as *const __m128i); - if xp < wm { - src_ptr += CN; - xp += 1; - } + sum_out = _mm_sub_epi32(sum_out, stack_val); - let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack as *mut __m128i, src_pixel); + if xp < wm { + src_ptr += CN; + xp += 1; + } - sum_in = _mm_add_epi32(sum_in, src_pixel); - sums = _mm_add_epi32(sums, sum_in); + let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_storeu_si128(stack as *mut __m128i, src_pixel); - sp += 1; - if sp >= div { - sp = 0; - } - let stack = stacks.as_mut_ptr().add(sp as usize * 4); - let stack_val = _mm_loadu_si128(stack as *const __m128i); + sum_in = _mm_add_epi32(sum_in, src_pixel); + sums = _mm_add_epi32(sums, sum_in); + + sp += 1; + if sp >= div { + sp = 0; + } + let stack = stacks.as_mut_ptr().add(sp as usize * 4); + let stack_val = _mm_loadu_si128(stack as *const __m128i); - sum_out = _mm_add_epi32(sum_out, stack_val); - sum_in = _mm_sub_epi32(sum_in, stack_val); + sum_out = _mm_add_epi32(sum_out, stack_val); + sum_in = _mm_sub_epi32(sum_in, stack_val); + } } } } -#[target_feature(enable = "sse4.1")] -unsafe fn sse_horiz_pass_impl_def( - pixels: &UnsafeSlice, - stride: u32, - width: u32, - height: u32, - radius: u32, - thread: usize, - total_threads: usize, -) { - sse_horiz_pass_impl::(pixels, stride, width, height, radius, thread, total_threads); -} - impl StackBlurWorkingPass for HorizontalSseStackBlurPass { fn pass( &self, @@ -318,15 +309,7 @@ impl StackBlurWorkingPass for HorizontalSseStackBlurPas total_threads: usize, ) { unsafe { - sse_horiz_pass_impl_def::( - pixels, - stride, - width, - height, - radius, - thread, - total_threads, - ); + sse_horiz_pass_impl::(pixels, stride, width, height, radius, thread, total_threads); } } } diff --git a/src/stackblur/sse/horizontal_f16.rs b/src/stackblur/sse/horizontal_f16.rs index e171da25..44cb743c 100644 --- a/src/stackblur/sse/horizontal_f16.rs +++ b/src/stackblur/sse/horizontal_f16.rs @@ -26,57 +26,29 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -use crate::primitives::PrimitiveCast; use crate::sse::{load_f32_f16, store_f32_f16}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; use core::f16; -use num_traits::{AsPrimitive, FromPrimitive}; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; -use std::marker::PhantomData; -use std::ops::{AddAssign, Mul, Sub, SubAssign}; -pub(crate) struct HorizontalSseStackBlurPassFloat16 { - _phantom_t: PhantomData, - _phantom_j: PhantomData, -} +pub(crate) struct HorizontalSseStackBlurPassFloat16 {} -impl Default - for HorizontalSseStackBlurPassFloat16 -{ +impl Default for HorizontalSseStackBlurPassFloat16 { fn default() -> Self { - HorizontalSseStackBlurPassFloat16:: { - _phantom_t: Default::default(), - _phantom_j: Default::default(), - } + HorizontalSseStackBlurPassFloat16:: {} } } -impl HorizontalSseStackBlurPassFloat16 -where - J: Copy - + 'static - + FromPrimitive - + AddAssign - + Mul - + Sub - + PrimitiveCast - + SubAssign - + PrimitiveCast - + Default, - T: Copy + PrimitiveCast + Default, - i32: AsPrimitive, - u32: AsPrimitive, - f32: PrimitiveCast, - usize: AsPrimitive, -{ +impl HorizontalSseStackBlurPassFloat16 { #[target_feature(enable = "sse4.1,f16c")] - unsafe fn pass_impl( + fn pass_impl( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -85,12 +57,12 @@ where total_threads: usize, ) { unsafe { - let pixels: &UnsafeSlice = std::mem::transmute(pixels); let div = ((radius * 2) + 1) as usize; let mut xp; let mut sp; let mut stack_start; - let mut stacks = vec![0f32; 4 * div]; + let mut scratch_buffer = ScratchBuffer::::new(4 * div); + let stacks = scratch_buffer.as_mut_slice(); let wm = width - 1; let div = (radius * 2) + 1; @@ -188,28 +160,10 @@ where } } -impl StackBlurWorkingPass - for HorizontalSseStackBlurPassFloat16 -where - J: Copy - + 'static - + FromPrimitive - + AddAssign - + Mul - + Sub - + PrimitiveCast - + SubAssign - + PrimitiveCast - + Default, - T: Copy + PrimitiveCast + Default, - i32: AsPrimitive, - u32: AsPrimitive, - f32: PrimitiveCast, - usize: AsPrimitive, -{ +impl StackBlurWorkingPass for HorizontalSseStackBlurPassFloat16 { fn pass( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, diff --git a/src/stackblur/sse/horizontal_f32.rs b/src/stackblur/sse/horizontal_f32.rs index c600f219..206da7ad 100644 --- a/src/stackblur/sse/horizontal_f32.rs +++ b/src/stackblur/sse/horizontal_f32.rs @@ -45,7 +45,7 @@ impl Default for HorizontalSseStackBlurPassFloat32 { } #[target_feature(enable = "sse4.1")] -unsafe fn horiz_f32_pass_stack_impl( +fn horiz_f32_pass_stack_impl( pixels: &UnsafeSlice, stride: u32, width: u32, @@ -54,99 +54,101 @@ unsafe fn horiz_f32_pass_stack_impl( thread: usize, total_threads: usize, ) { - let div = ((radius * 2) + 1) as usize; - let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - let mut xp; - let mut sp; - let mut stack_start; - let mut stacks = vec![SseF32x4::default(); div]; + unsafe { + let div = ((radius * 2) + 1) as usize; + let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); + let mut xp; + let mut sp; + let mut stack_start; + let mut stacks = vec![SseF32x4::default(); div]; - let wm = width - 1; - let div = (radius * 2) + 1; + let wm = width - 1; + let div = (radius * 2) + 1; - let mut src_ptr; - let mut dst_ptr; + let mut src_ptr; + let mut dst_ptr; - let min_y = thread * height as usize / total_threads; - let max_y = (thread + 1) * height as usize / total_threads; + let min_y = thread * height as usize / total_threads; + let max_y = (thread + 1) * height as usize / total_threads; - for y in min_y..max_y { - let mut sums = _mm_setzero_ps(); - let mut sum_in = _mm_setzero_ps(); - let mut sum_out = _mm_setzero_ps(); + for y in min_y..max_y { + let mut sums = _mm_setzero_ps(); + let mut sum_in = _mm_setzero_ps(); + let mut sum_out = _mm_setzero_ps(); - src_ptr = stride as usize * y; + src_ptr = stride as usize * y; - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; - let src_pixel = load_f32::(src_ld); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; + let src_pixel = load_f32::(src_ld); - for i in 0..=radius { - let stack_value = stacks.as_mut_ptr().add(i as usize); - _mm_store_ps(stack_value.cast(), src_pixel); - sums = _mm_opt_fmlaf_ps(sums, src_pixel, _mm_set1_ps((i + 1) as f32)); - sum_out = _mm_add_ps(sum_out, src_pixel); - } + for i in 0..=radius { + let stack_value = stacks.as_mut_ptr().add(i as usize); + _mm_store_ps(stack_value.cast(), src_pixel); + sums = _mm_opt_fmlaf_ps(sums, src_pixel, _mm_set1_ps((i + 1) as f32)); + sum_out = _mm_add_ps(sum_out, src_pixel); + } - for i in 1..=radius { - if i <= wm { - src_ptr += CN; + for i in 1..=radius { + if i <= wm { + src_ptr += CN; + } + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; + let src_pixel = load_f32::(src_ld); + _mm_store_ps(stack_ptr.cast(), src_pixel); + sums = _mm_opt_fmlaf_ps(sums, src_pixel, _mm_set1_ps((radius + 1 - i) as f32)); + + sum_in = _mm_add_ps(sum_in, src_pixel); } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; - let src_pixel = load_f32::(src_ld); - _mm_store_ps(stack_ptr.cast(), src_pixel); - sums = _mm_opt_fmlaf_ps(sums, src_pixel, _mm_set1_ps((radius + 1 - i) as f32)); - sum_in = _mm_add_ps(sum_in, src_pixel); - } + sp = radius; + xp = radius; + if xp > wm { + xp = wm; + } - sp = radius; - xp = radius; - if xp > wm { - xp = wm; - } + src_ptr = CN * xp as usize + y * stride as usize; + dst_ptr = y * stride as usize; + for _ in 0..width { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut f32; + let blurred = _mm_mul_ps(sums, v_mul_value); + store_f32::(store_ld, blurred); + dst_ptr += CN; - src_ptr = CN * xp as usize + y * stride as usize; - dst_ptr = y * stride as usize; - for _ in 0..width { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut f32; - let blurred = _mm_mul_ps(sums, v_mul_value); - store_f32::(store_ld, blurred); - dst_ptr += CN; + sums = _mm_sub_ps(sums, sum_out); - sums = _mm_sub_ps(sums, sum_out); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } + let stack = stacks.as_mut_ptr().add(stack_start as usize); - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } - let stack = stacks.as_mut_ptr().add(stack_start as usize); + let stack_val = _mm_load_ps(stack.cast()); - let stack_val = _mm_load_ps(stack.cast()); + sum_out = _mm_sub_ps(sum_out, stack_val); - sum_out = _mm_sub_ps(sum_out, stack_val); + if xp < wm { + src_ptr += CN; + xp += 1; + } - if xp < wm { - src_ptr += CN; - xp += 1; - } + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; + let src_pixel = load_f32::(src_ld); + _mm_store_ps(stack.cast(), src_pixel); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const f32; - let src_pixel = load_f32::(src_ld); - _mm_store_ps(stack.cast(), src_pixel); + sum_in = _mm_add_ps(sum_in, src_pixel); + sums = _mm_add_ps(sums, sum_in); - sum_in = _mm_add_ps(sum_in, src_pixel); - sums = _mm_add_ps(sums, sum_in); + sp += 1; + if sp >= div { + sp = 0; + } + let stack = stacks.as_mut_ptr().add(sp as usize); + let stack_val = _mm_load_ps(stack.cast()); - sp += 1; - if sp >= div { - sp = 0; + sum_out = _mm_add_ps(sum_out, stack_val); + sum_in = _mm_sub_ps(sum_in, stack_val); } - let stack = stacks.as_mut_ptr().add(sp as usize); - let stack_val = _mm_load_ps(stack.cast()); - - sum_out = _mm_add_ps(sum_out, stack_val); - sum_in = _mm_sub_ps(sum_in, stack_val); } } } diff --git a/src/stackblur/sse/vertical.rs b/src/stackblur/sse/vertical.rs index c9c9af70..0abcb8dd 100644 --- a/src/stackblur/sse/vertical.rs +++ b/src/stackblur/sse/vertical.rs @@ -29,6 +29,7 @@ use crate::sse::{load_u8_s32_fast, store_u8_s32}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; +use crate::util::ScratchBuffer; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] @@ -44,7 +45,7 @@ impl Default for VerticalSseStackBlurPass { impl VerticalSseStackBlurPass { #[target_feature(enable = "sse4.1")] - unsafe fn pass_impl( + fn pass_impl( &self, pixels: &UnsafeSlice, stride: u32, @@ -54,357 +55,359 @@ impl VerticalSseStackBlurPass { thread: usize, total_threads: usize, ) { - let pixels: &UnsafeSlice = std::mem::transmute(pixels); - let div = ((radius * 2) + 1) as usize; - let mut yp; - let mut sp; - let mut stack_start; - let mut stacks = vec![0i32; 4 * div * 4]; + unsafe { + let div = ((radius * 2) + 1) as usize; + let mut yp; + let mut sp; + let mut stack_start; + let mut scratch_buffer = ScratchBuffer::::new(4 * div * 4); + let stacks = scratch_buffer.as_mut_slice(); - let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); + let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - let hm = height - 1; - let div = (radius * 2) + 1; + let hm = height - 1; + let div = (radius * 2) + 1; - let mut src_ptr; - let mut dst_ptr; + let mut src_ptr; + let mut dst_ptr; - let min_x = (thread * width as usize / total_threads) * CN; - let max_x = ((thread + 1) * width as usize / total_threads) * CN; + let min_x = (thread * width as usize / total_threads) * CN; + let max_x = ((thread + 1) * width as usize / total_threads) * CN; - let mut cx = min_x; + let mut cx = min_x; - while cx + 8 <= max_x { - let mut sums0 = _mm_setzero_si128(); - let mut sums1 = _mm_setzero_si128(); + while cx + 8 <= max_x { + let mut sums0 = _mm_setzero_si128(); + let mut sums1 = _mm_setzero_si128(); - let mut sum_in0 = _mm_setzero_si128(); - let mut sum_in1 = _mm_setzero_si128(); + let mut sum_in0 = _mm_setzero_si128(); + let mut sum_in1 = _mm_setzero_si128(); - let mut sum_out0 = _mm_setzero_si128(); - let mut sum_out1 = _mm_setzero_si128(); + let mut sum_out0 = _mm_setzero_si128(); + let mut sum_out1 = _mm_setzero_si128(); - let mut src_ptr = cx; // x,0 + let mut src_ptr = cx; // x,0 - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - { - let src_pixel0 = _mm_loadu_si64(src_ld as *const _); - let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); + { + let src_pixel0 = _mm_loadu_si64(src_ld as *const _); + let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); - let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); + let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - for i in 0..=radius { - let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4 * 4); + for i in 0..=radius { + let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4 * 4); - _mm_storeu_si128(stack_ptr as *mut _, i16_l0); - _mm_storeu_si128(stack_ptr.add(4) as *mut _, i16_h0); + _mm_storeu_si128(stack_ptr as *mut _, i16_l0); + _mm_storeu_si128(stack_ptr.add(4) as *mut _, i16_h0); - let j0 = _mm_madd_epi16(i16_l0, _mm_set1_epi32(i as i32 + 1)); - let j1 = _mm_madd_epi16(i16_h0, _mm_set1_epi32(i as i32 + 1)); + let j0 = _mm_madd_epi16(i16_l0, _mm_set1_epi32(i as i32 + 1)); + let j1 = _mm_madd_epi16(i16_h0, _mm_set1_epi32(i as i32 + 1)); - sums0 = _mm_add_epi32(sums0, j0); - sums1 = _mm_add_epi32(sums1, j1); + sums0 = _mm_add_epi32(sums0, j0); + sums1 = _mm_add_epi32(sums1, j1); - sum_out0 = _mm_add_epi32(sum_out0, i16_l0); - sum_out1 = _mm_add_epi32(sum_out1, i16_h0); + sum_out0 = _mm_add_epi32(sum_out0, i16_l0); + sum_out1 = _mm_add_epi32(sum_out1, i16_h0); + } } - } - { - for i in 1..=radius { - if i <= hm { - src_ptr += stride as usize; - } + { + for i in 1..=radius { + if i <= hm { + src_ptr += stride as usize; + } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4 * 4); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel0 = _mm_loadu_si64(src_ld as *const u8); - let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4 * 4); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel0 = _mm_loadu_si64(src_ld as *const u8); + let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); - let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); + let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - _mm_storeu_si128(stack_ptr as *mut _, i16_l0); - _mm_storeu_si128(stack_ptr.add(4) as *mut _, i16_h0); + _mm_storeu_si128(stack_ptr as *mut _, i16_l0); + _mm_storeu_si128(stack_ptr.add(4) as *mut _, i16_h0); - let vj = _mm_set1_epi32(radius as i32 + 1 - i as i32); + let vj = _mm_set1_epi32(radius as i32 + 1 - i as i32); - let j0 = _mm_madd_epi16(i16_l0, vj); - let j1 = _mm_madd_epi16(i16_h0, vj); + let j0 = _mm_madd_epi16(i16_l0, vj); + let j1 = _mm_madd_epi16(i16_h0, vj); - sums0 = _mm_add_epi32(sums0, j0); - sums1 = _mm_add_epi32(sums1, j1); + sums0 = _mm_add_epi32(sums0, j0); + sums1 = _mm_add_epi32(sums1, j1); - sum_in0 = _mm_add_epi32(sum_in0, i16_l0); - sum_in1 = _mm_add_epi32(sum_in1, i16_h0); + sum_in0 = _mm_add_epi32(sum_in0, i16_l0); + sum_in1 = _mm_add_epi32(sum_in1, i16_h0); + } } - } - sp = radius; - yp = radius; - if yp > hm { - yp = hm; - } - src_ptr = cx + yp as usize * stride as usize; - let mut dst_ptr = cx; - for _ in 0..height { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + sp = radius; + yp = radius; + if yp > hm { + yp = hm; + } + src_ptr = cx + yp as usize * stride as usize; + let mut dst_ptr = cx; + for _ in 0..height { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - let casted_sum0 = _mm_cvtepi32_ps(sums0); - let casted_sum1 = _mm_cvtepi32_ps(sums1); + let casted_sum0 = _mm_cvtepi32_ps(sums0); + let casted_sum1 = _mm_cvtepi32_ps(sums1); - let a0 = _mm_mul_ps(casted_sum0, v_mul_value); - let a1 = _mm_mul_ps(casted_sum1, v_mul_value); + let a0 = _mm_mul_ps(casted_sum0, v_mul_value); + let a1 = _mm_mul_ps(casted_sum1, v_mul_value); - let scaled_val0 = _mm_cvtps_epi32(a0); - let scaled_val1 = _mm_cvtps_epi32(a1); + let scaled_val0 = _mm_cvtps_epi32(a0); + let scaled_val1 = _mm_cvtps_epi32(a1); - let jv0 = _mm_packus_epi32(scaled_val0, scaled_val1); + let jv0 = _mm_packus_epi32(scaled_val0, scaled_val1); - _mm_storeu_si64(store_ld, _mm_packus_epi16(jv0, _mm_setzero_si128())); + _mm_storeu_si64(store_ld, _mm_packus_epi16(jv0, _mm_setzero_si128())); - dst_ptr += stride as usize; + dst_ptr += stride as usize; - sums0 = _mm_sub_epi32(sums0, sum_out0); - sums1 = _mm_sub_epi32(sums1, sum_out1); + sums0 = _mm_sub_epi32(sums0, sum_out0); + sums1 = _mm_sub_epi32(sums1, sum_out1); - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } - let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4 * 4); + let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4 * 4); - let stack_val0 = _mm_loadu_si128(stack_ptr as *const _); - let stack_val1 = _mm_loadu_si128(stack_ptr.add(4) as *const _); + let stack_val0 = _mm_loadu_si128(stack_ptr as *const _); + let stack_val1 = _mm_loadu_si128(stack_ptr.add(4) as *const _); - sum_out0 = _mm_sub_epi32(sum_out0, stack_val0); - sum_out1 = _mm_sub_epi32(sum_out1, stack_val1); + sum_out0 = _mm_sub_epi32(sum_out0, stack_val0); + sum_out1 = _mm_sub_epi32(sum_out1, stack_val1); - if yp < hm { - src_ptr += stride as usize; // stride - yp += 1; - } + if yp < hm { + src_ptr += stride as usize; // stride + yp += 1; + } + + let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_pixel0 = _mm_loadu_si64(src_ld as *const u8); + let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); - let src_pixel0 = _mm_loadu_si64(src_ld as *const u8); - let lo0 = _mm_unpacklo_epi8(src_pixel0, _mm_setzero_si128()); + let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); + let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); - let i16_l0 = _mm_unpacklo_epi16(lo0, _mm_setzero_si128()); - let i16_h0 = _mm_unpackhi_epi16(lo0, _mm_setzero_si128()); + _mm_storeu_si128(stack_ptr as *mut _, i16_l0); + _mm_storeu_si128(stack_ptr.add(4) as *mut _, i16_h0); - _mm_storeu_si128(stack_ptr as *mut _, i16_l0); - _mm_storeu_si128(stack_ptr.add(4) as *mut _, i16_h0); + sum_in0 = _mm_add_epi32(sum_in0, i16_l0); + sum_in1 = _mm_add_epi32(sum_in1, i16_h0); - sum_in0 = _mm_add_epi32(sum_in0, i16_l0); - sum_in1 = _mm_add_epi32(sum_in1, i16_h0); + sums0 = _mm_add_epi32(sums0, sum_in0); + sums1 = _mm_add_epi32(sums1, sum_in1); - sums0 = _mm_add_epi32(sums0, sum_in0); - sums1 = _mm_add_epi32(sums1, sum_in1); + sp += 1; - sp += 1; + if sp >= div { + sp = 0; + } + let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4 * 4); - if sp >= div { - sp = 0; - } - let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4 * 4); + let stack_val0 = _mm_loadu_si128(stack_ptr as *const _); + let stack_val1 = _mm_loadu_si128(stack_ptr.add(4) as *const _); - let stack_val0 = _mm_loadu_si128(stack_ptr as *const _); - let stack_val1 = _mm_loadu_si128(stack_ptr.add(4) as *const _); + sum_out0 = _mm_add_epi32(sum_out0, stack_val0); + sum_out1 = _mm_add_epi32(sum_out1, stack_val1); - sum_out0 = _mm_add_epi32(sum_out0, stack_val0); - sum_out1 = _mm_add_epi32(sum_out1, stack_val1); + sum_in0 = _mm_sub_epi32(sum_in0, stack_val0); + sum_in1 = _mm_sub_epi32(sum_in1, stack_val1); + } - sum_in0 = _mm_sub_epi32(sum_in0, stack_val0); - sum_in1 = _mm_sub_epi32(sum_in1, stack_val1); + cx += 8; } - cx += 8; - } + while cx + CN < max_x { + let mut sums = _mm_setzero_si128(); + let mut sum_in = _mm_setzero_si128(); + let mut sum_out = _mm_setzero_si128(); - while cx + CN < max_x { - let mut sums = _mm_setzero_si128(); - let mut sum_in = _mm_setzero_si128(); - let mut sum_out = _mm_setzero_si128(); + src_ptr = cx; // x,0 - src_ptr = cx; // x,0 + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + for i in 0..=radius { + let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); + _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + sums = _mm_add_epi32( + sums, + _mm_madd_epi16(src_pixel, _mm_set1_epi32(i as i32 + 1)), + ); + sum_out = _mm_add_epi32(sum_out, src_pixel); + } - for i in 0..=radius { - let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); - sums = _mm_add_epi32( - sums, - _mm_madd_epi16(src_pixel, _mm_set1_epi32(i as i32 + 1)), - ); - sum_out = _mm_add_epi32(sum_out, src_pixel); - } + for i in 1..=radius { + if i <= hm { + src_ptr += stride as usize; + } - for i in 1..=radius { - if i <= hm { - src_ptr += stride as usize; + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + sums = _mm_add_epi32( + sums, + _mm_madd_epi16(src_pixel, _mm_set1_epi32(radius as i32 + 1 - i as i32)), + ); + + sum_in = _mm_add_epi32(sum_in, src_pixel); } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); - sums = _mm_add_epi32( - sums, - _mm_madd_epi16(src_pixel, _mm_set1_epi32(radius as i32 + 1 - i as i32)), - ); - - sum_in = _mm_add_epi32(sum_in, src_pixel); - } + sp = radius; + yp = radius; + if yp > hm { + yp = hm; + } + src_ptr = cx + yp as usize * stride as usize; + dst_ptr = cx; + for _ in 0..height { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + let result = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(sums), v_mul_value)); + store_u8_s32::(store_ld, result); - sp = radius; - yp = radius; - if yp > hm { - yp = hm; - } - src_ptr = cx + yp as usize * stride as usize; - dst_ptr = cx; - for _ in 0..height { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - let result = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(sums), v_mul_value)); - store_u8_s32::(store_ld, result); + dst_ptr += stride as usize; - dst_ptr += stride as usize; + sums = _mm_sub_epi32(sums, sum_out); - sums = _mm_sub_epi32(sums, sum_out); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } + let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); + let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); + sum_out = _mm_sub_epi32(sum_out, stack_val); - let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); - let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); - sum_out = _mm_sub_epi32(sum_out, stack_val); + if yp < hm { + src_ptr += stride as usize; // stride + yp += 1; + } - if yp < hm { - src_ptr += stride as usize; // stride - yp += 1; - } + let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); - let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + sum_in = _mm_add_epi32(sum_in, src_pixel); + sums = _mm_add_epi32(sums, sum_in); - sum_in = _mm_add_epi32(sum_in, src_pixel); - sums = _mm_add_epi32(sums, sum_in); + sp += 1; - sp += 1; + if sp >= div { + sp = 0; + } + let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); + let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); - if sp >= div { - sp = 0; + sum_out = _mm_add_epi32(sum_out, stack_val); + sum_in = _mm_sub_epi32(sum_in, stack_val); } - let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); - let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); - sum_out = _mm_add_epi32(sum_out, stack_val); - sum_in = _mm_sub_epi32(sum_in, stack_val); + cx += CN; } - cx += CN; - } + const TAIL: usize = 1; - const TAIL: usize = 1; + while cx < max_x { + let mut sums = _mm_setzero_si128(); + let mut sum_in = _mm_setzero_si128(); + let mut sum_out = _mm_setzero_si128(); - while cx < max_x { - let mut sums = _mm_setzero_si128(); - let mut sum_in = _mm_setzero_si128(); - let mut sum_out = _mm_setzero_si128(); + src_ptr = cx; // x,0 - src_ptr = cx; // x,0 + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + for i in 0..=radius { + let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); + _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + sums = _mm_add_epi32( + sums, + _mm_madd_epi16(src_pixel, _mm_set1_epi32(i as i32 + 1)), + ); + sum_out = _mm_add_epi32(sum_out, src_pixel); + } - for i in 0..=radius { - let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); - sums = _mm_add_epi32( - sums, - _mm_madd_epi16(src_pixel, _mm_set1_epi32(i as i32 + 1)), - ); - sum_out = _mm_add_epi32(sum_out, src_pixel); - } + for i in 1..=radius { + if i <= hm { + src_ptr += stride as usize; + } - for i in 1..=radius { - if i <= hm { - src_ptr += stride as usize; + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + sums = _mm_add_epi32( + sums, + _mm_madd_epi16(src_pixel, _mm_set1_epi32(radius as i32 + 1 - i as i32)), + ); + + sum_in = _mm_add_epi32(sum_in, src_pixel); } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); - sums = _mm_add_epi32( - sums, - _mm_madd_epi16(src_pixel, _mm_set1_epi32(radius as i32 + 1 - i as i32)), - ); - - sum_in = _mm_add_epi32(sum_in, src_pixel); - } + sp = radius; + yp = radius; + if yp > hm { + yp = hm; + } + src_ptr = cx + yp as usize * stride as usize; + dst_ptr = cx; + for _ in 0..height { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + let result = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(sums), v_mul_value)); + store_u8_s32::(store_ld, result); - sp = radius; - yp = radius; - if yp > hm { - yp = hm; - } - src_ptr = cx + yp as usize * stride as usize; - dst_ptr = cx; - for _ in 0..height { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - let result = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(sums), v_mul_value)); - store_u8_s32::(store_ld, result); + dst_ptr += stride as usize; - dst_ptr += stride as usize; + sums = _mm_sub_epi32(sums, sum_out); - sums = _mm_sub_epi32(sums, sum_out); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } + let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); + let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); + sum_out = _mm_sub_epi32(sum_out, stack_val); - let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); - let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); - sum_out = _mm_sub_epi32(sum_out, stack_val); + if yp < hm { + src_ptr += stride as usize; // stride + yp += 1; + } - if yp < hm { - src_ptr += stride as usize; // stride - yp += 1; - } + let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); - let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - _mm_storeu_si128(stack_ptr as *mut __m128i, src_pixel); + sum_in = _mm_add_epi32(sum_in, src_pixel); + sums = _mm_add_epi32(sums, sum_in); - sum_in = _mm_add_epi32(sum_in, src_pixel); - sums = _mm_add_epi32(sums, sum_in); + sp += 1; - sp += 1; + if sp >= div { + sp = 0; + } + let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); + let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); - if sp >= div { - sp = 0; + sum_out = _mm_add_epi32(sum_out, stack_val); + sum_in = _mm_sub_epi32(sum_in, stack_val); } - let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); - let stack_val = _mm_loadu_si128(stack_ptr as *const __m128i); - sum_out = _mm_add_epi32(sum_out, stack_val); - sum_in = _mm_sub_epi32(sum_in, stack_val); + cx += TAIL; } - - cx += TAIL; } } } diff --git a/src/stackblur/sse/vertical_f16.rs b/src/stackblur/sse/vertical_f16.rs index 110f5e61..482f0209 100644 --- a/src/stackblur/sse/vertical_f16.rs +++ b/src/stackblur/sse/vertical_f16.rs @@ -26,55 +26,28 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -use crate::primitives::PrimitiveCast; use crate::sse::{load_f32_f16, store_f32_f16}; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; use core::f16; -use num_traits::{AsPrimitive, FromPrimitive}; #[cfg(target_arch = "x86")] use std::arch::x86::*; #[cfg(target_arch = "x86_64")] use std::arch::x86_64::*; -use std::marker::PhantomData; -use std::ops::{AddAssign, Mul, Sub, SubAssign}; -pub(crate) struct VerticalSseStackBlurPassFloat16 { - _phantom_t: PhantomData, - _phantom_j: PhantomData, -} +pub(crate) struct VerticalSseStackBlurPassFloat16 {} -impl Default for VerticalSseStackBlurPassFloat16 { +impl Default for VerticalSseStackBlurPassFloat16 { fn default() -> Self { - VerticalSseStackBlurPassFloat16:: { - _phantom_t: Default::default(), - _phantom_j: Default::default(), - } + VerticalSseStackBlurPassFloat16:: {} } } -impl VerticalSseStackBlurPassFloat16 -where - J: Copy - + 'static - + FromPrimitive - + AddAssign - + Mul - + Sub - + PrimitiveCast - + SubAssign - + PrimitiveCast - + Default, - T: Copy + PrimitiveCast + Default, - i32: AsPrimitive, - u32: AsPrimitive, - f32: PrimitiveCast, - usize: AsPrimitive, -{ +impl VerticalSseStackBlurPassFloat16 { #[target_feature(enable = "sse4.1")] - unsafe fn pass_impl( + fn pass_impl( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -83,12 +56,11 @@ where total_threads: usize, ) { unsafe { - let pixels: &UnsafeSlice = std::mem::transmute(pixels); let div = ((radius * 2) + 1) as usize; let mut yp; let mut sp; let mut stack_start; - let mut stacks = vec![0.; 4 * div * total_threads]; + let mut stacks = vec![0.; 4 * div]; let v_mul_value = _mm_set1_ps(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); @@ -188,28 +160,10 @@ where } } -impl StackBlurWorkingPass - for VerticalSseStackBlurPassFloat16 -where - J: Copy - + 'static - + FromPrimitive - + AddAssign - + Mul - + Sub - + PrimitiveCast - + SubAssign - + PrimitiveCast - + Default, - T: Copy + PrimitiveCast + Default, - i32: AsPrimitive, - u32: AsPrimitive, - f32: PrimitiveCast, - usize: AsPrimitive, -{ +impl StackBlurWorkingPass for VerticalSseStackBlurPassFloat16 { fn pass( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, diff --git a/src/stackblur/stack_blur.rs b/src/stackblur/stack_blur.rs index 4576df6f..589f4365 100644 --- a/src/stackblur/stack_blur.rs +++ b/src/stackblur/stack_blur.rs @@ -85,7 +85,7 @@ fn stack_blur_worker_horizontal( #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] fn select_blur_pass() -> impl StackBlurWorkingPass { - HorizontalWasmStackBlurPass::::default() + HorizontalWasmStackBlurPass::::default() } #[cfg(not(any( @@ -163,7 +163,7 @@ fn stack_blur_worker_vertical( #[cfg(all(target_arch = "wasm32", target_feature = "simd128"))] fn select_blur_pass() -> impl StackBlurWorkingPass { - VerticalWasmStackBlurPass::::default() + VerticalWasmStackBlurPass::::default() } #[cfg(not(any( diff --git a/src/stackblur/stack_blur_f16.rs b/src/stackblur/stack_blur_f16.rs index ce39a52f..71fa2a52 100644 --- a/src/stackblur/stack_blur_f16.rs +++ b/src/stackblur/stack_blur_f16.rs @@ -58,13 +58,12 @@ fn stack_blur_worker_horizontal( if std::arch::is_x86_feature_detected!("sse4.1") && std::arch::is_x86_feature_detected!("f16c") { - _executor = - Box::new(HorizontalSseStackBlurPassFloat16::::default()); + _executor = Box::new(HorizontalSseStackBlurPassFloat16::<1>::default()); } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _executor = Box::new(HorizontalNeonStackBlurPassFloat16::::default()); + _executor = Box::new(HorizontalNeonStackBlurPassFloat16::<1>::default()); } _executor.pass(slice, stride, width, height, radius, thread, thread_count); } @@ -77,13 +76,12 @@ fn stack_blur_worker_horizontal( if std::arch::is_x86_feature_detected!("sse4.1") && std::arch::is_x86_feature_detected!("f16c") { - _executor = - Box::new(HorizontalSseStackBlurPassFloat16::::default()); + _executor = Box::new(HorizontalSseStackBlurPassFloat16::<3>::default()); } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _executor = Box::new(HorizontalNeonStackBlurPassFloat16::::default()); + _executor = Box::new(HorizontalNeonStackBlurPassFloat16::<3>::default()); } _executor.pass(slice, stride, width, height, radius, thread, thread_count); } @@ -96,13 +94,12 @@ fn stack_blur_worker_horizontal( if std::arch::is_x86_feature_detected!("sse4.1") && std::arch::is_x86_feature_detected!("f16c") { - _executor = - Box::new(HorizontalSseStackBlurPassFloat16::::default()); + _executor = Box::new(HorizontalSseStackBlurPassFloat16::<4>::default()); } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _executor = Box::new(HorizontalNeonStackBlurPassFloat16::::default()); + _executor = Box::new(HorizontalNeonStackBlurPassFloat16::<4>::default()); } _executor.pass(slice, stride, width, height, radius, thread, thread_count); } @@ -129,12 +126,12 @@ fn stack_blur_worker_vertical( if std::arch::is_x86_feature_detected!("sse4.1") && std::arch::is_x86_feature_detected!("f16c") { - _executor = Box::new(VerticalSseStackBlurPassFloat16::::default()); + _executor = Box::new(VerticalSseStackBlurPassFloat16::<1>::default()); } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _executor = Box::new(VerticalNeonStackBlurPassFloat16::::default()); + _executor = Box::new(VerticalNeonStackBlurPassFloat16::<1>::default()); } _executor.pass(slice, stride, width, height, radius, thread, thread_count); } @@ -147,12 +144,12 @@ fn stack_blur_worker_vertical( if std::arch::is_x86_feature_detected!("sse4.1") && std::arch::is_x86_feature_detected!("f16c") { - _executor = Box::new(VerticalSseStackBlurPassFloat16::::default()); + _executor = Box::new(VerticalSseStackBlurPassFloat16::<3>::default()); } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _executor = Box::new(VerticalNeonStackBlurPassFloat16::::default()); + _executor = Box::new(VerticalNeonStackBlurPassFloat16::<3>::default()); } _executor.pass(slice, stride, width, height, radius, thread, thread_count); } @@ -165,12 +162,12 @@ fn stack_blur_worker_vertical( if std::arch::is_x86_feature_detected!("sse4.1") && std::arch::is_x86_feature_detected!("f16c") { - _executor = Box::new(VerticalSseStackBlurPassFloat16::::default()); + _executor = Box::new(VerticalSseStackBlurPassFloat16::<4>::default()); } } #[cfg(all(target_arch = "aarch64", feature = "neon"))] { - _executor = Box::new(VerticalNeonStackBlurPassFloat16::::default()); + _executor = Box::new(VerticalNeonStackBlurPassFloat16::<4>::default()); } _executor.pass(slice, stride, width, height, radius, thread, thread_count); } diff --git a/src/stackblur/wasm/horizontal.rs b/src/stackblur/wasm/horizontal.rs index 8499b3f7..1c57b5cb 100644 --- a/src/stackblur/wasm/horizontal.rs +++ b/src/stackblur/wasm/horizontal.rs @@ -26,52 +26,25 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -use crate::primitives::PrimitiveCast; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; use crate::wasm32::*; -use num_traits::AsPrimitive; use std::arch::wasm32::*; -use std::marker::PhantomData; -use std::ops::{AddAssign, Mul, Shr, Sub, SubAssign}; -pub struct HorizontalWasmStackBlurPass { - _phantom_t: PhantomData, - _phantom_j: PhantomData, -} +pub(crate) struct HorizontalWasmStackBlurPass {} -impl Default for HorizontalWasmStackBlurPass { +impl Default for HorizontalWasmStackBlurPass { fn default() -> Self { - HorizontalWasmStackBlurPass:: { - _phantom_t: Default::default(), - _phantom_j: Default::default(), - } + HorizontalWasmStackBlurPass:: {} } } -impl HorizontalWasmStackBlurPass -where - J: Copy - + 'static - + AddAssign - + Mul - + Shr - + Sub - + AsPrimitive - + SubAssign - + AsPrimitive - + Default, - T: Copy + AsPrimitive + Default, - i32: AsPrimitive, - u32: AsPrimitive, - f32: PrimitiveCast, - usize: AsPrimitive, -{ +impl HorizontalWasmStackBlurPass { #[inline] #[target_feature(enable = "simd128")] - unsafe fn pass_impl( + fn pass_impl( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -79,131 +52,114 @@ where thread: usize, total_threads: usize, ) { - let pixels: &UnsafeSlice = std::mem::transmute(pixels); - - let div = ((radius * 2) + 1) as usize; - let mut _xp; - let mut sp; - let mut stack_start; - let mut stacks = vec![0i32; 4 * div]; + unsafe { + let div = ((radius * 2) + 1) as usize; + let mut _xp; + let mut sp; + let mut stack_start; + let mut stacks = vec![0i32; 4 * div]; - let mul_value = f32x4_splat(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); + let mul_value = f32x4_splat(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - let wm = width - 1; - let div = (radius * 2) + 1; + let wm = width - 1; + let div = (radius * 2) + 1; - let min_y = thread * height as usize / total_threads; - let max_y = (thread + 1) * height as usize / total_threads; + let min_y = thread * height as usize / total_threads; + let max_y = (thread + 1) * height as usize / total_threads; - for y in min_y..max_y { - let mut sums = i32x4_splat(0i32); - let mut sum_in = i32x4_splat(0i32); - let mut sum_out = i32x4_splat(0i32); + for y in min_y..max_y { + let mut sums = i32x4_splat(0i32); + let mut sum_in = i32x4_splat(0i32); + let mut sum_out = i32x4_splat(0i32); - let mut src_ptr = stride as usize * y; + let mut src_ptr = stride as usize * y; - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - for i in 0..=radius { - let stack_value = stacks.as_mut_ptr().add(i as usize * 4); - v128_store(stack_value as *mut v128, src_pixel); - sums = i32x4_add(sums, i32x4_mul(src_pixel, i32x4_splat(i as i32 + 1))); - sum_out = i32x4_add(sum_out, src_pixel); - } + for i in 0..=radius { + let stack_value = stacks.as_mut_ptr().add(i as usize * 4); + v128_store(stack_value as *mut v128, src_pixel); + sums = i32x4_add(sums, i32x4_mul(src_pixel, i32x4_splat(i as i32 + 1))); + sum_out = i32x4_add(sum_out, src_pixel); + } - for i in 1..=radius { - if i <= wm { - src_ptr += CN; + for i in 1..=radius { + if i <= wm { + src_ptr += CN; + } + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + v128_store(stack_ptr as *mut v128, src_pixel); + sums = i32x4_add( + sums, + i32x4_mul(src_pixel, i32x4_splat(radius as i32 + 1 - i as i32)), + ); + + sum_in = i32x4_add(sum_in, src_pixel); } - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - v128_store(stack_ptr as *mut v128, src_pixel); - sums = i32x4_add( - sums, - i32x4_mul(src_pixel, i32x4_splat(radius as i32 + 1 - i as i32)), - ); - sum_in = i32x4_add(sum_in, src_pixel); - } + sp = radius; + _xp = radius; + if _xp > wm { + _xp = wm; + } - sp = radius; - _xp = radius; - if _xp > wm { - _xp = wm; - } + src_ptr = CN * _xp as usize + y * stride as usize; + let mut dst_ptr = y * stride as usize; + for _ in 0..width { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + let blurred = f32x4_nearest(f32x4_mul(f32x4_convert_i32x4(sums), mul_value)); + let prepared_u16 = u32x4_pack_trunc_u16x8(blurred, blurred); + let blurred = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); - src_ptr = CN * _xp as usize + y * stride as usize; - let mut dst_ptr = y * stride as usize; - for _ in 0..width { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - let blurred = f32x4_ceil(f32x4_mul(f32x4_convert_i32x4(sums), mul_value)); - let prepared_u16 = u32x4_pack_trunc_u16x8(blurred, blurred); - let blurred = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); + w_store_u8x8_m4::(store_ld, blurred); + dst_ptr += CN; - w_store_u8x8_m4::(store_ld, blurred); - dst_ptr += CN; + sums = i32x4_sub(sums, sum_out); - sums = i32x4_sub(sums, sum_out); + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } + let stack = stacks.as_mut_ptr().add(stack_start as usize * 4); - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; - } - let stack = stacks.as_mut_ptr().add(stack_start as usize * 4); + let stack_val = v128_load(stack as *const v128); - let stack_val = v128_load(stack as *const v128); + sum_out = i32x4_sub(sum_out, stack_val); - sum_out = i32x4_sub(sum_out, stack_val); + if _xp < wm { + src_ptr += CN; + _xp += 1; + } - if _xp < wm { - src_ptr += CN; - _xp += 1; - } + let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + v128_store(stack as *mut v128, src_pixel); - let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - v128_store(stack as *mut v128, src_pixel); + sum_in = i32x4_add(sum_in, src_pixel); + sums = i32x4_add(sums, sum_in); - sum_in = i32x4_add(sum_in, src_pixel); - sums = i32x4_add(sums, sum_in); + sp += 1; + if sp >= div { + sp = 0; + } + let stack = stacks.as_mut_ptr().add(sp as usize * 4); + let stack_val = v128_load(stack as *const v128); - sp += 1; - if sp >= div { - sp = 0; + sum_out = i32x4_add(sum_out, stack_val); + sum_in = i32x4_add(sum_in, stack_val); } - let stack = stacks.as_mut_ptr().add(sp as usize * 4); - let stack_val = v128_load(stack as *const v128); - - sum_out = i32x4_add(sum_out, stack_val); - sum_in = i32x4_add(sum_in, stack_val); } } } } -impl StackBlurWorkingPass for HorizontalWasmStackBlurPass -where - J: Copy - + 'static - + AddAssign - + Mul - + Shr - + Sub - + AsPrimitive - + SubAssign - + AsPrimitive - + Default, - T: Copy + AsPrimitive + Default, - i32: AsPrimitive, - u32: AsPrimitive, - f32: PrimitiveCast, - usize: AsPrimitive, -{ +impl StackBlurWorkingPass for HorizontalWasmStackBlurPass { fn pass( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -211,8 +167,6 @@ where thread: usize, total_threads: usize, ) { - unsafe { - self.pass_impl(pixels, stride, width, height, radius, thread, total_threads); - } + self.pass_impl(pixels, stride, width, height, radius, thread, total_threads); } } diff --git a/src/stackblur/wasm/mod.rs b/src/stackblur/wasm/mod.rs index 69d83889..c71fa6ab 100644 --- a/src/stackblur/wasm/mod.rs +++ b/src/stackblur/wasm/mod.rs @@ -29,5 +29,5 @@ mod horizontal; mod vertical; -pub use horizontal::HorizontalWasmStackBlurPass; -pub use vertical::VerticalWasmStackBlurPass; +pub(crate) use horizontal::HorizontalWasmStackBlurPass; +pub(crate) use vertical::VerticalWasmStackBlurPass; diff --git a/src/stackblur/wasm/vertical.rs b/src/stackblur/wasm/vertical.rs index f475b8c1..3c9551eb 100644 --- a/src/stackblur/wasm/vertical.rs +++ b/src/stackblur/wasm/vertical.rs @@ -26,54 +26,26 @@ * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -use crate::primitives::PrimitiveCast; use crate::stackblur::stack_blur_pass::StackBlurWorkingPass; use crate::unsafe_slice::UnsafeSlice; use crate::wasm32::{ load_u8_s32_fast, u16x8_pack_trunc_u8x16, u32x4_pack_trunc_u16x8, w_store_u8x8_m4, }; -use num_traits::{AsPrimitive, FromPrimitive}; use std::arch::wasm32::*; -use std::marker::PhantomData; -use std::ops::{AddAssign, Mul, Shr, Sub, SubAssign}; -pub struct VerticalWasmStackBlurPass { - _phantom_t: PhantomData, - _phantom_j: PhantomData, -} +pub(crate) struct VerticalWasmStackBlurPass {} -impl Default for VerticalWasmStackBlurPass { +impl Default for VerticalWasmStackBlurPass { fn default() -> Self { - VerticalWasmStackBlurPass:: { - _phantom_t: Default::default(), - _phantom_j: Default::default(), - } + VerticalWasmStackBlurPass:: {} } } -impl VerticalWasmStackBlurPass -where - J: Copy - + 'static - + FromPrimitive - + AddAssign - + Mul - + Shr - + Sub - + AsPrimitive - + SubAssign - + AsPrimitive - + Default, - T: Copy + AsPrimitive + Default, - i32: AsPrimitive, - u32: AsPrimitive, - f32: PrimitiveCast, - usize: AsPrimitive, -{ +impl VerticalWasmStackBlurPass { #[target_feature(enable = "simd128")] - unsafe fn pass_impl( + fn pass_impl( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -81,135 +53,118 @@ where thread: usize, total_threads: usize, ) { - let pixels: &UnsafeSlice = std::mem::transmute(pixels); - let div = ((radius * 2) + 1) as usize; - let mut yp; - let mut sp; - let mut stack_start; - let mut stacks = vec![0i32; 4 * div * total_threads]; - - let v_mul_value = f32x4_splat(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - - let hm = height - 1; - let div = (radius * 2) + 1; - - let mut src_ptr; - let mut dst_ptr; - - let min_x = thread * width as usize / total_threads; - let max_x = (thread + 1) * width as usize / total_threads; + unsafe { + let div = ((radius * 2) + 1) as usize; + let mut yp; + let mut sp; + let mut stack_start; + let mut stacks = vec![0i32; 4 * div]; - for x in min_x..max_x { - let mut sums = i32x4_splat(0i32); - let mut sum_in = i32x4_splat(0i32); - let mut sum_out = i32x4_splat(0i32); + let v_mul_value = f32x4_splat(1. / ((radius as f32 + 1.) * (radius as f32 + 1.))); - src_ptr = CN * x; // x,0 + let hm = height - 1; + let div = (radius * 2) + 1; - let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let mut src_ptr; + let mut dst_ptr; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + let min_x = thread * width as usize / total_threads; + let max_x = (thread + 1) * width as usize / total_threads; - for i in 0..=radius { - let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); - v128_store(stack_ptr as *mut v128, src_pixel); - sums = i32x4_add(sums, i32x4_mul(src_pixel, i32x4_splat(i as i32 + 1))); - sum_out = i32x4_add(sum_out, src_pixel); - } + for x in min_x..max_x { + let mut sums = i32x4_splat(0i32); + let mut sum_in = i32x4_splat(0i32); + let mut sum_out = i32x4_splat(0i32); - for i in 1..=radius { - if i <= hm { - src_ptr += stride as usize; - } + src_ptr = CN * x; // x,0 - let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - v128_store(stack_ptr as *mut v128, src_pixel); - sums = i32x4_add( - sums, - i32x4_mul(src_pixel, i32x4_splat(radius as i32 + 1 - i as i32)), - ); - sum_in = i32x4_add(sum_in, src_pixel); - } + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - sp = radius; - yp = radius; - if yp > hm { - yp = hm; - } - src_ptr = CN * x + yp as usize * stride as usize; - dst_ptr = CN * x; - for _ in 0..height { - let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; - let blurred = f32x4_ceil(f32x4_mul(f32x4_convert_i32x4(sums), v_mul_value)); - let prepared_u16 = u32x4_pack_trunc_u16x8(blurred, blurred); - let blurred = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); - w_store_u8x8_m4::(store_ld, blurred); - - dst_ptr += stride as usize; - - sums = i32x4_sub(sums, sum_out); - - stack_start = sp + div - radius; - if stack_start >= div { - stack_start -= div; + for i in 0..=radius { + let stack_ptr = stacks.as_mut_ptr().add(i as usize * 4); + v128_store(stack_ptr as *mut v128, src_pixel); + sums = i32x4_add(sums, i32x4_mul(src_pixel, i32x4_splat(i as i32 + 1))); + sum_out = i32x4_add(sum_out, src_pixel); } - let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); - let stack_val = v128_load(stack_ptr as *const v128); - sum_out = i32x4_sub(sum_out, stack_val); - - if yp < hm { - src_ptr += stride as usize; // stride - yp += 1; + for i in 1..=radius { + if i <= hm { + src_ptr += stride as usize; + } + + let stack_ptr = stacks.as_mut_ptr().add((i + radius) as usize * 4); + let src_ld = pixels.slice.as_ptr().add(src_ptr) as *const i32; + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + v128_store(stack_ptr as *mut v128, src_pixel); + sums = i32x4_add( + sums, + i32x4_mul(src_pixel, i32x4_splat(radius as i32 + 1 - i as i32)), + ); + + sum_in = i32x4_add(sum_in, src_pixel); } - let src_ld = pixels.slice.as_ptr().add(src_ptr); - let src_pixel = load_u8_s32_fast::(src_ld as *const u8); - v128_store(stack_ptr as *mut v128, src_pixel); - - sum_in = i32x4_add(sum_in, src_pixel); - sums = i32x4_add(sums, sum_in); - - sp += 1; - - if sp >= div { - sp = 0; + sp = radius; + yp = radius; + if yp > hm { + yp = hm; + } + src_ptr = CN * x + yp as usize * stride as usize; + dst_ptr = CN * x; + for _ in 0..height { + let store_ld = pixels.slice.as_ptr().add(dst_ptr) as *mut u8; + let blurred = f32x4_nearest(f32x4_mul(f32x4_convert_i32x4(sums), v_mul_value)); + let prepared_u16 = u32x4_pack_trunc_u16x8(blurred, blurred); + let blurred = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); + w_store_u8x8_m4::(store_ld, blurred); + + dst_ptr += stride as usize; + + sums = i32x4_sub(sums, sum_out); + + stack_start = sp + div - radius; + if stack_start >= div { + stack_start -= div; + } + + let stack_ptr = stacks.as_mut_ptr().add(stack_start as usize * 4); + let stack_val = v128_load(stack_ptr as *const v128); + sum_out = i32x4_sub(sum_out, stack_val); + + if yp < hm { + src_ptr += stride as usize; // stride + yp += 1; + } + + let src_ld = pixels.slice.as_ptr().add(src_ptr); + let src_pixel = load_u8_s32_fast::(src_ld as *const u8); + v128_store(stack_ptr as *mut v128, src_pixel); + + sum_in = i32x4_add(sum_in, src_pixel); + sums = i32x4_add(sums, sum_in); + + sp += 1; + + if sp >= div { + sp = 0; + } + let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); + let stack_val = v128_load(stack_ptr as *const v128); + + sum_out = i32x4_add(sum_out, stack_val); + sum_in = i32x4_add(sum_in, stack_val); } - let stack_ptr = stacks.as_mut_ptr().add(sp as usize * 4); - let stack_val = v128_load(stack_ptr as *const v128); - - sum_out = i32x4_add(sum_out, stack_val); - sum_in = i32x4_add(sum_in, stack_val); } } } } -impl StackBlurWorkingPass for VerticalWasmStackBlurPass -where - J: Copy - + 'static - + FromPrimitive - + AddAssign - + Mul - + Shr - + Sub - + AsPrimitive - + SubAssign - + AsPrimitive - + Default, - T: Copy + AsPrimitive + Default, - i32: AsPrimitive, - u32: AsPrimitive, - f32: PrimitiveCast, - usize: AsPrimitive, -{ +impl StackBlurWorkingPass for VerticalWasmStackBlurPass { fn pass( &self, - pixels: &UnsafeSlice, + pixels: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -217,8 +172,6 @@ where thread: usize, total_threads: usize, ) { - unsafe { - self.pass_impl(pixels, stride, width, height, radius, thread, total_threads); - } + self.pass_impl(pixels, stride, width, height, radius, thread, total_threads); } } diff --git a/src/unsafe_slice.rs b/src/unsafe_slice.rs index 09c8d55f..3232541b 100644 --- a/src/unsafe_slice.rs +++ b/src/unsafe_slice.rs @@ -50,7 +50,9 @@ impl<'a, T> UnsafeSlice<'a, T> { #[inline(always)] pub unsafe fn write(&self, i: usize, value: T) { let ptr = unsafe { self.slice.get_unchecked(i) }.get(); - *ptr = value; + unsafe { + *ptr = value; + } } #[allow(dead_code)] #[inline(always)] diff --git a/src/util.rs b/src/util.rs index 2d6d506a..42dfa879 100644 --- a/src/util.rs +++ b/src/util.rs @@ -113,3 +113,86 @@ pub(crate) fn check_slice_size( } Ok(()) } + +pub(crate) enum ScratchBuffer { + Stack([T; N], usize), + Heap(Vec), +} + +impl ScratchBuffer { + pub(crate) fn new(size: usize) -> Self { + if size <= N { + Self::Stack([T::default(); N], size) + } else { + Self::Heap(vec![T::default(); size]) + } + } + + pub(crate) fn as_mut_slice(&mut self) -> &mut [T] { + match self { + Self::Stack(buf, len) => &mut buf[..*len], + Self::Heap(buf) => buf.as_mut_slice(), + } + } + + // pub(crate) fn as_slice(&self) -> &[T] { + // match self { + // Self::Stack(buf, len) => &buf[..*len], + // Self::Heap(buf) => buf.as_slice(), + // } + // } +} + +use std::fmt::Debug; + +pub(crate) enum CowScratch { + Uninit, + Stack([T; N], usize), + Heap(Vec), +} + +impl CowScratch { + pub(crate) const fn new() -> Self { + Self::Uninit + } + + pub(crate) fn is_empty(&self) -> bool { + matches!(self, Self::Uninit) + } + + /// Lazily initialize with a given size, no-op if already initialized + pub(crate) fn get_or_init(&mut self, size: usize) -> &mut [T] { + if self.is_empty() { + *self = if size <= N { + Self::Stack([T::default(); N], size) + } else { + Self::Heap(vec![T::default(); size]) + }; + } + self.borrow_mut() + } + + // pub(crate) fn borrow(&self) -> &[T] { + // match self { + // Self::Uninit => &[], + // Self::Stack(buf, len) => &buf[..*len], + // Self::Heap(v) => v, + // } + // } + + pub(crate) fn borrow_mut(&mut self) -> &mut [T] { + match self { + Self::Uninit => &mut [], + Self::Stack(buf, len) => &mut buf[..*len], + Self::Heap(v) => v, + } + } + + pub(crate) fn get_or_unwrap_mut(&mut self) -> &mut [T] { + match self { + Self::Uninit => panic!("CowScratch is uninitialized"), + Self::Stack(buf, len) => &mut buf[..*len], + Self::Heap(v) => v, + } + } +} diff --git a/src/wasm32/fast_gaussian.rs b/src/wasm32/fast_gaussian.rs index 9a8090ff..463c4b41 100644 --- a/src/wasm32/fast_gaussian.rs +++ b/src/wasm32/fast_gaussian.rs @@ -25,16 +25,16 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::unsafe_slice::UnsafeSlice; use crate::wasm32::utils::{ load_u8_s32_fast, u16x8_pack_trunc_u8x16, u32x4_pack_trunc_u16x8, w_store_u8x8_m4, }; -use crate::EdgeMode; use std::arch::wasm32::*; -pub fn fg_horizontal_pass_wasm_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_horizontal_pass_wasm_u8( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -43,24 +43,15 @@ pub fn fg_horizontal_pass_wasm_u8( end: u32, edge_mode: EdgeMode, ) { - unsafe { - fast_gaussian_horizontal_pass_impl::( - undefined_slice, - stride, - width, - height, - radius, - start, - end, - edge_mode, - ); - } + fast_gaussian_horizontal_pass_impl::( + bytes, stride, width, height, radius, start, end, edge_mode, + ); } #[inline] #[target_feature(enable = "simd128")] -unsafe fn fast_gaussian_horizontal_pass_impl( - undefined_slice: &UnsafeSlice, +fn fast_gaussian_horizontal_pass_impl( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -69,73 +60,74 @@ unsafe fn fast_gaussian_horizontal_pass_impl( end: u32, edge_mode: EdgeMode, ) { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - let mut buffer: [[i32; 4]; 1024] = [[0; 4]; 1024]; - let initial_sum = ((radius * radius) >> 1) as i32; - - let radius_64 = radius as i64; - let width_wide = width as i64; - let v_weight = f32x4_splat((1f64 / (radius as f64 * radius as f64)) as f32); - for y in start..std::cmp::min(height, end) { - let mut diffs = i32x4_splat(0); - let mut summs = i32x4_splat(initial_sum); - - let current_y = ((y as i64) * (stride as i64)) as usize; - - let start_x = 0 - 2 * radius_64; - for x in start_x..(width as i64) { - if x >= 0 { - let current_px = ((x.max(0) as u32) * CN as u32) as usize; - - let prepared_px_s32 = i32x4_trunc_sat_f32x4(f32x4_nearest(f32x4_mul( - f32x4_convert_i32x4(summs), - v_weight, - ))); - let prepared_u16 = u32x4_pack_trunc_u16x8(prepared_px_s32, prepared_px_s32); - let prepared_u8 = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); - - let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - w_store_u8x8_m4::(dst_ptr, prepared_u8); - - let arr_index = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let d_buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let mut d_stored = v128_load(d_buf_ptr as *const v128); - d_stored = i32x4_shr(d_stored, 1); - + unsafe { + let mut buffer: [[i32; 4]; 1024] = [[0; 4]; 1024]; + let initial_sum = ((radius * radius) >> 1) as i32; + + let radius_64 = radius as i64; + let width_wide = width as i64; + let v_weight = f32x4_splat((1f64 / (radius as f64 * radius as f64)) as f32); + for y in start..std::cmp::min(height, end) { + let mut diffs = i32x4_splat(0); + let mut summs = i32x4_splat(initial_sum); + + let current_y = ((y as i64) * (stride as i64)) as usize; + + let start_x = 0 - 2 * radius_64; + for x in start_x..(width as i64) { + if x >= 0 { + let current_px = ((x.max(0) as u32) * CN as u32) as usize; + + let prepared_px_s32 = i32x4_trunc_sat_f32x4(f32x4_nearest(f32x4_mul( + f32x4_convert_i32x4(summs), + v_weight, + ))); + let prepared_u16 = u32x4_pack_trunc_u16x8(prepared_px_s32, prepared_px_s32); + let prepared_u8 = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); + + let bytes_offset = current_y + current_px; + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + w_store_u8x8_m4::(dst_ptr, prepared_u8); + + let arr_index = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; + + let d_buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let mut d_stored = v128_load(d_buf_ptr as *const v128); + d_stored = i32x4_shr(d_stored, 1); + + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let a_stored = v128_load(buf_ptr as *const v128); + + diffs = i32x4_add(diffs, i32x4_sub(a_stored, d_stored)); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let mut stored = v128_load(buf_ptr as *const v128); + stored = i32x4_shr(stored, 1); + diffs = i32x4_sub(diffs, stored); + } + + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; + let pixel_color = load_u8_s32_fast::(s_ptr); + + let arr_index = ((x + radius_64) & 1023) as usize; let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - let a_stored = v128_load(buf_ptr as *const v128); - diffs = i32x4_add(diffs, i32x4_sub(a_stored, d_stored)); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - let mut stored = v128_load(buf_ptr as *const v128); - stored = i32x4_shr(stored, 1); - diffs = i32x4_sub(diffs, stored); + diffs = i32x4_add(diffs, pixel_color); + summs = i32x4_add(summs, diffs); + v128_store(buf_ptr as *mut v128, pixel_color); } - - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + radius_64, 0, width_wide); - let next_row_px = next_row_x * CN; - - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; - let pixel_color = load_u8_s32_fast::(s_ptr); - - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - - diffs = i32x4_add(diffs, pixel_color); - summs = i32x4_add(summs, diffs); - v128_store(buf_ptr as *mut v128, pixel_color); } } } -pub fn fg_vertical_pass_wasm_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fg_vertical_pass_wasm_u8( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -144,24 +136,15 @@ pub fn fg_vertical_pass_wasm_u8( end: u32, edge_mode: EdgeMode, ) { - unsafe { - fast_gaussian_vertical_pass_wasm::( - undefined_slice, - stride, - width, - height, - radius, - start, - end, - edge_mode, - ); - } + fast_gaussian_vertical_pass_wasm::( + bytes, stride, width, height, radius, start, end, edge_mode, + ); } #[inline] #[target_feature(enable = "simd128")] -unsafe fn fast_gaussian_vertical_pass_wasm( - undefined_slice: &UnsafeSlice, +fn fast_gaussian_vertical_pass_wasm( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -170,69 +153,70 @@ unsafe fn fast_gaussian_vertical_pass_wasm( end: u32, edge_mode: EdgeMode, ) { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - let mut buffer: [[i32; 4]; 1024] = [[0; 4]; 1024]; - let initial_sum = ((radius * radius) >> 1) as i32; + unsafe { + let mut buffer: [[i32; 4]; 1024] = [[0; 4]; 1024]; + let initial_sum = ((radius * radius) >> 1) as i32; - let height_wide = height as i64; + let height_wide = height as i64; - let radius_64 = radius as i64; - let v_weight = f32x4_splat((1f64 / (radius as f64 * radius as f64)) as f32); - for x in start..std::cmp::min(width, end) { - let mut diffs = i32x4_splat(0); - let mut summs = i32x4_splat(initial_sum); + let radius_64 = radius as i64; + let v_weight = f32x4_splat((1f64 / (radius as f64 * radius as f64)) as f32); + for x in start..std::cmp::min(width, end) { + let mut diffs = i32x4_splat(0); + let mut summs = i32x4_splat(initial_sum); - let start_y = 0 - 2 * radius as i64; - for y in start_y..height_wide { - let current_y = (y * (stride as i64)) as usize; + let start_y = 0 - 2 * radius as i64; + for y in start_y..height_wide { + let current_y = (y * (stride as i64)) as usize; - if y >= 0 { - let current_px = (x * CN as u32) as usize; + if y >= 0 { + let current_px = (x * CN as u32) as usize; - let prepared_px_s32 = i32x4_trunc_sat_f32x4(f32x4_nearest(f32x4_mul( - f32x4_convert_i32x4(summs), - v_weight, - ))); - let prepared_u16 = u32x4_pack_trunc_u16x8(prepared_px_s32, prepared_px_s32); - let prepared_u8 = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); + let prepared_px_s32 = i32x4_trunc_sat_f32x4(f32x4_nearest(f32x4_mul( + f32x4_convert_i32x4(summs), + v_weight, + ))); + let prepared_u16 = u32x4_pack_trunc_u16x8(prepared_px_s32, prepared_px_s32); + let prepared_u8 = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); - let bytes_offset = current_y + current_px; + let bytes_offset = current_y + current_px; - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - w_store_u8x8_m4::(dst_ptr, prepared_u8); + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + w_store_u8x8_m4::(dst_ptr, prepared_u8); - let arr_index = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; + let arr_index = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let d_buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let mut d_stored = v128_load(d_buf_ptr as *const v128); - d_stored = i32x4_shr(d_stored, 1); + let d_buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let mut d_stored = v128_load(d_buf_ptr as *const v128); + d_stored = i32x4_shr(d_stored, 1); - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - let a_stored = v128_load(buf_ptr as *const v128); + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let a_stored = v128_load(buf_ptr as *const v128); - diffs = i32x4_add(diffs, i32x4_sub(a_stored, d_stored)); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - let mut stored = v128_load(buf_ptr as *const v128); - stored = i32x4_shr(stored, 1); - diffs = i32x4_sub(diffs, stored); - } + diffs = i32x4_add(diffs, i32x4_sub(a_stored, d_stored)); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let mut stored = v128_load(buf_ptr as *const v128); + stored = i32x4_shr(stored, 1); + diffs = i32x4_sub(diffs, stored); + } - let next_row_y = - clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let next_row_y = + clamp_edge!(edge_mode, y + radius_64, 0, height_wide) * (stride as usize); + let next_row_x = (x * CN as u32) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; - let pixel_color = load_u8_s32_fast::(s_ptr); + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; + let pixel_color = load_u8_s32_fast::(s_ptr); - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - diffs = i32x4_add(diffs, pixel_color); - summs = i32x4_add(summs, diffs); - v128_store(buf_ptr as *mut v128, pixel_color); + diffs = i32x4_add(diffs, pixel_color); + summs = i32x4_add(summs, diffs); + v128_store(buf_ptr as *mut v128, pixel_color); + } } } } diff --git a/src/wasm32/fast_gaussian_next.rs b/src/wasm32/fast_gaussian_next.rs index 5d8094b4..c1df51ba 100644 --- a/src/wasm32/fast_gaussian_next.rs +++ b/src/wasm32/fast_gaussian_next.rs @@ -25,17 +25,17 @@ // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +use crate::EdgeMode; use crate::edge_mode::clamp_edge; use crate::unsafe_slice::UnsafeSlice; use crate::wasm32::utils::{ i32x4_mul_by_3, load_u8_s32_fast, u16x8_pack_trunc_u8x16, u32x4_pack_trunc_u16x8, w_store_u8x8_m4, }; -use crate::EdgeMode; use std::arch::wasm32::*; -pub fn fast_gaussian_next_vertical_pass_wasm_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fast_gaussian_next_vertical_pass_wasm_u8( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -44,24 +44,15 @@ pub fn fast_gaussian_next_vertical_pass_wasm_u8( end: u32, edge_mode: EdgeMode, ) { - unsafe { - fast_gaussian_next_vertical_pass_impl::( - undefined_slice, - stride, - width, - height, - radius, - start, - end, - edge_mode, - ); - } + fast_gaussian_next_vertical_pass_impl::( + bytes, stride, width, height, radius, start, end, edge_mode, + ); } #[inline] #[target_feature(enable = "simd128")] -unsafe fn fast_gaussian_next_vertical_pass_impl( - undefined_slice: &UnsafeSlice, +fn fast_gaussian_next_vertical_pass_impl( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -70,92 +61,93 @@ unsafe fn fast_gaussian_next_vertical_pass_impl( end: u32, edge_mode: EdgeMode, ) { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - let mut buffer: [[i32; 4]; 1024] = [[0; 4]; 1024]; - - let height_wide = height as i64; - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let f_weight = f32x4_splat(weight); - for x in start..std::cmp::min(width, end) { - let mut diffs = i32x4_splat(0); - let mut ders = i32x4_splat(0); - let mut summs = i32x4_splat(0); - - let start_y = 0 - 3 * radius as i64; - for y in start_y..height_wide { - let current_y = (y * (stride as i64)) as usize; - - if y >= 0 { - let current_px = ((std::cmp::max(x, 0)) * CN as u32) as usize; - let prepared_px_s32 = i32x4_trunc_sat_f32x4(f32x4_nearest(f32x4_mul( - f32x4_convert_i32x4(summs), - f_weight, - ))); - let prepared_u16 = u32x4_pack_trunc_u16x8(prepared_px_s32, prepared_px_s32); - let prepared_u8 = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); - - let bytes_offset = current_y + current_px; - - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - w_store_u8x8_m4::(dst_ptr, prepared_u8); - - let d_arr_index_1 = ((y + radius_64) & 1023) as usize; - let d_arr_index_2 = ((y - radius_64) & 1023) as usize; - let d_arr_index = (y & 1023) as usize; - - let buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let stored = v128_load(buf_ptr as *const v128); - - let buf_ptr_1 = buffer.as_mut_ptr().add(d_arr_index_1); - let stored_1 = v128_load(buf_ptr_1 as *const v128); - - let buf_ptr_2 = buffer.as_mut_ptr().add(d_arr_index_2); - let stored_2 = v128_load(buf_ptr_2 as *const v128); - - let new_diff = i32x4_sub(i32x4_mul_by_3(i32x4_sub(stored, stored_1)), stored_2); - diffs = i32x4_add(diffs, new_diff); - } else if y + radius_64 >= 0 { - let arr_index = (y & 1023) as usize; - let arr_index_1 = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - let stored = v128_load(buf_ptr as *const v128); + unsafe { + let mut buffer: [[i32; 4]; 1024] = [[0; 4]; 1024]; - let buf_ptr_1 = buffer.get_unchecked_mut(arr_index_1).as_mut_ptr(); - let stored_1 = v128_load(buf_ptr_1 as *const v128); + let height_wide = height as i64; - let new_diff = i32x4_mul_by_3(i32x4_sub(stored, stored_1)); + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let f_weight = f32x4_splat(weight); + for x in start..std::cmp::min(width, end) { + let mut diffs = i32x4_splat(0); + let mut ders = i32x4_splat(0); + let mut summs = i32x4_splat(0); - diffs = i32x4_add(diffs, new_diff); - } else if y + 2 * radius_64 >= 0 { - let arr_index = ((y + radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); - let stored = v128_load(buf_ptr as *const v128); - diffs = i32x4_sub(diffs, i32x4_mul_by_3(stored)); - } + let start_y = 0 - 3 * radius as i64; + for y in start_y..height_wide { + let current_y = (y * (stride as i64)) as usize; + + if y >= 0 { + let current_px = ((std::cmp::max(x, 0)) * CN as u32) as usize; + let prepared_px_s32 = i32x4_trunc_sat_f32x4(f32x4_nearest(f32x4_mul( + f32x4_convert_i32x4(summs), + f_weight, + ))); + let prepared_u16 = u32x4_pack_trunc_u16x8(prepared_px_s32, prepared_px_s32); + let prepared_u8 = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); + + let bytes_offset = current_y + current_px; + + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + w_store_u8x8_m4::(dst_ptr, prepared_u8); - let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) - * (stride as usize); - let next_row_x = (x * CN as u32) as usize; + let d_arr_index_1 = ((y + radius_64) & 1023) as usize; + let d_arr_index_2 = ((y - radius_64) & 1023) as usize; + let d_arr_index = (y & 1023) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; + let buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let stored = v128_load(buf_ptr as *const v128); - let pixel_color = load_u8_s32_fast::(s_ptr); + let buf_ptr_1 = buffer.as_mut_ptr().add(d_arr_index_1); + let stored_1 = v128_load(buf_ptr_1 as *const v128); - let arr_index = ((y + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let buf_ptr_2 = buffer.as_mut_ptr().add(d_arr_index_2); + let stored_2 = v128_load(buf_ptr_2 as *const v128); - diffs = i32x4_add(diffs, pixel_color); - ders = i32x4_add(ders, diffs); - summs = i32x4_add(summs, ders); - v128_store(buf_ptr as *mut v128, pixel_color); + let new_diff = i32x4_sub(i32x4_mul_by_3(i32x4_sub(stored, stored_1)), stored_2); + diffs = i32x4_add(diffs, new_diff); + } else if y + radius_64 >= 0 { + let arr_index = (y & 1023) as usize; + let arr_index_1 = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let stored = v128_load(buf_ptr as *const v128); + + let buf_ptr_1 = buffer.get_unchecked_mut(arr_index_1).as_mut_ptr(); + let stored_1 = v128_load(buf_ptr_1 as *const v128); + + let new_diff = i32x4_mul_by_3(i32x4_sub(stored, stored_1)); + + diffs = i32x4_add(diffs, new_diff); + } else if y + 2 * radius_64 >= 0 { + let arr_index = ((y + radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let stored = v128_load(buf_ptr as *const v128); + diffs = i32x4_sub(diffs, i32x4_mul_by_3(stored)); + } + + let next_row_y = clamp_edge!(edge_mode, y + ((3 * radius_64) >> 1), 0, height_wide) + * (stride as usize); + let next_row_x = (x * CN as u32) as usize; + + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_x) as *mut u8; + + let pixel_color = load_u8_s32_fast::(s_ptr); + + let arr_index = ((y + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + + diffs = i32x4_add(diffs, pixel_color); + ders = i32x4_add(ders, diffs); + summs = i32x4_add(summs, ders); + v128_store(buf_ptr as *mut v128, pixel_color); + } } } } -pub fn fast_gaussian_next_horizontal_pass_wasm_u8( - undefined_slice: &UnsafeSlice, +pub(crate) fn fast_gaussian_next_horizontal_pass_wasm_u8( + slice: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -164,24 +156,15 @@ pub fn fast_gaussian_next_horizontal_pass_wasm_u8( - undefined_slice, - stride, - width, - height, - radius, - start, - end, - edge_mode, - ); - } + fast_gaussian_next_horizontal_pass_impl::( + slice, stride, width, height, radius, start, end, edge_mode, + ); } #[inline] #[target_feature(enable = "simd128")] -unsafe fn fast_gaussian_next_horizontal_pass_impl( - undefined_slice: &UnsafeSlice, +fn fast_gaussian_next_horizontal_pass_impl( + bytes: &UnsafeSlice, stride: u32, width: u32, height: u32, @@ -190,87 +173,88 @@ unsafe fn fast_gaussian_next_horizontal_pass_impl( end: u32, edge_mode: EdgeMode, ) { - let bytes: &UnsafeSlice<'_, u8> = std::mem::transmute(undefined_slice); - let mut buffer: [[i32; 4]; 1024] = [[0; 4]; 1024]; - - let width_wide = width as i64; - - let radius_64 = radius as i64; - let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); - let f_weight = f32x4_splat(weight); - for y in start..std::cmp::min(height, end) { - let mut diffs = i32x4_splat(0); - let mut ders = i32x4_splat(0); - let mut summs = i32x4_splat(0); - - let current_y = ((y as i64) * (stride as i64)) as usize; - - for x in (0 - 3 * radius_64)..(width as i64) { - if x >= 0 { - let current_px = x as usize * CN; - - let prepared_px_s32 = i32x4_trunc_sat_f32x4(f32x4_nearest(f32x4_mul( - f32x4_convert_i32x4(summs), - f_weight, - ))); - let prepared_u16 = u32x4_pack_trunc_u16x8(prepared_px_s32, prepared_px_s32); - let prepared_u8 = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); - - let bytes_offset = current_y + current_px; - - let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); - w_store_u8x8_m4::(dst_ptr, prepared_u8); - - let d_arr_index_1 = ((x + radius_64) & 1023) as usize; - let d_arr_index_2 = ((x - radius_64) & 1023) as usize; - let d_arr_index = (x & 1023) as usize; - - let buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); - let stored = v128_load(buf_ptr as *const v128); - - let buf_ptr_1 = buffer.get_unchecked_mut(d_arr_index_1).as_mut_ptr(); - let stored_1 = v128_load(buf_ptr_1 as *const v128); - - let buf_ptr_2 = buffer.get_unchecked_mut(d_arr_index_2).as_mut_ptr(); - let stored_2 = v128_load(buf_ptr_2 as *const v128); - - let ck = i32x4_sub(stored, stored_1); - let new_diff = i32x4_sub(i32x4_add(i32x4_shl(ck, 1), ck), stored_2); - diffs = i32x4_add(diffs, new_diff); - } else if x + radius_64 >= 0 { - let arr_index = (x & 1023) as usize; - let arr_index_1 = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer.as_mut_ptr().add(arr_index) as *mut i32; - let stored = v128_load(buf_ptr as *const v128); - - let buf_ptr_1 = buffer.as_mut_ptr().add(arr_index_1); - let stored_1 = v128_load(buf_ptr_1 as *const v128); - - let new_diff = i32x4_mul_by_3(i32x4_sub(stored, stored_1)); - - diffs = i32x4_add(diffs, new_diff); - } else if x + 2 * radius_64 >= 0 { - let arr_index = ((x + radius_64) & 1023) as usize; - let buf_ptr = buffer.as_mut_ptr().add(arr_index); - let stored = v128_load(buf_ptr as *const v128); - diffs = i32x4_sub(diffs, i32x4_add(i32x4_shl(stored, 1), stored)); - } + unsafe { + let mut buffer: [[i32; 4]; 1024] = [[0; 4]; 1024]; + + let width_wide = width as i64; + + let radius_64 = radius as i64; + let weight = 1.0f32 / ((radius as f32) * (radius as f32) * (radius as f32)); + let f_weight = f32x4_splat(weight); + for y in start..std::cmp::min(height, end) { + let mut diffs = i32x4_splat(0); + let mut ders = i32x4_splat(0); + let mut summs = i32x4_splat(0); + + let current_y = ((y as i64) * (stride as i64)) as usize; + + for x in (0 - 3 * radius_64)..(width as i64) { + if x >= 0 { + let current_px = x as usize * CN; + + let prepared_px_s32 = i32x4_trunc_sat_f32x4(f32x4_nearest(f32x4_mul( + f32x4_convert_i32x4(summs), + f_weight, + ))); + let prepared_u16 = u32x4_pack_trunc_u16x8(prepared_px_s32, prepared_px_s32); + let prepared_u8 = u16x8_pack_trunc_u8x16(prepared_u16, prepared_u16); + + let bytes_offset = current_y + current_px; + + let dst_ptr = (bytes.slice.as_ptr() as *mut u8).add(bytes_offset); + w_store_u8x8_m4::(dst_ptr, prepared_u8); - let next_row_y = (y as usize) * (stride as usize); - let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); - let next_row_px = next_row_x * CN; + let d_arr_index_1 = ((x + radius_64) & 1023) as usize; + let d_arr_index_2 = ((x - radius_64) & 1023) as usize; + let d_arr_index = (x & 1023) as usize; - let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; + let buf_ptr = buffer.get_unchecked_mut(d_arr_index).as_mut_ptr(); + let stored = v128_load(buf_ptr as *const v128); - let pixel_color = load_u8_s32_fast::(s_ptr); + let buf_ptr_1 = buffer.get_unchecked_mut(d_arr_index_1).as_mut_ptr(); + let stored_1 = v128_load(buf_ptr_1 as *const v128); - let arr_index = ((x + 2 * radius_64) & 1023) as usize; - let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + let buf_ptr_2 = buffer.get_unchecked_mut(d_arr_index_2).as_mut_ptr(); + let stored_2 = v128_load(buf_ptr_2 as *const v128); - diffs = i32x4_add(diffs, pixel_color); - ders = i32x4_add(ders, diffs); - summs = i32x4_add(summs, ders); - v128_store(buf_ptr as *mut v128, pixel_color); + let ck = i32x4_sub(stored, stored_1); + let new_diff = i32x4_sub(i32x4_add(i32x4_shl(ck, 1), ck), stored_2); + diffs = i32x4_add(diffs, new_diff); + } else if x + radius_64 >= 0 { + let arr_index = (x & 1023) as usize; + let arr_index_1 = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer.as_mut_ptr().add(arr_index) as *mut i32; + let stored = v128_load(buf_ptr as *const v128); + + let buf_ptr_1 = buffer.as_mut_ptr().add(arr_index_1); + let stored_1 = v128_load(buf_ptr_1 as *const v128); + + let new_diff = i32x4_mul_by_3(i32x4_sub(stored, stored_1)); + + diffs = i32x4_add(diffs, new_diff); + } else if x + 2 * radius_64 >= 0 { + let arr_index = ((x + radius_64) & 1023) as usize; + let buf_ptr = buffer.as_mut_ptr().add(arr_index); + let stored = v128_load(buf_ptr as *const v128); + diffs = i32x4_sub(diffs, i32x4_add(i32x4_shl(stored, 1), stored)); + } + + let next_row_y = (y as usize) * (stride as usize); + let next_row_x = clamp_edge!(edge_mode, x + 3 * radius_64 / 2, 0, width_wide); + let next_row_px = next_row_x * CN; + + let s_ptr = bytes.slice.as_ptr().add(next_row_y + next_row_px) as *mut u8; + + let pixel_color = load_u8_s32_fast::(s_ptr); + + let arr_index = ((x + 2 * radius_64) & 1023) as usize; + let buf_ptr = buffer.get_unchecked_mut(arr_index).as_mut_ptr(); + + diffs = i32x4_add(diffs, pixel_color); + ders = i32x4_add(ders, diffs); + summs = i32x4_add(summs, ders); + v128_store(buf_ptr as *mut v128, pixel_color); + } } } } diff --git a/src/wasm32/mod.rs b/src/wasm32/mod.rs index 9cb7f267..ebb7c955 100644 --- a/src/wasm32/mod.rs +++ b/src/wasm32/mod.rs @@ -30,8 +30,8 @@ mod fast_gaussian; mod fast_gaussian_next; mod utils; -pub use fast_gaussian::{fg_horizontal_pass_wasm_u8, fg_vertical_pass_wasm_u8}; -pub use fast_gaussian_next::{ +pub(crate) use fast_gaussian::{fg_horizontal_pass_wasm_u8, fg_vertical_pass_wasm_u8}; +pub(crate) use fast_gaussian_next::{ fast_gaussian_next_horizontal_pass_wasm_u8, fast_gaussian_next_vertical_pass_wasm_u8, }; pub use utils::*; diff --git a/src/wasm32/utils.rs b/src/wasm32/utils.rs index 79205e2c..ef5ea859 100644 --- a/src/wasm32/utils.rs +++ b/src/wasm32/utils.rs @@ -31,30 +31,32 @@ use std::arch::wasm32::*; /// Stores up to 4 values from uint8x8_t #[inline] #[target_feature(enable = "simd128")] -pub unsafe fn w_store_u8x8_m4(dst_ptr: *mut u8, in_regi: v128) { - let pixel = i32x4_extract_lane::<0>(in_regi) as u32; +pub unsafe fn w_store_u8x8_m4(dst_ptr: *mut u8, in_regi: v128) { + unsafe { + let pixel = i32x4_extract_lane::<0>(in_regi) as u32; - if CHANNELS_COUNT == 4 { - (dst_ptr as *mut u32).write_unaligned(pixel); - } else if CHANNELS_COUNT == 3 { - let bits = pixel.to_le_bytes(); - let first_byte = u16::from_le_bytes([bits[0], bits[1]]); - (dst_ptr as *mut u16).write_unaligned(first_byte); - dst_ptr.add(2).write_unaligned(bits[2]); - } else if CHANNELS_COUNT == 2 { - let bits = pixel.to_le_bytes(); - let first_byte = u16::from_le_bytes([bits[0], bits[1]]); - (dst_ptr as *mut u16).write_unaligned(first_byte); - } else { - let bits = pixel.to_le_bytes(); - dst_ptr.write_unaligned(bits[0]); + if CN == 4 { + (dst_ptr as *mut u32).write_unaligned(pixel); + } else if CN == 3 { + let bits = pixel.to_le_bytes(); + let first_byte = u16::from_le_bytes([bits[0], bits[1]]); + (dst_ptr as *mut u16).write_unaligned(first_byte); + dst_ptr.add(2).write_unaligned(bits[2]); + } else if CN == 2 { + let bits = pixel.to_le_bytes(); + let first_byte = u16::from_le_bytes([bits[0], bits[1]]); + (dst_ptr as *mut u16).write_unaligned(first_byte); + } else { + let bits = pixel.to_le_bytes(); + dst_ptr.write_unaligned(bits[0]); + } } } /// Packs two u32x4 into one u16x8 using truncation #[inline] #[target_feature(enable = "simd128")] -pub unsafe fn u32x4_pack_trunc_u16x8(a: v128, b: v128) -> v128 { +pub fn u32x4_pack_trunc_u16x8(a: v128, b: v128) -> v128 { u8x16_shuffle::<0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29>(a, b) } @@ -62,7 +64,7 @@ pub unsafe fn u32x4_pack_trunc_u16x8(a: v128, b: v128) -> v128 { #[inline] #[target_feature(enable = "simd128")] #[allow(dead_code)] -pub unsafe fn u16x8_pack_trunc_u8x16(a: v128, b: v128) -> v128 { +pub fn u16x8_pack_trunc_u8x16(a: v128, b: v128) -> v128 { u8x16_shuffle::<0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30>(a, b) } @@ -70,40 +72,42 @@ pub unsafe fn u16x8_pack_trunc_u8x16(a: v128, b: v128) -> v128 { #[inline] #[target_feature(enable = "simd128")] #[allow(dead_code)] -pub unsafe fn i32x4_pack_trunc_i64x2(a: v128, b: v128) -> v128 { +pub fn i32x4_pack_trunc_i64x2(a: v128, b: v128) -> v128 { u8x16_shuffle::<0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27>(a, b) } #[inline] #[target_feature(enable = "simd128")] -pub unsafe fn load_u8_s32_fast(ptr: *const u8) -> v128 { - // LLVM generates a little trash code until opt-level is 3 so better here is to use assembly - if CHANNELS_COUNT == 4 { - let mut undef = i32x4_splat(0); - undef = i32x4_replace_lane::<0>(undef, (ptr as *const i32).read_unaligned()); - let k1 = u16x8_extend_low_u8x16(undef); - u32x4_extend_low_u16x8(k1) - } else if CHANNELS_COUNT == 3 { - let mut undef = i32x4_splat(0); - undef = i16x8_replace_lane::<0>(undef, (ptr as *const i16).read_unaligned()); - undef = u8x16_replace_lane::<2>(undef, ptr.add(2).read_unaligned()); - let k1 = u16x8_extend_low_u8x16(undef); - u32x4_extend_low_u16x8(k1) - } else if CHANNELS_COUNT == 2 { - let mut undef = i32x4_splat(0); - undef = i16x8_replace_lane::<0>(undef, (ptr as *const i16).read_unaligned()); - let k1 = u16x8_extend_low_u8x16(undef); - u32x4_extend_low_u16x8(k1) - } else { - let mut undef = i32x4_splat(0); - undef = u8x16_replace_lane::<0>(undef, ptr.read_unaligned()); - let k1 = u16x8_extend_low_u8x16(undef); - u32x4_extend_low_u16x8(k1) +pub unsafe fn load_u8_s32_fast(ptr: *const u8) -> v128 { + unsafe { + // LLVM generates a little trash code until opt-level is 3 so better here is to use assembly + if CN == 4 { + let mut undef = i32x4_splat(0); + undef = i32x4_replace_lane::<0>(undef, (ptr as *const i32).read_unaligned()); + let k1 = u16x8_extend_low_u8x16(undef); + u32x4_extend_low_u16x8(k1) + } else if CN == 3 { + let mut undef = i32x4_splat(0); + undef = i16x8_replace_lane::<0>(undef, (ptr as *const i16).read_unaligned()); + undef = u8x16_replace_lane::<2>(undef, ptr.add(2).read_unaligned()); + let k1 = u16x8_extend_low_u8x16(undef); + u32x4_extend_low_u16x8(k1) + } else if CN == 2 { + let mut undef = i32x4_splat(0); + undef = i16x8_replace_lane::<0>(undef, (ptr as *const i16).read_unaligned()); + let k1 = u16x8_extend_low_u8x16(undef); + u32x4_extend_low_u16x8(k1) + } else { + let mut undef = i32x4_splat(0); + undef = u8x16_replace_lane::<0>(undef, ptr.read_unaligned()); + let k1 = u16x8_extend_low_u8x16(undef); + u32x4_extend_low_u16x8(k1) + } } } #[inline] #[target_feature(enable = "simd128")] -pub unsafe fn i32x4_mul_by_3(v: v128) -> v128 { +pub fn i32x4_mul_by_3(v: v128) -> v128 { i32x4_add(i32x4_shl(v, 1), v) }