From de968dd48aa33e4020fade1d131101beee61a2fd Mon Sep 17 00:00:00 2001 From: el-j Date: Mon, 2 Nov 2026 07:54:42 +0100 Subject: [PATCH 01/13] feat(codecs): centralize pure-Rust video and audio codecs into craft-libs --- Cargo.lock | 287 + Cargo.toml | 11 + crates/aac/Cargo.toml | 20 + crates/aac/README.md | 191 + crates/aac/src/config.rs | 447 ++ crates/aac/src/decoder.rs | 615 +++ crates/aac/src/encoder.rs | 842 +++ crates/aac/src/encoder/psy.rs | 176 + crates/aac/src/encoder/quant.rs | 510 ++ crates/aac/src/huffman.rs | 425 ++ crates/aac/src/huffman_tables.rs | 1404 +++++ crates/aac/src/ics.rs | 151 + crates/aac/src/lib.rs | 74 + crates/aac/src/mdct.rs | 361 ++ crates/aac/src/tables.rs | 163 + crates/aac/src/tns.rs | 313 ++ crates/aac/tools/derive.py | 128 + crates/aac/tools/gen.py | 29 + crates/aac/tools/probe.py | 78 + crates/audio-io/Cargo.toml | 17 + crates/audio-io/src/aiff.rs | 283 + crates/audio-io/src/buffer.rs | 71 + crates/audio-io/src/caf.rs | 98 + crates/audio-io/src/flac.rs | 563 ++ crates/audio-io/src/lib.rs | 459 ++ crates/audio-io/src/mp4.rs | 106 + crates/audio-io/src/pcm.rs | 279 + crates/audio-io/src/peaks.rs | 280 + crates/audio-io/src/symph.rs | 751 +++ crates/audio-io/src/wav.rs | 574 ++ crates/av1enc/Cargo.toml | 13 + crates/av1enc/README.md | 76 + crates/av1enc/src/bitw.rs | 87 + crates/av1enc/src/cdf.rs | 172 + crates/av1enc/src/decide.rs | 1076 ++++ crates/av1enc/src/lib.rs | 663 +++ crates/av1enc/src/loopfilter.rs | 217 + crates/av1enc/src/pred.rs | 454 ++ crates/av1enc/src/symbol.rs | 267 + crates/av1enc/src/tables.rs | 4836 +++++++++++++++++ crates/av1enc/src/transform.rs | 575 ++ crates/av1enc/src/writer.rs | 1040 ++++ crates/bitstream/Cargo.toml | 14 + crates/bitstream/src/lib.rs | 5 + crates/codecs/Cargo.toml | 31 + crates/codecs/src/lib.rs | 33 + crates/h264enc/Cargo.toml | 20 + crates/h264enc/README.md | 89 + crates/h264enc/src/cabac.rs | 344 ++ crates/h264enc/src/cabac_mb.rs | 446 ++ crates/h264enc/src/cavlc.rs | 288 + crates/h264enc/src/deblock.rs | 263 + crates/h264enc/src/dsp.rs | 255 + crates/h264enc/src/intra.rs | 303 ++ crates/h264enc/src/lib.rs | 1020 ++++ crates/h264enc/src/lookahead.rs | 156 + crates/h264enc/src/mbinfo.rs | 126 + crates/h264enc/src/nal.rs | 337 ++ crates/h264enc/src/picture.rs | 313 ++ crates/h264enc/src/ratecontrol.rs | 322 ++ crates/h264enc/src/slice.rs | 486 ++ crates/h264enc/src/slice/b_slices.rs | 321 ++ crates/h264enc/src/slice/intra.rs | 481 ++ crates/h264enc/src/slice/macroblock.rs | 148 + crates/h264enc/src/slice/motion.rs | 386 ++ crates/h264enc/src/slice/residual.rs | 320 ++ crates/h264enc/src/syntax.rs | 120 + crates/h264enc/src/tables.rs | 1097 ++++ crates/h264enc/src/transform.rs | 523 ++ .../fixtures/h264enc/oracle/avcc.h264 | Bin 0 -> 15264 bytes .../h264enc/oracle/baseline_lowqp_slices.h264 | Bin 0 -> 79356 bytes .../h264enc/oracle/baseline_speed.h264 | Bin 0 -> 26145 bytes .../fixtures/h264enc/oracle/bframes.h264 | Bin 0 -> 14029 bytes .../fixtures/h264enc/oracle/cbr.h264 | Bin 0 -> 569281 bytes .../fixtures/h264enc/oracle/high_crf_b2.h264 | Bin 0 -> 75092 bytes .../oracle/high_quality_b3_slices.h264 | Bin 0 -> 73167 bytes .../h264enc/oracle/high_speed_odd_size.h264 | Bin 0 -> 13984 bytes .../fixtures/h264enc/oracle/main_b2.h264 | Bin 0 -> 23330 bytes .../fixtures/h264enc/oracle/mandelbrot.h264 | Bin 0 -> 84328 bytes .../fixtures/h264enc/oracle/psnr1080.h264 | Bin 0 -> 1337708 bytes .../fixtures/h264enc/oracle/testsrc2.h264 | Bin 0 -> 52896 bytes .../fixtures/h264enc/oracle/twopass.h264 | Bin 0 -> 315083 bytes .../fixtures/h264enc/oracle/vbr1pass.h264 | Bin 0 -> 367892 bytes crates/hevcenc/Cargo.toml | 13 + crates/hevcenc/README.md | 88 + crates/hevcenc/src/bitstream.rs | 135 + crates/hevcenc/src/cabac.rs | 220 + crates/hevcenc/src/deblock.rs | 148 + crates/hevcenc/src/frame.rs | 1489 +++++ crates/hevcenc/src/inter.rs | 157 + crates/hevcenc/src/intra.rs | 254 + crates/hevcenc/src/lib.rs | 357 ++ crates/hevcenc/src/me.rs | 144 + crates/hevcenc/src/params.rs | 262 + crates/hevcenc/src/tables.rs | 236 + crates/hevcenc/src/transform.rs | 328 ++ crates/prores/Cargo.toml | 20 + crates/prores/README.md | 143 + crates/prores/src/bits.rs | 280 + crates/prores/src/dct.rs | 160 + crates/prores/src/decode.rs | 520 ++ crates/prores/src/encode.rs | 772 +++ crates/prores/src/header.rs | 234 + crates/prores/src/lib.rs | 213 + crates/prores/src/tables.rs | 133 + crates/vp9enc/Cargo.toml | 13 + crates/vp9enc/README.md | 47 + crates/vp9enc/src/bool.rs | 259 + crates/vp9enc/src/lf.rs | 312 ++ crates/vp9enc/src/lib.rs | 497 ++ crates/vp9enc/src/mc.rs | 269 + crates/vp9enc/src/probs.rs | 466 ++ crates/vp9enc/src/tables.rs | 208 + crates/vp9enc/src/tile.rs | 940 ++++ crates/vp9enc/src/transform.rs | 270 + 115 files changed, 37016 insertions(+) create mode 100644 crates/aac/Cargo.toml create mode 100644 crates/aac/README.md create mode 100644 crates/aac/src/config.rs create mode 100644 crates/aac/src/decoder.rs create mode 100644 crates/aac/src/encoder.rs create mode 100644 crates/aac/src/encoder/psy.rs create mode 100644 crates/aac/src/encoder/quant.rs create mode 100644 crates/aac/src/huffman.rs create mode 100644 crates/aac/src/huffman_tables.rs create mode 100644 crates/aac/src/ics.rs create mode 100644 crates/aac/src/lib.rs create mode 100644 crates/aac/src/mdct.rs create mode 100644 crates/aac/src/tables.rs create mode 100644 crates/aac/src/tns.rs create mode 100644 crates/aac/tools/derive.py create mode 100644 crates/aac/tools/gen.py create mode 100644 crates/aac/tools/probe.py create mode 100644 crates/audio-io/Cargo.toml create mode 100644 crates/audio-io/src/aiff.rs create mode 100644 crates/audio-io/src/buffer.rs create mode 100644 crates/audio-io/src/caf.rs create mode 100644 crates/audio-io/src/flac.rs create mode 100644 crates/audio-io/src/lib.rs create mode 100644 crates/audio-io/src/mp4.rs create mode 100644 crates/audio-io/src/pcm.rs create mode 100644 crates/audio-io/src/peaks.rs create mode 100644 crates/audio-io/src/symph.rs create mode 100644 crates/audio-io/src/wav.rs create mode 100644 crates/av1enc/Cargo.toml create mode 100644 crates/av1enc/README.md create mode 100644 crates/av1enc/src/bitw.rs create mode 100644 crates/av1enc/src/cdf.rs create mode 100644 crates/av1enc/src/decide.rs create mode 100644 crates/av1enc/src/lib.rs create mode 100644 crates/av1enc/src/loopfilter.rs create mode 100644 crates/av1enc/src/pred.rs create mode 100644 crates/av1enc/src/symbol.rs create mode 100644 crates/av1enc/src/tables.rs create mode 100644 crates/av1enc/src/transform.rs create mode 100644 crates/av1enc/src/writer.rs create mode 100644 crates/bitstream/Cargo.toml create mode 100644 crates/bitstream/src/lib.rs create mode 100644 crates/codecs/Cargo.toml create mode 100644 crates/codecs/src/lib.rs create mode 100644 crates/h264enc/Cargo.toml create mode 100644 crates/h264enc/README.md create mode 100644 crates/h264enc/src/cabac.rs create mode 100644 crates/h264enc/src/cabac_mb.rs create mode 100644 crates/h264enc/src/cavlc.rs create mode 100644 crates/h264enc/src/deblock.rs create mode 100644 crates/h264enc/src/dsp.rs create mode 100644 crates/h264enc/src/intra.rs create mode 100644 crates/h264enc/src/lib.rs create mode 100644 crates/h264enc/src/lookahead.rs create mode 100644 crates/h264enc/src/mbinfo.rs create mode 100644 crates/h264enc/src/nal.rs create mode 100644 crates/h264enc/src/picture.rs create mode 100644 crates/h264enc/src/ratecontrol.rs create mode 100644 crates/h264enc/src/slice.rs create mode 100644 crates/h264enc/src/slice/b_slices.rs create mode 100644 crates/h264enc/src/slice/intra.rs create mode 100644 crates/h264enc/src/slice/macroblock.rs create mode 100644 crates/h264enc/src/slice/motion.rs create mode 100644 crates/h264enc/src/slice/residual.rs create mode 100644 crates/h264enc/src/syntax.rs create mode 100644 crates/h264enc/src/tables.rs create mode 100644 crates/h264enc/src/transform.rs create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/avcc.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/baseline_lowqp_slices.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/baseline_speed.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/bframes.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/cbr.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/high_crf_b2.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/high_quality_b3_slices.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/high_speed_odd_size.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/main_b2.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/mandelbrot.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/psnr1080.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/testsrc2.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/twopass.h264 create mode 100644 crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/vbr1pass.h264 create mode 100644 crates/hevcenc/Cargo.toml create mode 100644 crates/hevcenc/README.md create mode 100644 crates/hevcenc/src/bitstream.rs create mode 100644 crates/hevcenc/src/cabac.rs create mode 100644 crates/hevcenc/src/deblock.rs create mode 100644 crates/hevcenc/src/frame.rs create mode 100644 crates/hevcenc/src/inter.rs create mode 100644 crates/hevcenc/src/intra.rs create mode 100644 crates/hevcenc/src/lib.rs create mode 100644 crates/hevcenc/src/me.rs create mode 100644 crates/hevcenc/src/params.rs create mode 100644 crates/hevcenc/src/tables.rs create mode 100644 crates/hevcenc/src/transform.rs create mode 100644 crates/prores/Cargo.toml create mode 100644 crates/prores/README.md create mode 100644 crates/prores/src/bits.rs create mode 100644 crates/prores/src/dct.rs create mode 100644 crates/prores/src/decode.rs create mode 100644 crates/prores/src/encode.rs create mode 100644 crates/prores/src/header.rs create mode 100644 crates/prores/src/lib.rs create mode 100644 crates/prores/src/tables.rs create mode 100644 crates/vp9enc/Cargo.toml create mode 100644 crates/vp9enc/README.md create mode 100644 crates/vp9enc/src/bool.rs create mode 100644 crates/vp9enc/src/lf.rs create mode 100644 crates/vp9enc/src/lib.rs create mode 100644 crates/vp9enc/src/mc.rs create mode 100644 crates/vp9enc/src/probs.rs create mode 100644 crates/vp9enc/src/tables.rs create mode 100644 crates/vp9enc/src/tile.rs create mode 100644 crates/vp9enc/src/transform.rs diff --git a/Cargo.lock b/Cargo.lock index 4fcd223b56..bac4cc06ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -751,6 +751,15 @@ dependencies = [ "libc", ] +[[package]] +name = "craft-aac" +version = "0.1.0" +dependencies = [ + "craft-bitstream", + "rayon", + "thiserror 2.0.21", +] + [[package]] name = "craft-ai-core" version = "0.1.0" @@ -767,6 +776,27 @@ dependencies = [ "rustix 1.1.5", ] +[[package]] +name = "craft-audio-io" +version = "0.1.0" +dependencies = [ + "log", + "serde", + "symphonia", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-av1enc" +version = "0.1.0" + +[[package]] +name = "craft-bitstream" +version = "0.1.0" +dependencies = [ + "craft-media", +] + [[package]] name = "craft-cms" version = "0.1.0" @@ -775,6 +805,19 @@ dependencies = [ "rayon", ] +[[package]] +name = "craft-codecs" +version = "0.1.0" +dependencies = [ + "craft-aac", + "craft-audio-io", + "craft-av1enc", + "craft-h264enc", + "craft-hevcenc", + "craft-prores", + "craft-vp9enc", +] + [[package]] name = "craft-crw" version = "0.1.0" @@ -862,6 +905,19 @@ dependencies = [ "craft-fonts-build", ] +[[package]] +name = "craft-h264enc" +version = "0.1.0" +dependencies = [ + "craft-bitstream", + "rayon", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-hevcenc" +version = "0.1.0" + [[package]] name = "craft-i18n" version = "0.1.0" @@ -1002,6 +1058,15 @@ dependencies = [ "winsafe", ] +[[package]] +name = "craft-prores" +version = "0.1.0" +dependencies = [ + "craft-bitstream", + "rayon", + "thiserror 2.0.21", +] + [[package]] name = "craft-quit" version = "0.1.0" @@ -1093,6 +1158,10 @@ dependencies = [ "zip", ] +[[package]] +name = "craft-vp9enc" +version = "0.1.0" + [[package]] name = "craft-window" version = "0.1.0" @@ -1674,6 +1743,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fastrand" version = "2.5.0" @@ -3260,6 +3335,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -4355,6 +4439,12 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.11" @@ -5252,6 +5342,203 @@ dependencies = [ "siphasher", ] +[[package]] +name = "symphonia" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7edef6a96b696d4e0cab5ee9ebb7ca155ed95f30a6b45bbb8b97d2727f02424" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-adpcm", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-caf", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6405d5c34ff6f8f7ca08a4101efe66d21e180f7322ef9359900a0e55698a7892" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ea5ffc8716bff677dfb3b01b420c7b758de901a72b8c330bf2040ab74b4add" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5bf8e39552d34a3c4c98333370e62f48c92456d2e814f273b5c3ad7c4a5f45c" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445932ecb0c59362fde9c082dd63a75380650c5077fdb8b80b8cac850442a74c" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "920a78f96f3cf62932d0c497959c0cc2100ef36b03c7ca1417b260f432a482d8" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04ba75686acbe43542fdd374571195f0530c0b7785ca25cc6840e9c6c4b6eea" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73d90b4fcf796137cc683c538282804ff9629f8ad9dbfd881fcbba331ac4e986" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acc3fcc18ec9b8cdd48614e259c4cf0d27b71d41e5d9b120b42c5adab12d7c4" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c412864d599d4750d0c3d684d7e093ec05e5309681ef5252cc1096a437f6e0" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-caf" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab64327ee1920531c5bf86cf7e9cd1a599d57013ddc30691da62683cefc65191" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e681a70e1870d34e02abf1dbc51e4267c3f1827801474e8870be8c689fc4dc3" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d015c5c0558864665894b3f4cbd95e10abb01b9c868e751c72670f326a56360e" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5495e7f7e3c7035328d82b6d6e377eef289bb0c4105bdeb557fc93a833f994" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff70929083a8c1a5f6cd7c904b6071c7914ad04739b510c2f7239dfc9b7dabe" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83713a97705d77bdef7cdbc0768fd6e5a54e4cd7e48d60a806ae85639e2c87c6" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index a98e231bd3..49d7a4ed9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,17 @@ craft-text = { path = "crates/text" } craft-ui = { path = "crates/ui" } craft-update = { path = "crates/update" } craft-window = { path = "crates/window" } +craft-bitstream = { path = "crates/bitstream" } +craft-h264enc = { path = "crates/h264enc" } +craft-prores = { path = "crates/prores" } +craft-hevcenc = { path = "crates/hevcenc" } +craft-av1enc = { path = "crates/av1enc" } +craft-vp9enc = { path = "crates/vp9enc" } +craft-aac = { path = "crates/aac" } +craft-audio-io = { path = "crates/audio-io" } +craft-codecs = { path = "crates/codecs" } +symphonia = { version = "0.6", default-features = false, features = ["mp1", "mp2", "mp3", "flac", "vorbis", "ogg", "wav", "pcm", "aiff", "isomp4", "aac", "alac", "adpcm", "caf", "mkv"] } +log = "0.4" egui = { version = "0.37", default-features = false } rayon = "1" serde = { version = "1", features = ["derive"] } diff --git a/crates/aac/Cargo.toml b/crates/aac/Cargo.toml new file mode 100644 index 0000000000..0f09df76c2 --- /dev/null +++ b/crates/aac/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "craft-aac" +description = "Clean-room pure-Rust AAC-LC encoder and decoder for the Craft ecosystem" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[features] +default = [] +rayon = ["dep:rayon"] + +[dependencies] +filmcraft-bitstream = { package = "craft-bitstream", path = "../bitstream" } +thiserror = { workspace = true } +rayon = { workspace = true, optional = true } diff --git a/crates/aac/README.md b/crates/aac/README.md new file mode 100644 index 0000000000..ffd5bf571c --- /dev/null +++ b/crates/aac/README.md @@ -0,0 +1,191 @@ +# filmcraft-aac + +A clean-room AAC-LC encoder and decoder (ISO/IEC 14496-3, MPEG-4 Audio, Low Complexity profile), +written from the standard and the public psychoacoustics literature. Layer L0: it depends only on +`filmcraft-bitstream` and `thiserror` (plus optional `rayon`), has no `unsafe`, and builds for +`wasm32-unknown-unknown`. + +No FFmpeg/FAAC/FDK-AAC/libfaad or other GPL/LGPL code was read or ported. ffmpeg is used only as an +external **test oracle** (decoding our streams, producing reference streams). + +## API + +```rust +use filmcraft_aac::{Decoder, Encoder, EncoderConfig, BitrateMode, AdtsHeader, split_adts}; + +// Encoder: planar f32 (±1.0) → raw access units +let mut enc = Encoder::new(EncoderConfig::cbr(48_000, 2, 128_000))?; // or EncoderConfig::vbr(48_000, 2, 3.0) +let mut aus: Vec> = enc.encode(&[&left, &right]); // 0..n AUs per call, any chunk size +aus.extend(enc.flush()); // zero-pads and drains the look-ahead +let asc = enc.audio_specific_config(); // bytes for the MP4 `esds` +let delay = enc.priming_samples(); // 1024 → edit list media_time +let stats = enc.stats(); // frames, bytes, bitrate(), short blocks, M/S, TNS + +// Decoder: AudioSpecificConfig + raw AUs → planar f32, 1024 samples per channel per AU +let mut dec = Decoder::new(&asc)?; +let pcm: Vec> = dec.decode(&aus[0])?; + +// ADTS +let frames = split_adts(&adts_bytes)?; // [(AdtsHeader, raw payload)] +let mut dec = Decoder::from_adts(&frames[0].0)?; // channel config 0 → layout from the in-band PCE +``` + +`EncoderConfig` fields: `sample_rate` (the 13 AAC rates, 7.35–96 kHz), `channels` (1–8), `mode` +(`BitrateMode::Cbr(bps)` / `BitrateMode::Vbr(quality 1.0..=5.0)`), `bandwidth` (Hz, `None` = automatic +from bitrate/quality), `tns`, `ms_stereo`, `window_shape` (sine/KBD), `adts` (prefix each AU with a +7-byte ADTS header). + +**Channel order** is AAC element order, for both encoder input and decoder output: + +| channels | configuration | order | +|---|---|---| +| 1 | 1 | C | +| 2 | 2 | L R | +| 3 | 3 | C L R | +| 4 | 4 | C L R Cs | +| 5 | 5 | C L R Ls Rs | +| 6 | 6 | C L R Ls Rs LFE | +| 7 | 0 + PCE | C L R Ls Rs Cs LFE (PCE in the ASC; repeated in every AU when ADTS-framed) | +| 8 | 7 | C Lc Rc L R Ls Rs LFE | + +The decoder also reads configurations 11 (6.1: C L R Ls Rs Cs LFE) and 12 (7.1: C L R Ls Rs Lsr Rsr +LFE) from ISO/IEC 14496-3 Amd 4; ffmpeg 9 writes 7.1 AAC as configuration 12. + +`Decoder::layout()` reports the element layout (it follows a PCE for configuration 0). + +## Encoder + +Per 1024-sample frame and channel element: + +1. **Transient detection**: energy of the first-difference (high-passed) signal in 128-sample + sub-blocks against the mean of the previous eight; one frame of look-ahead selects + `ONLY_LONG` / `LONG_START` / `EIGHT_SHORT` / `LONG_STOP`. Short windows are grouped by energy + similarity (an attack starts a new group). LFE is always long. +2. **MDCT** (2048/256 points) via a DCT-IV on an N/4-point radix-2 complex FFT; sine or KBD + (α = 4 long, 6 short) windows. +3. **Psychoacoustic model** in the MDCT domain: band energies, tonality from the spectral flatness of + a ≥16-line neighbourhood, Schroeder's spreading function in the Bark domain, tonality-dependent + masking offset (30 dB tonal … 6 dB noise), Terhardt threshold in quiet (capped at 40 dB SPL) as a + floor, pre-echo control (threshold ≤ 2× previous frame), perceptual entropy. +4. **TNS** on long windows: Levinson–Durbin (order 8) over the spectrum above 1.4 kHz; applied when the + prediction gain exceeds 1.4, with 4-bit reflection coefficients. +5. **M/S** per band (and group) for channel pairs (always common window): chosen when the estimated + perceptual cost of M and S (with the stricter L/R threshold, halved) is below that of L and R. +6. **Rate–distortion loop**: one multiplier λ scales every band's allowed noise. Each band gets the + coarsest scalefactor whose *measured* quantisation error meets its target (binary search, + `0.4054` rounding), scalefactor differences are kept within ±60, codebooks and sections are chosen + by dynamic programming over all 12 codebook states, and λ is bracketed and bisected until the frame + fits its budget (CBR) or fixed from the quality (VBR). +7. **Bit reservoir** (CBR): frame budgets follow the perceptual entropy (short-window frames get extra), + bounded by the 6144-bit-per-channel decoder buffer; overflow is padded with fill elements and the + final frame drains the reservoir, so the stream is exactly `frames × bitrate × 1024 / rate`. + +Bandwidth: automatic from bitrate per full-bandwidth channel (e.g. 64 kbps/ch → 16 kHz, 96 → 19 kHz, +≥144 → 22 kHz) or from quality in VBR. + +Delay: 1024 priming samples (`priming_samples()`); after `flush()` the decoded stream minus the priming +covers every input sample (`ceil(n / 1024) + 1` access units). + +Performance (Apple Silicon, release, single thread): 10 s of 44.1 kHz stereo encodes at ~85× realtime +and decodes at ~1000× realtime. With the `rayon` feature channels are quantised in parallel inside the +rate loop. + +## Decoder + +Full AAC-LC syntax: SCE/CPE/LFE, DSE, FIL, PCE (reconfigures a configuration-0 stream), all window +sequences and both window shapes (shape of the previous frame for the left half), section data, +scalefactors, pulse data, TNS (all-pole synthesis, long and short), M/S (per band / all bands), +intensity stereo (in/out of phase, M/S-inverted), PNS (with correlated noise for `ms_used` noise bands +as specified), IMDCT and overlap-add. Output is planar f32 at ±1.0 full scale. All reads are bounds +checked: malformed access units return `Err`, never panic. + +## Test results + +All produced by `cargo test -p filmcraft-aac -- --nocapture` (ffmpeg 9.0 at `/opt/homebrew/bin` as +oracle; tests skip when it is absent). Fixtures go to `target/fixtures/aac/`. + +### Encoder quality / bitrate (2 s synthetic signals, decoded by ffmpeg) + +SNR is full-band; bwSNR is measured after low-passing both signals at the coded bandwidth; +segSNR averages 1024-sample segments above −50 dBFS (clamped to −10…60 dB). "us-vs-ff" is the +largest difference between our decoder and ffmpeg's on the same stream. Every stream decodes without +ffmpeg errors, no output clips (peak < 1.0), and CBR is within ±0.1% of target (test limit ±5%). + +| signal | kbps | actual | bw Hz | SNR | bwSNR | segSNR | short frames | us-vs-ff | +|---|---|---|---|---|---|---|---|---| +| sweep 2ch 44.1k | 64 | 64.00 | 11000 | 12.1 | 36.1 | 43.6 | 1 | 5e-7 | +| sweep 2ch 44.1k | 128 | 127.92 | 16000 | 26.1 | 27.3 | 54.2 | 1 | 3e-7 | +| sweep 2ch 44.1k | 320 | 319.90 | 21609 | 46.6 | 46.6 | 59.6 | 1 | 3e-7 | +| white noise 2ch 44.1k | 64 | 64.00 | 11000 | 1.3 | 3.3 | 1.3 | 1 | 5e-7 | +| white noise 2ch 44.1k | 320 | 319.91 | 21609 | 16.4 | 16.5 | 16.6 | 1 | 1.5e-6 | +| speech-like 2ch 48k | 64 | 64.00 | 11000 | 13.6 | 14.0 | 15.2 | 13 | 2e-7 | +| speech-like 2ch 48k | 128 | 127.98 | 16000 | 21.3 | 23.0 | 20.7 | 13 | 2e-7 | +| speech-like 2ch 48k | 320 | 319.96 | 22000 | 31.2 | 31.5 | 31.3 | 13 | 2e-7 | +| chords 2ch 44.1k | 64 | 64.01 | 11000 | 12.4 | 12.4 | 20.6 | 4 | 2e-7 | +| chords 2ch 44.1k | 128 | 128.00 | 16000 | 20.1 | 20.1 | 38.5 | 4 | 2e-7 | +| chords 2ch 44.1k | 320 | 319.99 | 21609 | 35.9 | 35.9 | 55.5 | 4 | 2e-7 | +| clicks 2ch 48k | 64 | 64.00 | 11000 | 10.4 | 11.2 | 48.6 | 11 | 9e-8 | +| clicks 2ch 48k | 128 | 128.00 | 16000 | 20.2 | 25.1 | 53.9 | 11 | 2e-7 | +| clicks 2ch 48k | 320 | 319.95 | 22000 | 39.6 | 39.6 | 58.0 | 11 | 1e-7 | +| speech-like 1ch 44.1k | 64 | 64.00 | 16000 | 18.4 | 19.2 | 18.0 | 15 | 2e-7 | +| chords 5.1 48k | 320 | 320.00 | 16000 | 27.2 | 27.2 | 37.9 | 9 | 2e-7 | +| chords 3/4/5/7/8ch 48k | 192–448 | exact | 16000 | 26–28 | 26–28 | 38 | 7–13 | 2e-7 | +| chords 2ch 8 k … 96 k (10 rates) | 24–256 | exact | 3.9–21 k | 13–38 | | 17–47 | | ≤3e-7 | + +Reference points from ffmpeg's own AAC encoder on the same signals (stereo 48 kHz, segSNR): +sweep 128k 39.7, speech 128k 21.2, chords 128k 32.8, clicks 128k 43.3 (ours: 52/20/39/54). Pure SNR +is not a perceptual measure: the model deliberately leaves noise under the masking threshold, and +full-band white noise is the worst case by design. + +VBR (`Vbr(q)`, 44.1 kHz stereo): chords q1 62 kbps / q3 139 / q5 225 (segSNR 18 / 39 / 51); +speech-like q1 42 / q3 166 / q5 272 kbps (segSNR 13 / 29 / 41). Bitrate and quality grow +monotonically with q. + +### Decoder accuracy vs ffmpeg (streams from `ffmpeg -c:a aac`, PNS off) + +| stream | max abs error | +|---|---| +| stereo 44.1k 128k / 48k 256k / 96k 256k | 1.2e-7 / 1.2e-7 / 1.3e-7 | +| mono 48k 64k / 22.05k 32k / 16k 24k / 8k 16k | 1.5e-7 / 1.2e-7 / 1.2e-7 / 8.9e-8 | +| stereo 44.1k 96k with intensity stereo + M/S + TNS | 1.2e-7 | +| stereo 32k 48k with intensity stereo | 1.2e-7 | +| 3.0, 5.1 (320k), 7.1 wide with PCE (448k) | 1.2e-7, 1.5e-7, 1.3e-7 | +| stereo 44.1k 160k `-aac_coder fast` | 1.2e-7 | + +Our own encoder's streams: ≤1.6e-6 against ffmpeg for every case above. PNS (ffmpeg's encoder does not +emit it, so streams are hand-built): total energy within 0.15 dB of ffmpeg; per-frame energy within +0.8 dB (random vectors differ). For noise bands flagged `ms_used` we produce identical L/R noise as +the standard specifies; ffmpeg produces independent noise there. + +### Robustness + +12 000 mutated access units (bit flips, byte overwrites, truncation, extension, splices, random +buffers) over mono/stereo/5.1/7-channel streams, 20 000 random `AudioSpecificConfig`s and 2 000 random +ADTS streams: no panics. The encoder accepts NaN/±inf/over-range input, any chunk size, and missing +channels. + +## Clean-room notes + +- Syntax, tables (scalefactor bands, TNS limits), filterbank, TNS, PNS, M/S and intensity stereo are + implemented from ISO/IEC 14496-3. +- The Huffman codebooks (`src/huffman_tables.rs`) are the normative values of the standard's Annex + 4.A, recovered by black-box probing of the ffmpeg *binary* as an oracle: `tools/` contains the + Python (stdlib-only) scripts that craft single-frame streams repeating a candidate bit string, fit the + decoded output to the IMDCT basis, and walk the code tree. Every table is verified to be a complete + prefix code (Kraft sum exactly 1) covering each index once (unit test), and the decoder matches + ffmpeg to ~1e-7 on real streams. +- Psychoacoustics follow textbook models (Bark scale, Schroeder spreading, Terhardt ATH, Johnston-style + tonality/SFM offsets, perceptual entropy). + +## Gaps / not implemented + +- HE-AAC v1/v2 (SBR/PS): explicit signalling is parsed and only the AAC-LC core is decoded (half-rate, + no SBR); implicit SBR fill elements are skipped. +- AAC Main (prediction), SSR (gain control), LTP, ER/LD/ELD object types, coupling channel elements + (CCE → `Err(Unsupported)`), 960-sample frames. +- Encoder: no PNS or intensity stereo (decoder supports both), no TNS on short windows, no pulse data; + window shape is fixed per stream (sine or KBD). Short blocks at onsets cost segmental SNR on + speech-like material relative to long windows + TNS; perceptually they avoid pre-echo. +- No error concealment: a corrupt AU returns `Err`; the caller may substitute silence. +- ADTS with CRC is parsed (header skipped, CRC not verified); multi-block ADTS frames are split per + frame only. diff --git a/crates/aac/src/config.rs b/crates/aac/src/config.rs new file mode 100644 index 0000000000..113874f2e0 --- /dev/null +++ b/crates/aac/src/config.rs @@ -0,0 +1,447 @@ +//! `AudioSpecificConfig` (ISO/IEC 14496-3 §1.6.2.1), `program_config_element` (§4.4.1.1) and ADTS +//! framing (§1.A.2.2). + +use filmcraft_bitstream::{BitReader, BitWriter}; + +use crate::tables::{SAMPLE_RATES, table_index_for_rate}; +use crate::{Error, Result}; + +/// Syntactic element types (`id_syn_ele`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ElementType { + /// single_channel_element + Sce, + /// channel_pair_element + Cpe, + /// lfe_channel_element + Lfe, +} + +impl ElementType { + pub fn channels(self) -> usize { + if self == ElementType::Cpe { 2 } else { 1 } + } +} + +/// A `program_config_element` (only the parts that affect channel mapping are interpreted). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ProgramConfig { + pub element_instance_tag: u8, + pub object_type: u8, + pub sf_index: u8, + /// (is_cpe, tag) for front, side and back channel elements. + pub front: Vec<(bool, u8)>, + pub side: Vec<(bool, u8)>, + pub back: Vec<(bool, u8)>, + pub lfe: Vec, + pub assoc_data: Vec, + /// (is_ind_sw, tag) + pub cc: Vec<(bool, u8)>, + pub comment: Vec, +} + +impl ProgramConfig { + pub fn parse(br: &mut BitReader) -> Result { + let mut p = ProgramConfig { + element_instance_tag: br.read_bits(4)? as u8, + object_type: br.read_bits(2)? as u8, + sf_index: br.read_bits(4)? as u8, + ..Default::default() + }; + let nf = br.read_bits(4)?; + let ns = br.read_bits(4)?; + let nb = br.read_bits(4)?; + let nl = br.read_bits(2)?; + let na = br.read_bits(3)?; + let nc = br.read_bits(4)?; + if br.read_bits(1)? == 1 { + br.skip(4)?; + } + if br.read_bits(1)? == 1 { + br.skip(4)?; + } + if br.read_bits(1)? == 1 { + br.skip(3)?; + } + for (n, v) in [(nf, &mut p.front), (ns, &mut p.side), (nb, &mut p.back)] { + for _ in 0..n { + let cpe = br.read_bits(1)? == 1; + v.push((cpe, br.read_bits(4)? as u8)); + } + } + for _ in 0..nl { + p.lfe.push(br.read_bits(4)? as u8); + } + for _ in 0..na { + p.assoc_data.push(br.read_bits(4)? as u8); + } + for _ in 0..nc { + let ind = br.read_bits(1)? == 1; + p.cc.push((ind, br.read_bits(4)? as u8)); + } + br.byte_align(); + let len = br.read_bits(8)?; + for _ in 0..len { + p.comment.push(br.read_bits(8)? as u8); + } + Ok(p) + } + + pub fn write(&self, bw: &mut BitWriter) { + bw.write_bits(self.element_instance_tag as u32, 4); + bw.write_bits(self.object_type as u32, 2); + bw.write_bits(self.sf_index as u32, 4); + bw.write_bits(self.front.len() as u32, 4); + bw.write_bits(self.side.len() as u32, 4); + bw.write_bits(self.back.len() as u32, 4); + bw.write_bits(self.lfe.len() as u32, 2); + bw.write_bits(self.assoc_data.len() as u32, 3); + bw.write_bits(self.cc.len() as u32, 4); + bw.write_bits(0, 3); // no mixdowns + for v in [&self.front, &self.side, &self.back] { + for &(cpe, tag) in v { + bw.write_bits(cpe as u32, 1); + bw.write_bits(tag as u32, 4); + } + } + for &t in &self.lfe { + bw.write_bits(t as u32, 4); + } + for &t in &self.assoc_data { + bw.write_bits(t as u32, 4); + } + for &(ind, t) in &self.cc { + bw.write_bits(ind as u32, 1); + bw.write_bits(t as u32, 4); + } + bw.align_zero(); + bw.write_bits(self.comment.len().min(255) as u32, 8); + for &b in self.comment.iter().take(255) { + bw.write_bits(b as u32, 8); + } + } + + /// Elements in output channel order (front, side, back, LFE). + pub fn layout(&self) -> Vec<(ElementType, u8)> { + let mut out = Vec::new(); + for v in [&self.front, &self.side, &self.back] { + for &(cpe, tag) in v { + out.push((if cpe { ElementType::Cpe } else { ElementType::Sce }, tag)); + } + } + for &t in &self.lfe { + out.push((ElementType::Lfe, t)); + } + out + } +} + +/// Element layout of `channelConfiguration` 1..=7, 11 and 12 (tags as conventionally assigned; 11 and +/// 12 from ISO/IEC 14496-3 Amd 4: 6.1 with a back centre, 7.1 with back surrounds). +pub fn config_layout(channel_config: u8) -> Option> { + use ElementType::*; + Some(match channel_config { + 1 => vec![(Sce, 0)], + 2 => vec![(Cpe, 0)], + 3 => vec![(Sce, 0), (Cpe, 0)], + 4 => vec![(Sce, 0), (Cpe, 0), (Sce, 1)], + 5 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1)], + 6 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Lfe, 0)], + 7 | 12 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Cpe, 2), (Lfe, 0)], + 11 => vec![(Sce, 0), (Cpe, 0), (Cpe, 1), (Sce, 1), (Lfe, 0)], + _ => return None, + }) +} + +/// Parsed `AudioSpecificConfig`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AudioSpecificConfig { + /// Audio object type of the core (2 = AAC LC). + pub object_type: u8, + /// Core sampling rate. + pub sample_rate: u32, + /// Index into the scalefactor-band tables (the signalled index, or derived from an explicit rate). + pub sf_index: u8, + pub channel_config: u8, + pub frame_length_flag: bool, + /// Explicitly signalled SBR/PS (HE-AAC v1/v2); only the AAC-LC core is decoded. + pub sbr: bool, + pub ps: bool, + /// Output (SBR) rate when `sbr` is set. + pub extension_sample_rate: Option, + pub pce: Option, +} + +fn read_aot(br: &mut BitReader) -> Result { + let a = br.read_bits(5)? as u8; + Ok(if a == 31 { 32 + br.read_bits(6)? as u8 } else { a }) +} + +fn read_rate(br: &mut BitReader) -> Result<(u8, u32)> { + let i = br.read_bits(4)? as u8; + if i == 15 { + let r = br.read_bits(24)?; + return Ok((table_index_for_rate(r), r)); + } + let r = *SAMPLE_RATES.get(i as usize).ok_or(Error::InvalidConfig("reserved sampling frequency index"))?; + Ok((i, r)) +} + +impl AudioSpecificConfig { + /// A plain AAC-LC configuration. + pub fn lc(sf_index: u8, channel_config: u8) -> AudioSpecificConfig { + AudioSpecificConfig { + object_type: 2, + sample_rate: SAMPLE_RATES[sf_index as usize], + sf_index, + channel_config, + frame_length_flag: false, + sbr: false, + ps: false, + extension_sample_rate: None, + pce: None, + } + } + + pub fn parse(data: &[u8]) -> Result { + let mut br = BitReader::new(data); + let mut aot = read_aot(&mut br)?; + let (sf_index, sample_rate) = read_rate(&mut br)?; + let channel_config = br.read_bits(4)? as u8; + let (mut sbr, mut ps, mut extension_sample_rate) = (false, false, None); + if aot == 5 || aot == 29 { + sbr = true; + ps = aot == 29; + let (_, ext) = read_rate(&mut br)?; + extension_sample_rate = Some(ext); + aot = read_aot(&mut br)?; + } + if aot != 2 { + return Err(Error::Unsupported(match aot { + 1 => "AAC Main profile", + 3 => "AAC SSR profile", + 4 => "AAC LTP profile", + 23 => "AAC-LD", + 39 => "AAC-ELD", + _ => "audio object type other than AAC LC", + })); + } + let frame_length_flag = br.read_bits(1)? == 1; + if br.read_bits(1)? == 1 { + br.skip(14)?; // coreCoderDelay + } + let _extension_flag = br.read_bits(1)?; + let pce = if channel_config == 0 { Some(ProgramConfig::parse(&mut br)?) } else { None }; + if frame_length_flag { + return Err(Error::Unsupported("960-sample frames")); + } + let cfg = + AudioSpecificConfig { object_type: aot, sample_rate, sf_index, channel_config, frame_length_flag, sbr, ps, extension_sample_rate, pce }; + if cfg.channels() == 0 { + return Err(Error::InvalidConfig("no channels")); + } + Ok(cfg) + } + + /// Serialise (AAC-LC, no SBR signalling). + pub fn to_bytes(&self) -> Vec { + let mut bw = BitWriter::new(); + bw.write_bits(self.object_type as u32, 5); + match crate::tables::sample_rate_index(self.sample_rate) { + Some(i) => bw.write_bits(i as u32, 4), + None => { + bw.write_bits(15, 4); + bw.write_bits(self.sample_rate, 24); + } + } + bw.write_bits(self.channel_config as u32, 4); + bw.write_bits(0, 3); // frameLengthFlag, dependsOnCoreCoder, extensionFlag + if self.channel_config == 0 + && let Some(p) = &self.pce + { + p.write(&mut bw); + } + bw.finish() + } + + /// Elements in output channel order. + pub fn layout(&self) -> Vec<(ElementType, u8)> { + match &self.pce { + Some(p) if self.channel_config == 0 => p.layout(), + _ => config_layout(self.channel_config).unwrap_or_default(), + } + } + + pub fn channels(&self) -> usize { + self.layout().iter().map(|e| e.0.channels()).sum() + } +} + +/// A parsed ADTS header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct AdtsHeader { + /// Audio object type (`profile + 1`). + pub object_type: u8, + pub sf_index: u8, + pub channel_config: u8, + /// Whole frame length including the header. + pub frame_length: usize, + pub buffer_fullness: u16, + /// Header length (7, or 9 with CRC). + pub header_length: usize, + pub raw_data_blocks: u8, +} + +impl AdtsHeader { + pub fn parse(data: &[u8]) -> Result { + let mut br = BitReader::new(data); + if br.read_bits(12)? != 0xFFF { + return Err(Error::Bitstream("missing ADTS syncword")); + } + br.skip(1)?; // ID + if br.read_bits(2)? != 0 { + return Err(Error::Bitstream("ADTS layer must be 0")); + } + let protection_absent = br.read_bits(1)? == 1; + let object_type = br.read_bits(2)? as u8 + 1; + let sf_index = br.read_bits(4)? as u8; + if sf_index as usize >= SAMPLE_RATES.len() { + return Err(Error::Bitstream("invalid ADTS sampling frequency index")); + } + br.skip(1)?; + let channel_config = br.read_bits(3)? as u8; + br.skip(4)?; + let frame_length = br.read_bits(13)? as usize; + let buffer_fullness = br.read_bits(11)? as u16; + let raw_data_blocks = br.read_bits(2)? as u8 + 1; + let header_length = if protection_absent { 7 } else { 9 }; + if frame_length < header_length { + return Err(Error::Bitstream("ADTS frame shorter than its header")); + } + Ok(AdtsHeader { object_type, sf_index, channel_config, frame_length, buffer_fullness, header_length, raw_data_blocks }) + } + + /// The equivalent `AudioSpecificConfig`. Channel configuration 0 (PCE in-band) is resolved by the + /// decoder from the first frame; this returns stereo-less layout in that case. + pub fn to_asc(&self) -> AudioSpecificConfig { + AudioSpecificConfig { object_type: self.object_type, ..AudioSpecificConfig::lc(self.sf_index, self.channel_config) } + } + + /// Write a 7-byte header (no CRC) for a frame whose raw payload is `payload_len` bytes. + pub fn write(object_type: u8, sf_index: u8, channel_config: u8, payload_len: usize, buffer_fullness: u16) -> [u8; 7] { + let len = (payload_len + 7) as u32; + let mut bw = BitWriter::new(); + bw.write_bits(0xFFF, 12); + bw.write_bits(0, 1); // MPEG-4 + bw.write_bits(0, 2); + bw.write_bits(1, 1); // protection absent + bw.write_bits((object_type.max(1) - 1) as u32 & 3, 2); + bw.write_bits(sf_index as u32, 4); + bw.write_bits(0, 1); + bw.write_bits(channel_config as u32 & 7, 3); + bw.write_bits(0, 4); + bw.write_bits(len & 0x1FFF, 13); + bw.write_bits(buffer_fullness as u32 & 0x7FF, 11); + bw.write_bits(0, 2); + let v = bw.finish(); + let mut out = [0u8; 7]; + out.copy_from_slice(&v); + out + } +} + +/// Split an ADTS stream into `(header, raw payload)` frames. Stops at the first malformed or +/// truncated frame (returning what was parsed so far); errors only if nothing parses. +pub fn split_adts(mut data: &[u8]) -> Result> { + let mut out = Vec::new(); + while data.len() >= 7 { + let h = match AdtsHeader::parse(data) { + Ok(h) => h, + Err(e) => { + if out.is_empty() { + return Err(e); + } + break; + } + }; + if h.frame_length > data.len() { + break; + } + out.push((h, &data[h.header_length..h.frame_length])); + data = &data[h.frame_length..]; + } + if out.is_empty() { + return Err(Error::Bitstream("no ADTS frames")); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn asc_roundtrip() { + let a = AudioSpecificConfig::lc(4, 2); + let b = a.to_bytes(); + assert_eq!(b, vec![0x12, 0x10]); + assert_eq!(AudioSpecificConfig::parse(&b).unwrap(), a); + // 48 kHz 5.1 + let a = AudioSpecificConfig::lc(3, 6); + assert_eq!(AudioSpecificConfig::parse(&a.to_bytes()).unwrap().channels(), 6); + } + + #[test] + fn amd4_channel_configurations() { + // 6.1 and 7.1 back (#922); 22.2 and 7.1 with heights stay unsupported + let channels = |c| AudioSpecificConfig::parse(&AudioSpecificConfig::lc(3, c).to_bytes()).map(|a| a.channels()).ok(); + assert_eq!([7, 11, 12].map(channels), [Some(8), Some(7), Some(8)]); + assert_eq!([13, 14].map(config_layout), [None, None]); + } + + #[test] + fn asc_with_pce() { + let pce = ProgramConfig { + sf_index: 3, + object_type: 1, + front: vec![(false, 0), (true, 0)], + side: vec![(true, 1)], + back: vec![(false, 1)], + lfe: vec![0], + ..Default::default() + }; + let a = AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) }; + let p = AudioSpecificConfig::parse(&a.to_bytes()).unwrap(); + assert_eq!(p.channels(), 7); + assert_eq!(p, a); + } + + #[test] + fn he_aac_asc_parses_core() { + // AOT 5, 24 kHz core, stereo, ext 48 kHz, core AOT 2 + let mut bw = BitWriter::new(); + bw.write_bits(5, 5); + bw.write_bits(6, 4); + bw.write_bits(2, 4); + bw.write_bits(3, 4); + bw.write_bits(2, 5); + bw.write_bits(0, 3); + let a = AudioSpecificConfig::parse(&bw.finish()).unwrap(); + assert!(a.sbr); + assert_eq!(a.sample_rate, 24000); + assert_eq!(a.extension_sample_rate, Some(48000)); + } + + #[test] + fn adts_roundtrip() { + let h = AdtsHeader::write(2, 4, 2, 100, 0x7FF); + let mut frame = h.to_vec(); + frame.extend(std::iter::repeat_n(0u8, 100)); + let p = AdtsHeader::parse(&frame).unwrap(); + assert_eq!(p.frame_length, 107); + assert_eq!((p.object_type, p.sf_index, p.channel_config), (2, 4, 2)); + let frames = split_adts(&frame).unwrap(); + assert_eq!(frames.len(), 1); + assert_eq!(frames[0].1.len(), 100); + } +} diff --git a/crates/aac/src/decoder.rs b/crates/aac/src/decoder.rs new file mode 100644 index 0000000000..4c56d31276 --- /dev/null +++ b/crates/aac/src/decoder.rs @@ -0,0 +1,615 @@ +//! AAC-LC decoder: raw access units (`raw_data_block`, ISO/IEC 14496-3 §4.4.2.1) → planar f32. + +use filmcraft_bitstream::BitReader; + +use crate::config::{AudioSpecificConfig, ElementType, ProgramConfig}; +use crate::huffman::{self, INTENSITY_HCB, INTENSITY_HCB2, NOISE_HCB, ZERO_HCB}; +use crate::ics::{IcsInfo, WindowSequence}; +use crate::mdct::{MdctScratch, WindowShape, mdct_long, mdct_short, window}; +use crate::tables::{pow43, pow43_table, sf_gain}; +use crate::tns::{self, TnsData}; +use crate::{Error, Result}; + +const ID_SCE: u32 = 0; +const ID_CPE: u32 = 1; +const ID_CCE: u32 = 2; +const ID_LFE: u32 = 3; +const ID_DSE: u32 = 4; +const ID_PCE: u32 = 5; +const ID_FIL: u32 = 6; +const ID_END: u32 = 7; + +/// `extension_type` of a fill element carrying SBR data (ISO/IEC 14496-3 §4.5.2.8.2, Table 4.121). +const EXT_SBR_DATA: u32 = 13; +const EXT_SBR_DATA_CRC: u32 = 14; + +/// One decoded `individual_channel_stream` before stereo processing and synthesis. +struct Ics { + info: IcsInfo, + /// Band type per (group, sfb). + band_type: [[u8; 64]; 8], + /// Scalefactor / noise energy / intensity position per (group, sfb). + sf: [[i32; 64]; 8], + tns: Option, + /// Dequantised spectrum (short windows at stride 128). + spec: Vec, +} + +#[derive(Clone)] +struct ChannelState { + overlap: Vec, + prev_shape: WindowShape, +} + +/// Buffers reused by every frame so the steady state does not allocate per channel. +struct Scratch { + /// Quantised values of the channel being parsed. + q: Vec, + /// Windowed output of the current block (2048) and the raw inverse transform (2048 / 256). + buf: Vec, + y: Vec, + long: MdctScratch, + short: MdctScratch, +} + +impl Scratch { + fn new() -> Scratch { + Scratch { q: vec![0; 1024], buf: vec![0.0; 2048], y: vec![0.0; 2048], long: mdct_long().scratch(), short: mdct_short().scratch() } + } +} + +/// AAC-LC decoder. +pub struct Decoder { + asc: AudioSpecificConfig, + layout: Vec<(ElementType, u8)>, + /// First output channel of each layout entry. + offsets: Vec, + channels: Vec, + noise_state: u32, + scratch: Scratch, + /// A fill element carried SBR data (implicitly signalled HE-AAC). + sbr_seen: bool, +} + +impl Decoder { + /// Create a decoder from an `AudioSpecificConfig`. + pub fn new(asc: &[u8]) -> Result { + Self::from_config(AudioSpecificConfig::parse(asc)?) + } + + /// Create a decoder for an ADTS stream from its first header. With channel configuration 0 the + /// channel layout is taken from the program config element carried in the first frame. + pub fn from_adts(header: &crate::config::AdtsHeader) -> Result { + Self::from_config(header.to_asc()) + } + + pub fn from_config(asc: AudioSpecificConfig) -> Result { + if asc.object_type != 2 { + return Err(Error::Unsupported("audio object type other than AAC LC")); + } + let layout = asc.layout(); + if layout.is_empty() && asc.channel_config != 0 { + return Err(Error::InvalidConfig("unsupported channel configuration")); + } + let mut d = Decoder { + asc, + layout: Vec::new(), + offsets: Vec::new(), + channels: Vec::new(), + noise_state: 0x2545_F491, + scratch: Scratch::new(), + sbr_seen: false, + }; + d.set_layout(layout); + Ok(d) + } + + fn set_layout(&mut self, layout: Vec<(ElementType, u8)>) { + let mut offsets = Vec::with_capacity(layout.len()); + let mut n = 0; + for e in &layout { + offsets.push(n); + n += e.0.channels(); + } + self.layout = layout; + self.offsets = offsets; + self.channels = vec![ChannelState { overlap: vec![0.0; 1024], prev_shape: WindowShape::Sine }; n]; + } + + pub fn config(&self) -> &AudioSpecificConfig { + &self.asc + } + pub fn sample_rate(&self) -> u32 { + self.asc.sample_rate + } + /// Whether the stream is HE-AAC: SBR signalled in the `AudioSpecificConfig` (explicit) or SBR + /// data seen in a fill element of a decoded access unit (implicit). Only the AAC-LC core is + /// decoded, at [`Self::sample_rate`]; the SBR high band is not reconstructed. + pub fn sbr(&self) -> bool { + self.asc.sbr || self.sbr_seen + } + pub fn channels(&self) -> usize { + self.channels.len() + } + /// Element layout in output channel order (C, L/R, surround pairs, LFE for configurations 1–7). + pub fn layout(&self) -> &[(ElementType, u8)] { + &self.layout + } + + /// Clear the overlap state (e.g. after a seek). + pub fn reset(&mut self) { + for c in &mut self.channels { + c.overlap.fill(0.0); + c.prev_shape = WindowShape::Sine; + } + } + + fn sf_index(&self) -> u8 { + self.asc.sf_index + } + + /// Decode one access unit (one `raw_data_block`) into 1024 samples per channel. + pub fn decode(&mut self, au: &[u8]) -> Result>> { + self.decode_block(au, true) + } + + fn decode_block(&mut self, au: &[u8], allow_reconfig: bool) -> Result>> { + let mut br = BitReader::new(au); + let nch = self.channels.len(); + let mut saw_element = false; + let mut out: Vec>> = vec![None; nch]; + let mut used = vec![false; self.layout.len()]; + loop { + let id = br.read_bits(3)?; + saw_element |= matches!(id, ID_SCE | ID_CPE | ID_LFE); + match id { + ID_SCE | ID_LFE => { + let tag = br.read_bits(4)? as u8; + let ty = if id == ID_SCE { ElementType::Sce } else { ElementType::Lfe }; + let slot = self.find_slot(ty, tag, &used)?; + used[slot] = true; + let mut ics = self.decode_ics(&mut br, None)?; + let ch = self.offsets[slot]; + self.fill_noise(&mut ics, None); + out[ch] = Some(self.finish_channel(ch, &mut ics)); + } + ID_CPE => { + let tag = br.read_bits(4)? as u8; + let slot = self.find_slot(ElementType::Cpe, tag, &used)?; + used[slot] = true; + let ch = self.offsets[slot]; + let [a, b] = self.decode_cpe(&mut br, ch)?; + out[ch] = Some(a); + out[ch + 1] = Some(b); + } + ID_CCE => return Err(Error::Unsupported("coupling channel element")), + ID_DSE => { + br.skip(4)?; + let align = br.read_bits(1)? == 1; + let mut count = br.read_bits(8)? as usize; + if count == 255 { + count += br.read_bits(8)? as usize; + } + if align { + br.byte_align(); + } + br.skip(count * 8)?; + } + ID_PCE => { + let pce = ProgramConfig::parse(&mut br)?; + if allow_reconfig && !saw_element && self.asc.channel_config == 0 && self.asc.pce.as_ref() != Some(&pce) { + let layout = pce.layout(); + if !layout.is_empty() && layout.iter().map(|e| e.0.channels()).sum::() <= 64 { + self.asc.pce = Some(pce); + self.set_layout(layout); + return self.decode_block(au, false); + } + } + } + ID_FIL => { + let mut count = br.read_bits(4)? as usize; + if count == 15 { + count += br.read_bits(8)? as usize; + count -= 1; + } + if count > 0 { + let ext = br.read_bits(4)?; + self.sbr_seen |= matches!(ext, EXT_SBR_DATA | EXT_SBR_DATA_CRC); + br.skip(count * 8 - 4)?; + } + } + _ => break, // ID_END + } + if id == ID_END { + break; + } + } + if nch == 0 && !saw_element { + return Err(Error::InvalidConfig("channel configuration 0 without a program config element")); + } + let mut res = Vec::with_capacity(nch); + for o in out { + res.push(o.ok_or(Error::Bitstream("access unit is missing a channel element"))?); + } + Ok(res) + } + + fn find_slot(&self, ty: ElementType, tag: u8, used: &[bool]) -> Result { + let compatible = |t: ElementType| t == ty || (ty != ElementType::Cpe && t != ElementType::Cpe); + if let Some(i) = self.layout.iter().enumerate().position(|(i, &(t, g))| t == ty && g == tag && !used[i]) { + return Ok(i); + } + if let Some(i) = self.layout.iter().enumerate().position(|(i, &(t, _))| t == ty && !used[i]) { + return Ok(i); + } + self.layout + .iter() + .enumerate() + .position(|(i, &(t, _))| compatible(t) && !used[i]) + .ok_or(Error::Bitstream("unexpected channel element for the channel configuration")) + } + + fn decode_cpe(&mut self, br: &mut BitReader, ch: usize) -> Result<[Vec; 2]> { + let common_window = br.read_bits(1)? == 1; + let mut ms_mask_present = 0; + let mut ms_used = [[false; 64]; 8]; + let mut info = None; + if common_window { + let i = IcsInfo::parse(br, self.sf_index())?; + ms_mask_present = br.read_bits(2)?; + if ms_mask_present == 3 { + return Err(Error::Bitstream("reserved ms_mask_present")); + } + if ms_mask_present == 1 { + for g in 0..i.num_groups { + for sfb in 0..i.max_sfb { + ms_used[g][sfb] = br.read_bits(1)? == 1; + } + } + } else if ms_mask_present == 2 { + ms_used = [[true; 64]; 8]; + } + info = Some(i); + } + let mut l = self.decode_ics(br, info)?; + let mut r = self.decode_ics(br, info)?; + let ms = if ms_mask_present > 0 { Some(&ms_used) } else { None }; + self.fill_noise(&mut l, None); + self.fill_noise(&mut r, if common_window { Some((&l, ms)) } else { None }); + if common_window { + let swb = l.info.swb_offsets(self.sf_index()); + for g in 0..l.info.num_groups { + let w0 = l.info.group_start(g); + for sfb in 0..l.info.max_sfb { + let (bl, br_) = (l.band_type[g][sfb], r.band_type[g][sfb]); + let range = swb[sfb] as usize..swb[sfb + 1] as usize; + if br_ == INTENSITY_HCB || br_ == INTENSITY_HCB2 { + let mut c: f32 = if br_ == INTENSITY_HCB { 1.0 } else { -1.0 }; + if ms_mask_present > 0 && ms_used[g][sfb] { + c = -c; + } + let scale = c * (-0.25 * r.sf[g][sfb].clamp(-400, 400) as f64).exp2() as f32; + for w in w0..w0 + l.info.group_len[g] as usize { + for k in range.clone() { + r.spec[w * 128 + k] = l.spec[w * 128 + k] * scale; + } + } + } else if ms_mask_present > 0 && ms_used[g][sfb] && bl < NOISE_HCB && br_ < NOISE_HCB { + for w in w0..w0 + l.info.group_len[g] as usize { + for k in range.clone() { + let (m, s) = (l.spec[w * 128 + k], r.spec[w * 128 + k]); + l.spec[w * 128 + k] = m + s; + r.spec[w * 128 + k] = m - s; + } + } + } + } + } + } + Ok([self.finish_channel(ch, &mut l), self.finish_channel(ch + 1, &mut r)]) + } + + fn decode_ics(&mut self, br: &mut BitReader, common: Option) -> Result { + let sfi = self.sf_index(); + let global_gain = br.read_bits(8)? as i32; + let info = match common { + Some(i) => i, + None => IcsInfo::parse(br, sfi)?, + }; + let swb = info.swb_offsets(sfi); + let num_swb = swb.len() - 1; + let mut ics = Ics { info, band_type: [[0; 64]; 8], sf: [[0; 64]; 8], tns: None, spec: vec![0.0; 1024] }; + + // section_data + let (sect_bits, esc) = if info.is_short() { (3, 7) } else { (5, 31) }; + for g in 0..info.num_groups { + let mut k = 0; + while k < info.max_sfb { + let cb = br.read_bits(4)? as u8; + if cb == 12 { + return Err(Error::Bitstream("reserved codebook 12")); + } + let mut len = 0usize; + loop { + let inc = br.read_bits(sect_bits)?; + len += inc as usize; + if inc != esc { + break; + } + } + if k + len > info.max_sfb { + return Err(Error::Bitstream("section exceeds max_sfb")); + } + if len == 0 { + // zero-length sections are legal but useless; guard against endless loops + if br.bits_left() == 0 { + return Err(Error::Eof); + } + } + for sfb in k..k + len { + ics.band_type[g][sfb] = cb; + } + k += len; + } + } + + // scale_factor_data + let mut sf = global_gain; + let mut noise = global_gain - 90; + let mut is_pos = 0i32; + let mut noise_first = true; + for g in 0..info.num_groups { + for sfb in 0..info.max_sfb { + match ics.band_type[g][sfb] { + ZERO_HCB => ics.sf[g][sfb] = 0, + INTENSITY_HCB | INTENSITY_HCB2 => { + is_pos += huffman::decode_sf(br)?; + ics.sf[g][sfb] = is_pos; + } + NOISE_HCB => { + if noise_first { + noise += br.read_bits(9)? as i32 - 256; + noise_first = false; + } else { + noise += huffman::decode_sf(br)?; + } + ics.sf[g][sfb] = noise; + } + _ => { + sf += huffman::decode_sf(br)?; + if !(0..=255).contains(&sf) { + return Err(Error::Bitstream("scalefactor out of range")); + } + ics.sf[g][sfb] = sf; + } + } + } + } + + // pulse_data + let mut pulses: Vec<(usize, i32)> = Vec::new(); + if br.read_bits(1)? == 1 { + if info.is_short() { + return Err(Error::Bitstream("pulse data in short window")); + } + let n = br.read_bits(2)? as usize + 1; + let start = br.read_bits(6)? as usize; + if start >= num_swb { + return Err(Error::Bitstream("pulse_start_sfb out of range")); + } + let mut k = swb[start] as usize; + for _ in 0..n { + k += br.read_bits(5)? as usize; + let amp = br.read_bits(4)? as i32; + if k >= 1024 { + return Err(Error::Bitstream("pulse position out of range")); + } + pulses.push((k, amp)); + } + } + if br.read_bits(1)? == 1 { + ics.tns = Some(TnsData::parse(br, &info)?); + } + if br.read_bits(1)? == 1 { + return Err(Error::Unsupported("gain control data (AAC SSR)")); + } + + // spectral_data + let mut q = std::mem::take(&mut self.scratch.q); + q.clear(); + q.resize(1024, 0); + let mut vals = [0i32; 4]; + for g in 0..info.num_groups { + let w0 = info.group_start(g); + let glen = info.group_len[g] as usize; + for sfb in 0..info.max_sfb { + let cb = ics.band_type[g][sfb]; + if cb == ZERO_HCB || cb >= NOISE_HCB { + continue; + } + let dim = huffman::book(cb).dim; + let (lo, hi) = (swb[sfb] as usize, swb[sfb + 1] as usize); + for w in w0..w0 + glen { + let base = w * 128; + let mut k = lo; + while k < hi { + huffman::decode_spectral(br, cb, &mut vals)?; + q[base + k..base + k + dim].copy_from_slice(&vals[..dim]); + k += dim; + } + } + } + } + for (k, amp) in pulses { + if q[k] > 0 { + q[k] += amp; + } else { + q[k] -= amp; + } + } + + // dequantise + let t43 = pow43_table(); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + let cb = ics.band_type[g][sfb]; + if cb == ZERO_HCB || cb >= NOISE_HCB { + continue; + } + let gain = sf_gain(ics.sf[g][sfb]); + for w in w0..w0 + info.group_len[g] as usize { + let r = w * 128 + swb[sfb] as usize..w * 128 + swb[sfb + 1] as usize; + for (o, &v) in ics.spec[r.clone()].iter_mut().zip(&q[r]) { + let a = v.unsigned_abs(); + let m = t43.get(a as usize).copied().unwrap_or_else(|| pow43(a)) * gain; + *o = if v < 0 { -m } else { m }; + } + } + } + } + self.scratch.q = q; + Ok(ics) + } + + fn next_random(&mut self) -> f32 { + self.noise_state = self.noise_state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (self.noise_state as i32) as f32 + } + + /// Perceptual noise substitution. `pair` = (left channel, ms mask) for the right channel of a CPE: + /// bands that are noise in both channels with `ms_used` reuse the left channel's noise vector. + fn fill_noise(&mut self, ics: &mut Ics, pair: Option<(&Ics, Option<&[[bool; 64]; 8]>)>) { + let info = ics.info; + let swb = info.swb_offsets(self.sf_index()); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + if ics.band_type[g][sfb] != NOISE_HCB { + continue; + } + let (lo, hi) = (swb[sfb] as usize, swb[sfb + 1] as usize); + let target = (0.25 * ics.sf[g][sfb].clamp(-400, 400) as f64).exp2(); + let correlated = match pair { + Some((l, Some(ms))) => ms[g][sfb] && l.band_type[g][sfb] == NOISE_HCB, + _ => false, + }; + for w in w0..w0 + info.group_len[g] as usize { + let band = w * 128 + lo..w * 128 + hi; + if correlated && let Some((l, _)) = pair { + ics.spec[band.clone()].copy_from_slice(&l.spec[band.clone()]); + } else { + for k in band.clone() { + ics.spec[k] = self.next_random(); + } + } + let energy: f64 = ics.spec[band.clone()].iter().map(|&v| v as f64 * v as f64).sum(); + let scale = if energy > 0.0 { (target / energy.sqrt()) as f32 } else { 0.0 }; + for k in band { + ics.spec[k] *= scale; + } + } + } + } + } + + /// TNS + filterbank for a single channel element; returns output samples. + fn finish_channel(&mut self, ch: usize, ics: &mut Ics) -> Vec { + if let Some(t) = &ics.tns { + tns::apply_decoder(&mut ics.spec, &ics.info, self.sf_index(), t); + } + synth(&mut self.channels[ch], &mut self.scratch, &ics.info, &ics.spec) + } +} + +/// Inverse MDCT, windowing and overlap-add (§4.6.18), output scaled to ±1.0 full scale. +fn synth(state: &mut ChannelState, sc: &mut Scratch, info: &IcsInfo, spec: &[f32]) -> Vec { + let Scratch { buf, y, long, short, .. } = sc; + buf.fill(0.0); + let prev = state.prev_shape; + let cur = info.window_shape; + let norm = 1.0 / 32768.0; + match info.window_sequence { + WindowSequence::EightShort => { + let t = mdct_short(); + let y = &mut y[..256]; + let wc = window(cur, false); + for w in 0..8 { + t.inverse_with(&spec[w * 128..(w + 1) * 128], y, short); + let wl = window(if w == 0 { prev } else { cur }, false); + let off = 448 + 128 * w; + let s = norm / 128.0; + for n in 0..128 { + buf[off + n] += y[n] * wl[n] * s; + buf[off + 128 + n] += y[128 + n] * wc[127 - n] * s; + } + } + } + seq => { + mdct_long().inverse_with(&spec[..1024], y, long); + let s = norm / 1024.0; + match seq { + WindowSequence::LongStop => { + let ws = window(prev, false); + for n in 448..576 { + buf[n] = y[n] * ws[n - 448] * s; + } + for n in 576..1024 { + buf[n] = y[n] * s; + } + } + _ => { + let wl = window(prev, true); + for n in 0..1024 { + buf[n] = y[n] * wl[n] * s; + } + } + } + match seq { + WindowSequence::LongStart => { + let ws = window(cur, false); + for n in 1024..1472 { + buf[n] = y[n] * s; + } + for n in 1472..1600 { + buf[n] = y[n] * ws[127 - (n - 1472)] * s; + } + } + _ => { + let wl = window(cur, true); + for n in 1024..2048 { + buf[n] = y[n] * wl[2047 - n] * s; + } + } + } + } + } + let out: Vec = (0..1024).map(|n| state.overlap[n] + buf[n]).collect(); + state.overlap.copy_from_slice(&buf[1024..]); + state.prev_shape = cur; + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Implicitly signalled HE-AAC (an AAC-LC config) is recognised from the SBR fill element. + #[test] + fn sbr_fill_element_marks_implicit_he_aac() { + // AAC-LC, 22.05 kHz, stereo + let mut dec = Decoder::new(&[0x13, 0x90]).unwrap(); + assert!(!dec.sbr()); + // ID_FIL, count 1, extension_type EXT_SBR_DATA (+ 4 bits of payload), ID_END + let _ = dec.decode(&[0b1100_0011, 0b1010_0001, 0b1100_0000]); + assert!(dec.sbr()); + // other fill payloads (EXT_FILL) don't + let mut dec = Decoder::new(&[0x13, 0x90]).unwrap(); + let _ = dec.decode(&[0b1100_0010, 0b0000_0001, 0b1100_0000]); + assert!(!dec.sbr()); + // explicit signalling: AOT 5, core 22.05 kHz, stereo, extension 44.1 kHz, core AOT 2 + let dec = Decoder::new(&[0x2B, 0x92, 0x08, 0x00]).unwrap(); + assert!(dec.sbr()); + assert_eq!(dec.sample_rate(), 22_050); + } +} diff --git a/crates/aac/src/encoder.rs b/crates/aac/src/encoder.rs new file mode 100644 index 0000000000..572f2454bf --- /dev/null +++ b/crates/aac/src/encoder.rs @@ -0,0 +1,842 @@ +//! AAC-LC encoder. +//! +//! Pipeline per 1024-sample frame and channel element: +//! 1. transient detection on high-passed 128-sample sub-blocks (one frame of look-ahead) → +//! window sequence (`ONLY_LONG` / `LONG_START` / `EIGHT_SHORT` / `LONG_STOP`) and short-window grouping; +//! 2. windowed MDCT (sine or KBD); +//! 3. psychoacoustic model → per-band masking thresholds and perceptual entropy; +//! 4. TNS analysis (long windows; Levinson–Durbin, applied when the prediction gain is high); +//! 5. per-band M/S decision for channel pairs (common window); +//! 6. rate–distortion loop: a single multiplier λ scales every band's allowed noise; each band gets the +//! coarsest scalefactor meeting its noise target, sections/codebooks are chosen by dynamic +//! programming, and λ is bisected until the frame fits its bit budget (CBR) or fixed (VBR); +//! 7. bit reservoir (CBR): budget follows the frame's PE, bounded by the decoder buffer +//! (6144 bits/channel); overflow is padded with fill elements so the rate stays constant. + +mod psy; +mod quant; + +use filmcraft_bitstream::BitWriter; + +use crate::config::{AdtsHeader, AudioSpecificConfig, ElementType, ProgramConfig, config_layout}; +use crate::ics::{IcsInfo, WindowSequence}; +use crate::mdct::{WindowShape, mdct_long, mdct_short, window}; +use crate::tables::{sample_rate_index, swb_offsets_long, swb_offsets_short, tns_max_bands}; +use crate::tns::{self, TnsData, TnsFilter}; +use crate::{Error, FRAME_LEN, Result}; +use psy::Psy; +use quant::{ChannelInput, QuantCache, QuantChannel, quantise_channel, write_ics}; + +/// Rate control mode. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum BitrateMode { + /// Constant bitrate in bits per second (bit reservoir + fill elements). + Cbr(u32), + /// Constant quality, 1.0 (small) ..= 5.0 (transparent-ish). 3.0 targets the masking threshold. + Vbr(f32), +} + +/// Encoder settings. +#[derive(Debug, Clone, PartialEq)] +pub struct EncoderConfig { + /// One of the 13 AAC sampling rates (8–96 kHz). + pub sample_rate: u32, + /// 1–8. Channel order is the AAC element order: C; L R; C L R; C L R Cs; C L R Ls Rs; + /// C L R Ls Rs LFE; (7) C L R Ls Rs Cs LFE; (8) C Lc Rc L R Ls Rs LFE. + pub channels: usize, + pub mode: BitrateMode, + /// Audio bandwidth in Hz (`None` = chosen from the bitrate / quality). + pub bandwidth: Option, + /// Temporal noise shaping on long windows. + pub tns: bool, + /// Per-band mid/side stereo for channel pairs. + pub ms_stereo: bool, + pub window_shape: WindowShape, + /// Prefix every access unit with an ADTS header. + pub adts: bool, +} + +impl EncoderConfig { + pub fn cbr(sample_rate: u32, channels: usize, bitrate_bps: u32) -> EncoderConfig { + EncoderConfig { + sample_rate, + channels, + mode: BitrateMode::Cbr(bitrate_bps), + bandwidth: None, + tns: true, + ms_stereo: true, + window_shape: WindowShape::Sine, + adts: false, + } + } + pub fn vbr(sample_rate: u32, channels: usize, quality: f32) -> EncoderConfig { + EncoderConfig { mode: BitrateMode::Vbr(quality), ..Self::cbr(sample_rate, channels, 0) } + } +} + +/// Running statistics. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct EncoderStats { + pub frames: u64, + /// Raw access-unit bytes (without ADTS headers). + pub bytes: u64, + /// Element-frames coded with eight short windows. + pub short_blocks: u64, + pub ms_bands: u64, + pub stereo_bands: u64, + pub tns_filters: u64, + /// Bits spent on fill elements (CBR padding). + pub fill_bits: u64, +} + +impl EncoderStats { + /// Average bitrate of the raw access units. + pub fn bitrate(&self, sample_rate: u32) -> f64 { + if self.frames == 0 { 0.0 } else { self.bytes as f64 * 8.0 * sample_rate as f64 / (self.frames as f64 * FRAME_LEN as f64) } + } +} + +struct ElementState { + ty: ElementType, + tag: u8, + first_ch: usize, + prev_seq: WindowSequence, + short_next: Option, + /// Previous long-window thresholds per channel (pre-echo control). + prev_thr: Vec>, +} + +/// Per-frame analysis of one element. +struct ElementFrame { + info: IcsInfo, + chans: Vec, + /// Rate-loop memo per channel, filled while the frame is quantised. + caches: Vec, + ms_mask: Option<(u32, Vec)>, +} + +const HISTORY: usize = 2048; + +/// AAC-LC encoder. +pub struct Encoder { + cfg: EncoderConfig, + sf_index: u8, + asc: AudioSpecificConfig, + psy: Psy, + elements: Vec, + /// Input history per channel; `hist[c][0]` is absolute sample `origin`. + hist: Vec>, + origin: i64, + total_in: u64, + frames_done: u64, + flushed: bool, + bandwidth: u32, + // rate control + mean_acc: u64, + level: i64, + res_max: i64, + max_frame_bits: i64, + pe_avg: f32, + lambda: f32, + stats: EncoderStats, +} + +impl Encoder { + pub fn new(cfg: EncoderConfig) -> Result { + let sf_index = + sample_rate_index(cfg.sample_rate).ok_or(Error::InvalidConfig("sample rate must be one of the 13 AAC rates (7350–96000 Hz)"))?; + if !(1..=8).contains(&cfg.channels) { + return Err(Error::InvalidConfig("channels must be 1..=8")); + } + use ElementType::*; + let (channel_config, layout) = match cfg.channels { + 7 => (0u8, vec![(Sce, 0u8), (Cpe, 0), (Cpe, 1), (Sce, 1), (Lfe, 0)]), + 8 => (7, config_layout(7).unwrap_or_default()), + n => (n as u8, config_layout(n as u8).unwrap_or_default()), + }; + let mut asc = AudioSpecificConfig::lc(sf_index, channel_config); + if channel_config == 0 { + asc.pce = Some(ProgramConfig { + element_instance_tag: 0, + object_type: 1, + sf_index, + front: vec![(false, 0), (true, 0)], + side: vec![(true, 1)], + back: vec![(false, 1)], + lfe: vec![0], + ..Default::default() + }); + } + let mut elements = Vec::new(); + let mut ch = 0; + for (ty, tag) in layout { + elements.push(ElementState { + ty, + tag, + first_ch: ch, + prev_seq: WindowSequence::OnlyLong, + short_next: None, + prev_thr: vec![Vec::new(); ty.channels()], + }); + ch += ty.channels(); + } + let nfull = cfg.channels - elements.iter().filter(|e| e.ty == Lfe).count(); + let nyq = cfg.sample_rate / 2; + let auto_bw = match cfg.mode { + BitrateMode::Cbr(b) => { + let per = b as f32 / nfull.max(1) as f32 / 1000.0; + let pts = [ + (12.0f32, 4000.0f32), + (16.0, 5500.0), + (24.0, 8000.0), + (32.0, 11000.0), + (48.0, 14000.0), + (64.0, 16000.0), + (80.0, 17500.0), + (96.0, 19000.0), + (112.0, 20000.0), + (144.0, 22000.0), + ]; + interp(&pts, per) as u32 + } + BitrateMode::Vbr(q) => interp(&[(1.0, 13000.0), (2.0, 15000.0), (3.0, 17000.0), (4.0, 19000.0), (5.0, 20000.0)], q) as u32, + }; + let bandwidth = cfg.bandwidth.unwrap_or(auto_bw).clamp(1000, nyq.saturating_sub(nyq / 50).max(1000)); + let max_frame_bits = 6144 * cfg.channels as i64; + let mean = match cfg.mode { + BitrateMode::Cbr(b) => b as i64 * FRAME_LEN as i64 / cfg.sample_rate as i64, + BitrateMode::Vbr(_) => 0, + }; + if let BitrateMode::Cbr(b) = cfg.mode + && (b < 1000 * cfg.channels as u32 || mean > max_frame_bits) + { + return Err(Error::InvalidConfig("bitrate out of range for the channel count and sample rate")); + } + let res_max = (max_frame_bits - mean).max(0); + let hist = vec![vec![0f32; HISTORY]; cfg.channels]; + Ok(Encoder { + psy: Psy::new(cfg.sample_rate, sf_index), + sf_index, + asc, + elements, + hist, + origin: -(HISTORY as i64), + total_in: 0, + frames_done: 0, + flushed: false, + bandwidth, + mean_acc: 0, + level: 0, + res_max, + max_frame_bits, + pe_avg: 0.0, + lambda: 1.0, + stats: EncoderStats::default(), + cfg, + }) + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// `AudioSpecificConfig` bytes for the MP4 `esds` decoder-specific info. + pub fn audio_specific_config(&self) -> Vec { + self.asc.to_bytes() + } + + /// Encoder delay: the first 1024 decoded samples are priming and should be skipped (MP4 edit + /// list `media_time`, or iTunSMPB). + pub fn priming_samples(&self) -> u32 { + FRAME_LEN as u32 + } + + /// Audio bandwidth in Hz. + pub fn bandwidth(&self) -> u32 { + self.bandwidth + } + + pub fn stats(&self) -> EncoderStats { + self.stats + } + + /// Feed planar samples (±1.0 full scale); returns any complete access units. Missing channels are + /// treated as silence; all channels are truncated to the shortest slice. + pub fn encode(&mut self, planar: &[&[f32]]) -> Vec> { + if self.flushed { + return Vec::new(); + } + let n = planar.iter().take(self.cfg.channels).map(|p| p.len()).min().unwrap_or(0); + for c in 0..self.cfg.channels { + match planar.get(c) { + Some(p) => self.hist[c].extend(p[..n].iter().map(|v| if v.is_finite() { v.clamp(-8.0, 8.0) } else { 0.0 })), + None => self.hist[c].extend(std::iter::repeat_n(0.0, n)), + } + } + self.total_in += n as u64; + let mut out = Vec::new(); + while self.available() >= self.needed_for(self.frames_done) { + out.push(self.encode_frame(false)); + } + out + } + + /// Encode the remaining input (zero-padded). Afterwards the decoded stream, minus + /// [`priming_samples`](Self::priming_samples), covers every input sample. + pub fn flush(&mut self) -> Vec> { + if self.flushed { + return Vec::new(); + } + let frames = if self.total_in == 0 { 0 } else { self.total_in.div_ceil(FRAME_LEN as u64) + 1 }; + let mut out = Vec::new(); + while self.frames_done < frames { + let need = self.needed_for(self.frames_done); + let have = self.available(); + if have < need { + for h in &mut self.hist { + h.extend(std::iter::repeat_n(0.0, (need - have) as usize)); + } + } + let last = self.frames_done + 1 == frames; + out.push(self.encode_frame(last)); + } + self.flushed = true; + out + } + + /// Absolute sample index one past the buffered input. + fn available(&self) -> i64 { + self.origin + self.hist[0].len() as i64 + } + /// Input needed (absolute end) before frame `t` can be encoded (window + transient look-ahead). + fn needed_for(&self, t: u64) -> i64 { + (t as i64 + 1) * FRAME_LEN as i64 + 640 + } + + fn sample(&self, ch: usize, abs: i64) -> f32 { + let i = abs - self.origin; + if i < 0 { 0.0 } else { self.hist[ch].get(i as usize).copied().unwrap_or(0.0) } + } + + fn subblock_energy(&self, ch: usize, j: i64) -> f32 { + let s = j * 128; + let mut e = 0f32; + let mut prev = self.sample(ch, s - 1); + for n in s..s + 128 { + let x = self.sample(ch, n); + let d = x - prev; + e += d * d; + prev = x; + } + e + } + + /// Transient in the region covered by frame `t`'s short windows. + fn transient(&self, first_ch: usize, nch: usize, t: i64) -> bool { + for ch in first_ch..first_ch + nch { + let base = 8 * t; + let mut hist: Vec = (base - 13..base - 5).map(|j| self.subblock_energy(ch, j)).collect(); + for j in base - 5..=base + 4 { + let e = self.subblock_energy(ch, j); + let mean = hist.iter().sum::() / hist.len() as f32; + if e > 1.5e-4 && e > 10.0 * mean { + return true; + } + hist.remove(0); + hist.push(e); + } + } + false + } + + fn max_sfb(&self, short: bool, lfe: bool) -> usize { + let bw = if lfe { 240.min(self.bandwidth) } else { self.bandwidth }; + let (swb, m) = if short { (swb_offsets_short(self.sf_index), 128f64) } else { (swb_offsets_long(self.sf_index), 1024f64) }; + let bin = (bw as f64 / (self.cfg.sample_rate as f64 / 2.0) * m).ceil() as u16; + swb.iter().take(swb.len() - 1).filter(|&&o| o < bin).count().max(1) + } + + fn mdct(&self, ch: usize, seq: WindowSequence) -> Vec { + let shape = self.cfg.window_shape; + let seg = &self.hist[ch][HISTORY - FRAME_LEN..HISTORY + FRAME_LEN]; + let mut spec = vec![0f32; 1024]; + let scale = 32768.0 * 2.0; + if seq == WindowSequence::EightShort { + let ws = window(shape, false); + let mut buf = vec![0f32; 256]; + for w in 0..8 { + let off = 448 + 128 * w; + for n in 0..128 { + buf[n] = seg[off + n] * ws[n] * scale; + buf[128 + n] = seg[off + 128 + n] * ws[127 - n] * scale; + } + mdct_short().forward(&buf, &mut spec[w * 128..(w + 1) * 128]); + } + return spec; + } + let wl = window(shape, true); + let ws = window(shape, false); + let mut buf = vec![0f32; 2048]; + for n in 0..1024 { + let w = match seq { + WindowSequence::LongStop => { + if n < 448 { + 0.0 + } else if n < 576 { + ws[n - 448] + } else { + 1.0 + } + } + _ => wl[n], + }; + buf[n] = seg[n] * w * scale; + } + for n in 1024..2048 { + let w = match seq { + WindowSequence::LongStart => { + if n < 1472 { + 1.0 + } else if n < 1600 { + ws[127 - (n - 1472)] + } else { + 0.0 + } + } + _ => wl[2047 - n], + }; + buf[n] = seg[n] * w * scale; + } + mdct_long().forward(&buf, &mut spec); + spec + } + + fn tns_analyse(&self, spec: &mut [f32], info: &IcsInfo) -> Option { + let swb = swb_offsets_long(self.sf_index); + let lim = tns_max_bands(self.sf_index, false).min(info.max_sfb); + let hz = self.cfg.sample_rate as f64 / 2048.0; + let start = swb.iter().position(|&o| o as f64 * hz >= 1400.0).unwrap_or(swb.len() - 1); + if lim < start + 4 { + return None; + } + let (lo, hi) = (swb[start] as usize, swb[lim] as usize); + let x = &spec[lo..hi]; + const ORDER: usize = 8; + let mut r = [0f64; ORDER + 1]; + for (lag, rv) in r.iter_mut().enumerate() { + let mut acc = 0f64; + for n in lag..x.len() { + acc += x[n] as f64 * x[n - lag] as f64; + } + // Gaussian lag window for robustness + *rv = acc * (-0.5 * (0.02 * lag as f64).powi(2)).exp(); + } + if r[0] <= 1e-3 { + return None; + } + let (ks, gain) = tns::levinson(&r, ORDER); + if gain < 1.4 { + return None; + } + let mut coef = [0i8; 12]; + let mut order = 0; + for (i, &k) in ks.iter().enumerate() { + coef[i] = tns::quant_coef(k, true); + if coef[i] != 0 { + order = i + 1; + } + } + if order == 0 { + return None; + } + let mut t = TnsData::default(); + t.windows[0].n_filt = 1; + t.windows[0].coef_res = true; + t.windows[0].filt[0] = TnsFilter { length: (swb.len() - 1 - start) as u8, order: order as u8, direction: false, coef_compress: false, coef }; + tns::apply_encoder(spec, info, self.sf_index, &t); + Some(t) + } + + fn analyse_element(&mut self, ei: usize, t: i64) -> (ElementFrame, f32) { + let (ty, first, nch, prev_seq, cached) = { + let e = &self.elements[ei]; + (e.ty, e.first_ch, e.ty.channels(), e.prev_seq, e.short_next) + }; + let lfe = ty == ElementType::Lfe; + let (short_now, short_next) = + if lfe { (false, false) } else { (cached.unwrap_or_else(|| self.transient(first, nch, t)), self.transient(first, nch, t + 1)) }; + use WindowSequence::*; + let seq = if short_now { + EightShort + } else if prev_seq == EightShort || prev_seq == LongStart { + if short_next { EightShort } else { LongStop } + } else if short_next { + LongStart + } else { + OnlyLong + }; + let specs: Vec> = (first..first + nch).map(|c| self.mdct(c, seq)).collect(); + let short = seq == EightShort; + let mut info = IcsInfo { window_sequence: seq, window_shape: self.cfg.window_shape, max_sfb: self.max_sfb(short, lfe), ..Default::default() }; + if short { + // group windows of similar energy; an attack starts a new group + let en: Vec = + (0..8).map(|w| specs.iter().map(|s| s[w * 128..(w + 1) * 128].iter().map(|v| v * v).sum::()).sum::() + 1.0).collect(); + info.num_groups = 1; + info.group_len = [1, 0, 0, 0, 0, 0, 0, 0]; + for w in 1..8 { + let r = en[w] / en[w - 1]; + if !(1.0 / 6.0..=3.0).contains(&r) { + info.num_groups += 1; + info.group_len[info.num_groups - 1] = 1; + } else { + info.group_len[info.num_groups - 1] += 1; + } + } + } + let mut pe = 0f32; + let mut specs = specs; + let mut thrs: Vec>> = Vec::with_capacity(nch); + let mut tnss = Vec::with_capacity(nch); + for (i, spec) in specs.iter_mut().enumerate() { + let mut r = self.psy.analyse(spec, short); + pe += r.pe; + let e = &mut self.elements[ei]; + if !short { + if prev_seq != EightShort && e.prev_thr[i].len() == r.thr[0].len() { + for (t, p) in r.thr[0].iter_mut().zip(&e.prev_thr[i]) { + *t = t.min(2.0 * p.max(1e-9)); + } + } + e.prev_thr[i] = r.thr[0].clone(); + } else { + e.prev_thr[i].clear(); + } + // group thresholds: the most sensitive window times the group length + let mut thr_g = vec![vec![0f32; 64]; info.num_groups]; + for g in 0..info.num_groups { + let w0 = info.group_start(g); + let gl = info.group_len[g] as usize; + for sfb in 0..info.max_sfb { + let m = (w0..w0 + gl).map(|w| r.thr[w][sfb]).fold(f32::MAX, f32::min); + thr_g[g][sfb] = m * gl as f32; + } + } + thrs.push(thr_g); + let tns = if self.cfg.tns && !short && !lfe { self.tns_analyse(spec, &info) } else { None }; + if let Some(t) = &tns { + self.stats.tns_filters += t.windows[0].n_filt as u64; + } + tnss.push(tns); + } + let swb = info.swb_offsets(self.sf_index); + let mut ms_mask = None; + if ty == ElementType::Cpe { + let mut used = Vec::with_capacity(info.num_groups * info.max_sfb); + let (l, r) = specs.split_at_mut(1); + let (l, r) = (&mut l[0], &mut r[0]); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + let gl = info.group_len[g] as usize; + for sfb in 0..info.max_sfb { + let (mut el, mut er, mut em, mut es) = (0f32, 0f32, 0f32, 0f32); + for w in w0..w0 + gl { + for k in w * 128 + swb[sfb] as usize..w * 128 + swb[sfb + 1] as usize { + let (a, b) = (l[k], r[k]); + el += a * a; + er += b * b; + em += 0.25 * (a + b) * (a + b); + es += 0.25 * (a - b) * (a - b); + } + } + let (tl, tr) = (thrs[0][g][sfb].max(1e-9), thrs[1][g][sfb].max(1e-9)); + let tm = 0.5 * tl.min(tr); + let c = |e: f32, t: f32| (e / t).max(1.0).log2(); + let use_ms = self.cfg.ms_stereo && c(em, tm) + c(es, tm) < c(el, tl) + c(er, tr); + used.push(use_ms); + self.stats.stereo_bands += 1; + if use_ms { + self.stats.ms_bands += 1; + thrs[0][g][sfb] = tm; + thrs[1][g][sfb] = tm; + for w in w0..w0 + gl { + for k in w * 128 + swb[sfb] as usize..w * 128 + swb[sfb + 1] as usize { + let (a, b) = (l[k], r[k]); + l[k] = 0.5 * (a + b); + r[k] = 0.5 * (a - b); + } + } + } + } + } + let n_ms = used.iter().filter(|&&u| u).count(); + let present = if n_ms == 0 { + 0 + } else if n_ms == used.len() { + 2 + } else { + 1 + }; + ms_mask = Some((present, used)); + } + let chans: Vec = specs.iter().zip(thrs.iter()).zip(tnss).map(|((s, th), tn)| ChannelInput::new(info, swb, s, th, tn)).collect(); + let caches = chans.iter().map(QuantCache::new).collect(); + let e = &mut self.elements[ei]; + e.prev_seq = seq; + e.short_next = Some(short_next); + if short { + self.stats.short_blocks += 1; + } + (ElementFrame { info, chans, caches, ms_mask }, pe) + } + + fn element_overhead(&self, ei: usize, f: &ElementFrame) -> u32 { + let info_bits = if f.info.is_short() { 15 } else { 11 }; + match self.elements[ei].ty { + ElementType::Cpe => { + let ms = match &f.ms_mask { + Some((1, used)) => used.len() as u32, + _ => 0, + }; + 3 + 4 + 1 + info_bits + 2 + ms + } + _ => 3 + 4 + info_bits, + } + } + + fn quantise_all(frames: &mut [ElementFrame], lambda: f32) -> Vec> { + #[cfg(feature = "rayon")] + { + use rayon::prelude::*; + frames + .par_iter_mut() + .map(|f| f.chans.par_iter().zip(f.caches.par_iter_mut()).map(|(c, cache)| quantise_channel(c, cache, lambda)).collect()) + .collect() + } + #[cfg(not(feature = "rayon"))] + { + frames + .iter_mut() + .map(|f| f.chans.iter().zip(f.caches.iter_mut()).map(|(c, cache)| quantise_channel(c, cache, lambda)).collect()) + .collect() + } + } + + /// `last`: final frame of the stream — a CBR stream spends (or pads away) the whole reservoir so + /// the total size is exactly `frames × bitrate × 1024 / rate`. + fn encode_frame(&mut self, last: bool) -> Vec { + let t = self.frames_done as i64; + let mut frames = Vec::with_capacity(self.elements.len()); + let mut pe = 0f32; + for ei in 0..self.elements.len() { + let (f, p) = self.analyse_element(ei, t); + pe += p; + frames.push(f); + } + let pce_bits = if self.cfg.adts && self.asc.channel_config == 0 { + let mut bw = BitWriter::new(); + if let Some(p) = &self.asc.pce { + p.write(&mut bw); + } + 3 + bw.bit_len() as u32 + 8 + } else { + 0 + }; + let overhead: u32 = (0..frames.len()).map(|i| self.element_overhead(i, &frames[i])).sum::() + 3 + 7 + pce_bits; + let total_bits = |q: &Vec>| overhead + q.iter().flatten().map(|c| c.bits).sum::(); + + let any_short = frames.iter().any(|f| f.info.is_short()); + let (mean, target) = match self.cfg.mode { + BitrateMode::Cbr(b) => { + self.mean_acc += b as u64 * FRAME_LEN as u64; + let mean = (self.mean_acc / self.cfg.sample_rate as u64) as i64; + self.mean_acc %= self.cfg.sample_rate as u64; + self.pe_avg = if self.pe_avg == 0.0 { pe.max(1.0) } else { 0.95 * self.pe_avg + 0.05 * pe.max(1.0) }; + let mut ratio = (pe.max(1.0) / self.pe_avg).clamp(0.7, 1.6); + if any_short { + ratio = (ratio * 1.3).min(2.0); + } + let mut desired = mean as f32 * ratio + (self.level as f32 - 0.5 * self.res_max as f32) * 0.15; + desired = desired.min((mean + self.level) as f32).max((mean - (self.res_max - self.level)) as f32); + desired = desired.min(self.max_frame_bits as f32); + if last { + desired = (mean + self.level) as f32; + } + (mean, desired.max(0.0) as u32) + } + BitrateMode::Vbr(_) => (0, self.max_frame_bits as u32), + }; + + let quant = match self.cfg.mode { + BitrateMode::Vbr(q) => { + let d = q.clamp(0.5, 6.0) - 1.0; + let mut lambda = 10f32.powf(-0.2 - 1.2 * d + 0.09 * d * d); + let mut res = Self::quantise_all(&mut frames, lambda); + while total_bits(&res) > target && lambda < 1e9 { + lambda *= 2.0; + res = Self::quantise_all(&mut frames, lambda); + } + res + } + BitrateMode::Cbr(_) => { + let mut lam = self.lambda.clamp(1e-6, 1e9); + let mut cur = Self::quantise_all(&mut frames, lam); + let (mut lo, mut hi, mut best); + if total_bits(&cur) <= target { + hi = lam; + best = cur; + lo = lam; + loop { + lo /= 4.0; + if lo < 1e-6 { + break; + } + cur = Self::quantise_all(&mut frames, lo); + if total_bits(&cur) > target { + break; + } + hi = lo; + best = cur; + } + } else { + lo = lam; + loop { + lam *= 4.0; + cur = Self::quantise_all(&mut frames, lam); + if total_bits(&cur) <= target || lam > 1e9 { + break; + } + lo = lam; + } + hi = lam; + best = cur; + } + for _ in 0..6 { + if hi / lo < 1.03 { + break; + } + let mid = (lo * hi).sqrt(); + let r = Self::quantise_all(&mut frames, mid); + if total_bits(&r) <= target { + hi = mid; + best = r; + } else { + lo = mid; + } + } + self.lambda = hi; + best + } + }; + + // assemble the raw_data_block + let mut bw = BitWriter::new(); + if pce_bits > 0 + && let Some(p) = &self.asc.pce + { + bw.write_bits(5, 3); + p.write(&mut bw); + } + for (ei, (f, q)) in frames.iter().zip(&quant).enumerate() { + let e = &self.elements[ei]; + match e.ty { + ElementType::Cpe => { + bw.write_bits(1, 3); + bw.write_bits(e.tag as u32, 4); + bw.write_bits(1, 1); // common_window + f.info.write(&mut bw); + let (present, used) = f.ms_mask.clone().unwrap_or((0, Vec::new())); + bw.write_bits(present, 2); + if present == 1 { + for &u in &used { + bw.write_bits(u as u32, 1); + } + } + write_ics(&mut bw, &f.chans[0], &q[0], false); + write_ics(&mut bw, &f.chans[1], &q[1], false); + } + ty => { + bw.write_bits(if ty == ElementType::Lfe { 3 } else { 0 }, 3); + bw.write_bits(e.tag as u32, 4); + write_ics(&mut bw, &f.chans[0], &q[0], true); + } + } + } + if let BitrateMode::Cbr(_) = self.cfg.mode { + let projected = ((bw.bit_len() as i64 + 3 + 7) / 8) * 8; + let excess = self.level + mean - projected - if last { 0 } else { self.res_max }; + if excess > 0 { + self.stats.fill_bits += write_fill(&mut bw, excess as u64); + } + } + bw.write_bits(7, 3); // ID_END + let au = bw.finish(); + let used = au.len() as i64 * 8; + if let BitrateMode::Cbr(_) = self.cfg.mode { + self.level = (self.level + mean - used).clamp(0, self.res_max); + } + self.stats.frames += 1; + self.stats.bytes += au.len() as u64; + + // slide the input window + for h in &mut self.hist { + h.drain(..FRAME_LEN); + } + self.origin += FRAME_LEN as i64; + self.frames_done += 1; + + if self.cfg.adts { + let fullness = match self.cfg.mode { + BitrateMode::Cbr(_) => ((self.level / (32 * self.cfg.channels as i64)) as u16).min(0x7FE), + BitrateMode::Vbr(_) => 0x7FF, + }; + let mut out = AdtsHeader::write(2, self.sf_index, self.asc.channel_config, au.len(), fullness).to_vec(); + out.extend_from_slice(&au); + out + } else { + au + } + } +} + +/// Write fill elements totalling at least `bits - 7` bits (at most `bits + 8`); returns bits written. +fn write_fill(bw: &mut BitWriter, bits: u64) -> u64 { + let start = bw.bit_len() as u64; + let mut left = bits as i64; + while left >= 7 { + let n = (((left - 7) / 8) as usize).min(269); + let n = if n >= 15 && (left - 15) / 8 < 15 { 14 } else { n }; + bw.write_bits(6, 3); + if n >= 15 { + bw.write_bits(15, 4); + bw.write_bits((n - 14) as u32, 8); + } else { + bw.write_bits(n as u32, 4); + } + if n > 0 { + bw.write_bits(0, 8); // extension_type EXT_FILL + fill_nibble + for _ in 1..n { + bw.write_bits(0xA5, 8); + } + } + let el = 7 + if n >= 15 { 8 } else { 0 } + 8 * n as i64; + left -= el; + if n == 0 { + break; + } + } + bw.bit_len() as u64 - start +} + +fn interp(pts: &[(f32, f32)], x: f32) -> f32 { + if x <= pts[0].0 { + return pts[0].1; + } + for w in pts.windows(2) { + if x <= w[1].0 { + let t = (x - w[0].0) / (w[1].0 - w[0].0); + return w[0].1 + t * (w[1].1 - w[0].1); + } + } + pts[pts.len() - 1].1 +} diff --git a/crates/aac/src/encoder/psy.rs b/crates/aac/src/encoder/psy.rs new file mode 100644 index 0000000000..1101a76234 --- /dev/null +++ b/crates/aac/src/encoder/psy.rs @@ -0,0 +1,176 @@ +//! Psychoacoustic model operating on MDCT coefficients. +//! +//! Per scalefactor band: energy; tonality from the spectral flatness measure (SFM) of a +//! neighbourhood of ≥ 16 lines; masking-threshold PSD by spreading the band PSDs with Schroeder's +//! spreading function in the Bark domain; a tonality-dependent masking offset (≈18 dB for tonal, +//! ≈6 dB for noise-like maskers); the absolute threshold of hearing (Terhardt) as a floor; and +//! perceptual entropy (PE) as an estimate of the bits the frame needs. + +use crate::tables::{swb_offsets_long, swb_offsets_short}; + +/// Precomputed per-band data for one window length. +pub struct BandModel { + pub offsets: &'static [u16], + /// Linear spreading factor `[maskee][masker]` (power). + spread: Vec>, + /// ATH as band energy in encoder spectrum units. + ath: Vec, + /// Neighbourhood `[lo, hi)` in lines used for the tonality estimate. + tonal_span: Vec<(usize, usize)>, +} + +fn bark(f: f64) -> f64 { + 13.0 * (0.00076 * f).atan() + 3.5 * (f / 7500.0).powi(2).atan() +} + +/// Terhardt's threshold in quiet (dB SPL), capped at 40 dB so content near the band edge is not +/// discarded wholesale (the bandwidth setting limits the coded range instead). +fn ath_db(f: f64) -> f64 { + let f = (f / 1000.0).max(0.02); + (3.64 * f.powf(-0.8) - 6.5 * (-0.6 * (f - 3.3).powi(2)).exp() + 1e-3 * f.powi(4)).min(40.0) +} + +impl BandModel { + fn new(sample_rate: u32, offsets: &'static [u16], m: usize) -> BandModel { + let nb = offsets.len() - 1; + let hz_per_line = sample_rate as f64 / (2.0 * m as f64); + let centre: Vec = (0..nb).map(|b| bark((offsets[b] as f64 + offsets[b + 1] as f64) * 0.5 * hz_per_line)).collect(); + let spread = (0..nb) + .map(|i| { + (0..nb) + .map(|j| { + let dz = centre[i] - centre[j] + 0.474; + let db = 15.81 + 7.5 * dz - 17.5 * (1.0 + dz * dz).sqrt(); + if db < -60.0 { 0.0 } else { 10f64.powf(db / 10.0) as f32 } + }) + .collect() + }) + .collect(); + // A full-scale (16-bit) sine of power P_ref = 32768²/2 has MDCT energy ≈ 2·M²·P per frame + // with the encoder's scaling; it is taken as 96 dB SPL. 3 dB of safety margin. + let p_ref = 32768f64 * 32768.0 / 2.0; + let ath = (0..nb) + .map(|b| { + let mut lo = f64::MAX; + for k in offsets[b]..offsets[b + 1] { + lo = lo.min(ath_db((k as f64 + 0.5) * hz_per_line)); + } + (2.0 * (m * m) as f64 * p_ref * 10f64.powf((lo - 96.0 - 3.0) / 10.0)) as f32 + }) + .collect(); + let tonal_span = (0..nb) + .map(|b| { + let (lo, hi) = (offsets[b] as usize, offsets[b + 1] as usize); + if hi - lo >= 16 { + (lo, hi) + } else { + let c = (lo + hi) / 2; + let lo = c.saturating_sub(8); + let hi = (lo + 16).min(m); + (hi.saturating_sub(16), hi) + } + }) + .collect(); + BandModel { offsets, spread, ath, tonal_span } + } + + pub fn num_bands(&self) -> usize { + self.offsets.len() - 1 + } +} + +pub struct Psy { + pub long: BandModel, + pub short: BandModel, +} + +/// Per-window, per-band analysis of one channel. +#[derive(Clone, Default)] +pub struct PsyResult { + /// `[window][band]` energy. + pub en: Vec>, + /// `[window][band]` masking threshold (allowed noise energy). + pub thr: Vec>, + pub pe: f32, +} + +impl Psy { + pub fn new(sample_rate: u32, sf_index: u8) -> Psy { + Psy { + long: BandModel::new(sample_rate, swb_offsets_long(sf_index), 1024), + short: BandModel::new(sample_rate, swb_offsets_short(sf_index), 128), + } + } + + /// Analyse a spectrum: 1024 lines (long) or 8×128 (short). + pub fn analyse(&self, spec: &[f32], short: bool) -> PsyResult { + let (model, nwin, wlen) = if short { (&self.short, 8, 128) } else { (&self.long, 1, 1024) }; + let nb = model.num_bands(); + let mut res = PsyResult { en: Vec::with_capacity(nwin), thr: Vec::with_capacity(nwin), pe: 0.0 }; + let mut psd = vec![0f32; nb]; + let mut tonality = vec![0f32; nb]; + for w in 0..nwin { + let s = &spec[w * wlen..(w + 1) * wlen]; + let mut en = vec![0f32; nb]; + for b in 0..nb { + let (lo, hi) = (model.offsets[b] as usize, model.offsets[b + 1] as usize); + en[b] = s[lo..hi].iter().map(|v| v * v).sum(); + psd[b] = en[b] / (hi - lo) as f32; + // spectral flatness over the neighbourhood + let (tlo, thi) = model.tonal_span[b]; + let n = (thi - tlo) as f64; + let mut sum = 0f64; + let mut log_sum = 0f64; + for &v in &s[tlo..thi] { + let p = v as f64 * v as f64 + 1e-3; + sum += p; + log_sum += p.ln(); + } + let sfm_db = 10.0 * ((log_sum / n).exp() / (sum / n)).log10(); + // real-valued Gaussian noise gives ≈ -5.5 dB; a pure tone ≤ -25 dB + tonality[b] = ((-sfm_db - 6.0) / 19.0).clamp(0.0, 1.0) as f32; + } + let mut thr = vec![0f32; nb]; + for b in 0..nb { + let row = &model.spread[b]; + let mut sp = 0f32; + for j in 0..nb { + sp += psd[j] * row[j]; + } + let width = (model.offsets[b + 1] - model.offsets[b]) as f32; + let offset_db = tonality[b] * 30.0 + (1.0 - tonality[b]) * 6.0; + let t = sp * width * 10f32.powf(-offset_db / 10.0); + thr[b] = t.max(model.ath[b]); + if en[b] > thr[b] { + res.pe += width * 0.5 * (en[b] / thr[b]).log2(); + } + } + res.en.push(en); + res.thr.push(thr); + } + res + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tone_is_masked_less_than_noise() { + let psy = Psy::new(44100, 4); + let tone: Vec = (0..1024).map(|k| if k == 100 { 1e6 } else { 0.0 }).collect(); + let mut seed = 1u32; + let noise: Vec = (0..1024) + .map(|_| { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + ((seed >> 8) as f32 / (1 << 24) as f32 - 0.5) * 1e4 + }) + .collect(); + let rt = psy.analyse(&tone, false); + let rn = psy.analyse(&noise, false); + let b = psy.long.offsets.iter().position(|&o| o > 100).unwrap() - 1; + assert!(rt.thr[0][b] < rt.en[0][b] / 30.0); + assert!(rn.thr[0][b] > rn.en[0][b] / 10.0); + } +} diff --git a/crates/aac/src/encoder/quant.rs b/crates/aac/src/encoder/quant.rs new file mode 100644 index 0000000000..f2c91a7f68 --- /dev/null +++ b/crates/aac/src/encoder/quant.rs @@ -0,0 +1,510 @@ +//! Quantisation, scalefactor selection, codebook/section selection, bit counting and writing of one +//! `individual_channel_stream`. +//! +//! Coefficients are kept in bitstream order ("interleaved": group → band → window → line), so every +//! (group, band) is a contiguous slice and short-window grouping needs no special casing. + +use filmcraft_bitstream::BitWriter; + +use crate::huffman::{self, ZERO_HCB}; +use crate::ics::IcsInfo; +use crate::tables::pow43_table; +use crate::tns::TnsData; + +/// Magic rounding offset for AAC quantisation (`0.5 - 0.0946`), minimising the error for the +/// `x^(3/4)` companding. +const ROUND: f32 = 0.4054; +const MAX_Q: i32 = 8191; + +/// One (group, band) of a channel, in bitstream order. +#[derive(Clone, Copy, Debug)] +pub struct Band { + /// Range in the interleaved coefficient array. + pub start: usize, + pub end: usize, + /// Allowed noise energy at λ = 1. + pub thr: f32, + /// Coefficient energy. + pub en: f32, + /// Largest `|x|^(3/4)`. + pub max34: f32, +} + +/// Everything the rate loop needs for one channel. +pub struct ChannelInput { + pub info: IcsInfo, + /// Interleaved (bitstream-order) coefficients. + pub x: Vec, + pub xabs: Vec, + pub x34: Vec, + pub bands: Vec, + pub tns: Option, +} + +impl ChannelInput { + /// `spec` is window-major (short windows at stride 128); `thr[g][sfb]` and the grouping come from + /// `info`. + pub fn new(info: IcsInfo, swb: &[u16], spec: &[f32], thr: &[Vec], tns: Option) -> ChannelInput { + let mut x = Vec::with_capacity(1024); + let mut bands = Vec::new(); + for g in 0..info.num_groups { + let w0 = info.group_start(g); + for sfb in 0..info.max_sfb { + let start = x.len(); + for w in w0..w0 + info.group_len[g] as usize { + x.extend_from_slice(&spec[w * 128 + swb[sfb] as usize..w * 128 + swb[sfb + 1] as usize]); + } + bands.push(Band { start, end: x.len(), thr: thr[g][sfb], en: 0.0, max34: 0.0 }); + } + } + let xabs: Vec = x.iter().map(|v| v.abs()).collect(); + let x34: Vec = xabs.iter().map(|v| v.powf(0.75)).collect(); + for b in &mut bands { + b.en = x[b.start..b.end].iter().map(|v| v * v).sum(); + b.max34 = x34[b.start..b.end].iter().cloned().fold(0.0, f32::max); + } + ChannelInput { info, x, xabs, x34, bands, tns } + } +} + +/// Scalefactor-keyed results of one band, valid until the band is quantised at another scalefactor. +#[derive(Clone, Copy)] +struct BandMemo { + /// Scalefactor the cached values belong to (`NO_SF` when empty). + sf: i32, + maxq: i32, + /// Spectral bits per codebook plus one for an all-zero band (see `band_costs`), once computed. + cost: Option<[u32; 12]>, +} + +const NO_SF: i32 = i32::MIN; + +/// Work that depends only on (band, scalefactor), not on λ, kept across the rate-loop iterations of +/// one frame. One cache per channel; callers hold it mutably, so parallel channels share nothing. +pub struct QuantCache { + /// Distortion per (band, scalefactor), NaN while unknown. + dist: Vec, + bands: Vec, + /// Quantised values of every band at its memoised scalefactor. + q: Vec, + /// Cost row of a band that quantises to zeros. + zero_cost: Vec>, +} + +impl QuantCache { + pub fn new(ch: &ChannelInput) -> QuantCache { + let nb = ch.bands.len(); + QuantCache { + dist: vec![f32::NAN; nb * 256], + bands: vec![BandMemo { sf: NO_SF, maxq: 0, cost: None }; nb], + q: vec![0; ch.x.len()], + zero_cost: vec![None; nb], + } + } +} + +/// A quantised channel ready to be written. +#[derive(Clone, Default)] +pub struct QuantChannel { + pub q: Vec, + /// Scalefactor per band (only meaningful where `cb != 0`). + pub sf: Vec, + pub cb: Vec, + /// Per group: (codebook, number of bands). + pub sections: Vec>, + pub global_gain: i32, + /// Bits of the ICS excluding `ics_info`. + pub bits: u32, +} + +#[inline] +fn gain34(sf: i32) -> f32 { + (-0.1875 * (sf as f32 - 100.0)).exp2() +} +#[inline] +fn gain(sf: i32) -> f32 { + (0.25 * (sf as f32 - 100.0)).exp2() +} + +/// Distortion of a slice quantised at `sf`. Kept out of line so the accumulator stays in a register. +#[inline(never)] +fn distortion(xabs: &[f32], x34: &[f32], sf: i32) -> f32 { + let g34 = gain34(sf); + let g = gain(sf); + let t = pow43_table(); + let mut d = 0f32; + for (&a, &b) in xabs.iter().zip(x34) { + let q = ((b * g34 + ROUND) as i32).min(MAX_Q); + let e = a - t[(q as usize) & 8191] * g; + d += e * e; + } + d +} + +/// Quantise a slice at `sf`, writing the (signed) values to `out`. +fn quantise(x: &[f32], sf: i32, x34: &[f32], out: &mut [i32]) { + let g34 = gain34(sf); + for ((o, &xv), &b) in out.iter_mut().zip(x).zip(x34) { + let q = ((b * g34 + ROUND) as i32).min(MAX_Q); + *o = if xv < 0.0 { -q } else { q }; + } +} + +/// Largest scalefactor whose distortion stays within `allowed`. +fn search_sf(ch: &ChannelInput, bi: usize, dist: &mut [f32], allowed: f32) -> i32 { + let b = &ch.bands[bi]; + let r = b.start..b.end; + let (xa, x34) = (&ch.xabs[r.clone()], &ch.x34[r]); + let dist = &mut dist[bi * 256..bi * 256 + 256]; + let mut dist_at = |sf: i32| { + let d = &mut dist[sf as usize]; + if d.is_nan() { + *d = distortion(xa, x34, sf); + } + *d + }; + let l2 = b.max34.log2(); + let lo = ((100.0 - ((MAX_Q as f32 + 0.5).log2() - l2) / 0.1875).ceil() as i32).clamp(0, 255); + let zero = ((100.0 - ((1.0 - ROUND).log2() - l2) / 0.1875).floor() as i32 + 1).clamp(0, 256); + let mut a = lo; + let mut hi = (zero - 1).clamp(lo, 255); + if dist_at(a) > allowed { + return a; + } + while a < hi { + let mid = (a + hi + 1) / 2; + if dist_at(mid) <= allowed { + a = mid; + } else { + hi = mid - 1; + } + } + a +} + +/// Candidate codebooks for a band whose largest magnitude is `m` (0 for all-zero bands). +fn candidates(m: i32) -> &'static [u8] { + match m { + 0 => &[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + 1 => &[1, 2, 3, 4, 5, 6], + 2 => &[3, 4, 5, 6, 7, 8], + 3..=4 => &[5, 6, 7, 8, 9, 10], + 5..=7 => &[7, 8, 9, 10, 11], + 8..=12 => &[9, 10, 11], + _ => &[11], + } +} + +/// Bring the scalefactor differences between coded bands (`sf >= 0`) within ±60 by lowering the +/// offenders, sweeping up and down until stable. +fn limit_sf_steps(sf: &mut [i32]) { + let coded: Vec = (0..sf.len()).filter(|&i| sf[i] >= 0).collect(); + for _ in 0..8 { + let mut changed = false; + for w in coded.windows(2) { + if sf[w[1]] > sf[w[0]] + 60 { + sf[w[1]] = sf[w[0]] + 60; + changed = true; + } + } + for w in coded.windows(2).rev() { + if sf[w[0]] > sf[w[1]] + 60 { + sf[w[0]] = sf[w[1]] + 60; + changed = true; + } + } + if !changed { + break; + } + } +} + +/// Quantise every band at its scalefactor into `q`, recording each band's largest magnitude in +/// `maxq`; a band already quantised at that scalefactor is taken from `cache`. A band that quantises +/// to all zeros leaves the chain (`sf = -1`); returns whether any did. +fn quantise_bands(ch: &ChannelInput, cache: &mut QuantCache, sf: &mut [i32], q: &mut [i32], maxq: &mut [i32]) -> bool { + let mut removed = false; + for (i, b) in ch.bands.iter().enumerate() { + let r = b.start..b.end; + if sf[i] < 0 { + q[r].fill(0); + maxq[i] = 0; + continue; + } + let memo = &mut cache.bands[i]; + if memo.sf != sf[i] { + quantise(&ch.x[r.clone()], sf[i], &ch.x34[r.clone()], &mut cache.q[r.clone()]); + *memo = BandMemo { sf: sf[i], maxq: cache.q[r.clone()].iter().map(|v| v.abs()).max().unwrap_or(0), cost: None }; + } + q[r.clone()].copy_from_slice(&cache.q[r]); + maxq[i] = memo.maxq; + if maxq[i] == 0 { + sf[i] = -1; + removed = true; + } + } + removed +} + +/// Cost sentinel for a codebook that cannot code a band. +const INF: u32 = u32::MAX / 4; + +/// Bits per candidate codebook for a band holding `vals` (largest magnitude `maxq`); `INF` +/// marks codebooks that cannot code it. An all-zero band inside a coded section also needs a (zero) +/// scalefactor difference, hence the extra bit. +fn band_costs(vals: &[i32], maxq: i32) -> [u32; 12] { + let mut cost = [INF; 12]; + let extra = (maxq == 0) as u32; + for &c in candidates(maxq) { + match c { + 0 => cost[0] = 0, + // odd codebook below 11 and its successor share everything but the length table + c if c % 2 == 1 && c < huffman::ESC_HCB => { + let (a, b) = huffman::count_bits_pair(c, vals); + cost[c as usize] = a + extra; + cost[c as usize + 1] = b + extra; + } + c if c % 2 == 0 => {} // filled in by the pair above + c => cost[c as usize] = huffman::count_bits(c, vals) + extra, + } + } + cost +} + +/// Bits to code each band of a group with each codebook (`INF` when the codebook cannot), from the +/// per-band memo. +fn codebook_costs(ch: &ChannelInput, cache: &mut QuantCache, sf: &[i32], q: &[i32], maxq: &[i32], base: usize, n: usize) -> Vec<[u32; 12]> { + let mut cost = vec![[INF; 12]; n]; + for i in 0..n { + let bi = base + i; + let b = &ch.bands[bi]; + let vals = &q[b.start..b.end]; + // A band that ends up uncoded (`sf < 0`) holds zeros, whatever its memo says. + let row = if sf[bi] >= 0 { &mut cache.bands[bi].cost } else { &mut cache.zero_cost[bi] }; + cost[i] = *row.get_or_insert_with(|| band_costs(vals, maxq[bi])); + } + cost +} + +/// Dynamic programming over the codebook of each band of a group, charging `new_sec` bits for every +/// section start. Returns the chosen codebook per band. +fn best_codebooks(cost: &[[u32; 12]], new_sec: u32) -> Vec { + let n = cost.len(); + let mut best = [INF; 12]; + let mut back = vec![[0u8; 12]; n]; + for c in 0..12 { + if cost[0][c] < INF { + best[c] = new_sec + cost[0][c]; + } + } + for i in 1..n { + let (mut min_c, mut min_v) = (0usize, INF); + for c in 0..12 { + if best[c] < min_v { + min_v = best[c]; + min_c = c; + } + } + let mut nb_ = [INF; 12]; + for c in 0..12 { + if cost[i][c] >= INF { + continue; + } + let stay = best[c]; + let switch = min_v + new_sec; + if stay <= switch { + nb_[c] = stay + cost[i][c]; + back[i][c] = c as u8; + } else { + nb_[c] = switch + cost[i][c]; + back[i][c] = min_c as u8; + } + } + best = nb_; + } + let mut cbs = vec![0u8; n]; + let mut c = (0..12).min_by_key(|&c| best[c]).unwrap_or(0); + for i in (0..n).rev() { + cbs[i] = c as u8; + c = back[i][c] as usize; + } + cbs +} + +/// Run-length encode a group's codebooks into `(codebook, length)` sections, and sum the spectral +/// bits of its non-zero sections from the cost rows. `base` is the index of the group's first band. +fn group_sections(cost: &[[u32; 12]], maxq: &[i32], cbs: &[u8], base: usize) -> (Vec<(u8, usize)>, u32) { + let mut secs: Vec<(u8, usize)> = Vec::new(); + let mut spectral_bits = 0u32; + for (i, &c) in cbs.iter().enumerate() { + match secs.last_mut() { + Some((lc, len)) if *lc == c => *len += 1, + _ => secs.push((c, 1)), + } + if c != ZERO_HCB { + // the cost row carries one extra bit for an all-zero band (its scalefactor difference) + spectral_bits += cost[i][c as usize] - (maxq[base + i] == 0) as u32; + } + } + (secs, spectral_bits) +} + +/// Section selection: a codebook per band, the sections of each group and the bits they cost +/// `(codebooks, sections, spectral_bits, section_bits)`. +fn select_sections(ch: &ChannelInput, cache: &mut QuantCache, sf: &[i32], q: &[i32], maxq: &[i32]) -> (Vec, Vec>, u32, u32) { + let info = &ch.info; + let (sect_bits, esc) = if info.is_short() { (3u32, 7usize) } else { (5u32, 31usize) }; + let mut cb = vec![ZERO_HCB; ch.bands.len()]; + let mut sections = Vec::with_capacity(info.num_groups); + let mut spectral_bits = 0u32; + let mut section_bits = 0u32; + let n = info.max_sfb; + for g in 0..info.num_groups { + let base = g * n; + if n == 0 { + sections.push(Vec::new()); + continue; + } + let cost = codebook_costs(ch, cache, sf, q, maxq, base, n); + cb[base..base + n].copy_from_slice(&best_codebooks(&cost, 4 + sect_bits)); + let (secs, bits) = group_sections(&cost, maxq, &cb[base..base + n], base); + spectral_bits += bits; + for &(_, len) in &secs { + section_bits += 4 + sect_bits * (len / esc + 1) as u32; + } + sections.push(secs); + } + (cb, sections, spectral_bits, section_bits) +} + +/// Quantise one channel at rate-distortion multiplier `lambda` and select sections. +pub fn quantise_channel(ch: &ChannelInput, cache: &mut QuantCache, lambda: f32) -> QuantChannel { + let nb = ch.bands.len(); + let mut sf = vec![-1i32; nb]; + for (i, b) in ch.bands.iter().enumerate() { + let allowed = lambda * b.thr; + if b.max34 <= 0.0 || b.en <= allowed { + continue; + } + sf[i] = search_sf(ch, i, &mut cache.dist, allowed); + } + // Scalefactor differences between coded bands must stay within ±60: lower (refine) offenders, + // then quantise; repeat if a band unexpectedly quantised to all zeros (it leaves the chain). + let mut q = vec![0i32; ch.x.len()]; + let mut maxq = vec![0i32; nb]; + loop { + limit_sf_steps(&mut sf); + if !quantise_bands(ch, cache, &mut sf, &mut q, &mut maxq) { + break; + } + } + + let (cb, sections, spectral_bits, section_bits) = select_sections(ch, cache, &sf, &q, &maxq); + + // Scalefactors: global gain is the first coded band's value; bands coded with a non-zero + // codebook but all-zero values reuse the previous value (difference 0). + let first = (0..nb).find(|&i| sf[i] >= 0).map(|i| sf[i]).unwrap_or(100); + let mut prev = first; + let mut sf_bits = 0u32; + for i in 0..nb { + if cb[i] == ZERO_HCB { + continue; + } + if sf[i] < 0 { + sf[i] = prev; + } + sf_bits += huffman::sf_bits(sf[i] - prev); + prev = sf[i]; + } + let tns_bits = 1 + ch.tns.as_ref().map(|t| t.bit_count(&ch.info)).unwrap_or(0); + let bits = 8 + section_bits + sf_bits + 1 + tns_bits + 1 + spectral_bits; + QuantChannel { q, sf, cb, sections, global_gain: first, bits } +} + +/// Write an `individual_channel_stream`; `write_info` is false inside a common-window CPE. +pub fn write_ics(bw: &mut BitWriter, ch: &ChannelInput, qc: &QuantChannel, write_info: bool) { + let info = &ch.info; + bw.write_bits(qc.global_gain as u32, 8); + if write_info { + info.write(bw); + } + let (sect_bits, esc) = if info.is_short() { (3u32, 7usize) } else { (5u32, 31usize) }; + for secs in &qc.sections { + for &(c, len) in secs { + bw.write_bits(c as u32, 4); + let mut l = len; + while l >= esc { + bw.write_bits(esc as u32, sect_bits); + l -= esc; + } + bw.write_bits(l as u32, sect_bits); + } + } + let mut prev = qc.global_gain; + for i in 0..ch.bands.len() { + if qc.cb[i] == ZERO_HCB { + continue; + } + huffman::write_sf(bw, qc.sf[i] - prev); + prev = qc.sf[i]; + } + bw.write_bits(0, 1); // pulse_data_present + match &ch.tns { + Some(t) => { + bw.write_bits(1, 1); + t.write(bw, info); + } + None => bw.write_bits(0, 1), + } + bw.write_bits(0, 1); // gain_control_data_present + for (i, b) in ch.bands.iter().enumerate() { + if qc.cb[i] != ZERO_HCB { + huffman::write_spectral(bw, qc.cb[i], &qc.q[b.start..b.end]); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ics::WindowSequence; + use crate::mdct::WindowShape; + use crate::tables::swb_offsets_long; + use filmcraft_bitstream::BitWriter; + + #[test] + fn bit_count_matches_written_bits() { + let swb = swb_offsets_long(4); + let info = IcsInfo { window_sequence: WindowSequence::OnlyLong, window_shape: WindowShape::Sine, max_sfb: 40, ..Default::default() }; + let spec: Vec = (0..1024).map(|i| ((i * 7919 % 1000) as f32 - 500.0) * (1000.0 / (i as f32 + 10.0))).collect(); + let thr = vec![vec![1000.0f32; 64]]; + let ch = ChannelInput::new(info, swb, &spec, &thr, None); + for lambda in [0.01f32, 1.0, 100.0, 1e5] { + let mut cache = QuantCache::new(&ch); + let qc = quantise_channel(&ch, &mut cache, lambda); + let mut bw = BitWriter::new(); + write_ics(&mut bw, &ch, &qc, false); + assert_eq!(bw.bit_len() as u32, qc.bits, "lambda {lambda}"); + } + } + + #[test] + fn reused_cache_gives_the_same_result_as_a_fresh_one() { + let swb = swb_offsets_long(4); + let info = IcsInfo { window_sequence: WindowSequence::OnlyLong, window_shape: WindowShape::Sine, max_sfb: 40, ..Default::default() }; + let spec: Vec = (0..1024).map(|i| ((i * 7919 % 1000) as f32 - 500.0) * (1000.0 / (i as f32 + 10.0))).collect(); + let thr = vec![vec![1000.0f32; 64]]; + let ch = ChannelInput::new(info, swb, &spec, &thr, None); + let mut shared = QuantCache::new(&ch); + // up and down, like the rate loop, with repeats + for lambda in [1.0f32, 4.0, 16.0, 0.25, 1.0, 100.0, 3.0, 3.0, 1e5, 0.01, 1.0] { + let a = quantise_channel(&ch, &mut shared, lambda); + let b = quantise_channel(&ch, &mut QuantCache::new(&ch), lambda); + assert_eq!( + (&a.q, &a.sf, &a.cb, &a.sections, a.global_gain, a.bits), + (&b.q, &b.sf, &b.cb, &b.sections, b.global_gain, b.bits), + "lambda {lambda}" + ); + } + } +} diff --git a/crates/aac/src/huffman.rs b/crates/aac/src/huffman.rs new file mode 100644 index 0000000000..02628b0a3c --- /dev/null +++ b/crates/aac/src/huffman.rs @@ -0,0 +1,425 @@ +//! Huffman coding of spectral values and scalefactors (ISO/IEC 14496-3 §4.6.3, Annex 4.A). + +use std::sync::OnceLock; + +use filmcraft_bitstream::{BitReader, BitWriter}; + +use crate::huffman_tables::{ + SCALEFACTOR, SPECTRUM_1, SPECTRUM_2, SPECTRUM_3, SPECTRUM_4, SPECTRUM_5, SPECTRUM_6, SPECTRUM_7, SPECTRUM_8, SPECTRUM_9, SPECTRUM_10, SPECTRUM_11, +}; +use crate::{Error, Result}; + +/// `ZERO_HCB`: band quantised to zero, nothing transmitted. +pub const ZERO_HCB: u8 = 0; +/// `ESC_HCB`: codebook 11 with escape sequences. +pub const ESC_HCB: u8 = 11; +/// `NOISE_HCB`: perceptual noise substitution. +pub const NOISE_HCB: u8 = 13; +/// `INTENSITY_HCB2`: intensity stereo, out of phase. +pub const INTENSITY_HCB2: u8 = 14; +/// `INTENSITY_HCB`: intensity stereo, in phase. +pub const INTENSITY_HCB: u8 = 15; + +/// Parameters of one spectrum codebook. +pub struct Book { + /// Values per codeword (4 = quad, 2 = pair). + pub dim: usize, + /// Signed values in the codeword (no separate sign bits). + pub signed: bool, + /// Largest absolute value (16 = escape for codebook 11). + pub lav: u32, + /// Radix of the index (`2*lav+1` signed, `lav+1` unsigned). + pub modulo: u32, + pub codes: &'static [(u32, u8)], +} + +static BOOKS: [Book; 11] = [ + Book { dim: 4, signed: true, lav: 1, modulo: 3, codes: &SPECTRUM_1 }, + Book { dim: 4, signed: true, lav: 1, modulo: 3, codes: &SPECTRUM_2 }, + Book { dim: 4, signed: false, lav: 2, modulo: 3, codes: &SPECTRUM_3 }, + Book { dim: 4, signed: false, lav: 2, modulo: 3, codes: &SPECTRUM_4 }, + Book { dim: 2, signed: true, lav: 4, modulo: 9, codes: &SPECTRUM_5 }, + Book { dim: 2, signed: true, lav: 4, modulo: 9, codes: &SPECTRUM_6 }, + Book { dim: 2, signed: false, lav: 7, modulo: 8, codes: &SPECTRUM_7 }, + Book { dim: 2, signed: false, lav: 7, modulo: 8, codes: &SPECTRUM_8 }, + Book { dim: 2, signed: false, lav: 12, modulo: 13, codes: &SPECTRUM_9 }, + Book { dim: 2, signed: false, lav: 12, modulo: 13, codes: &SPECTRUM_10 }, + Book { dim: 2, signed: false, lav: 16, modulo: 17, codes: &SPECTRUM_11 }, +]; + +/// Spectrum codebook 1..=11. +#[inline] +pub fn book(cb: u8) -> &'static Book { + &BOOKS[(cb as usize).clamp(1, 11) - 1] +} + +/// Bits resolved by the first table level. +const L1: u32 = 9; +/// Flag of a first-level entry that points to a sub-table. +const SUB: u32 = 1 << 31; + +/// Two-level decoding table for one prefix code. A leaf entry is `len << 24 | symbol`; a first-level +/// entry with `SUB` set is `SUB | offset << 5 | width` and selects a sub-table of `1 << width` +/// leaves indexed by the next `width` bits. An entry of 0 is a bit pattern that no codeword starts +/// with (the codebooks are complete, so this does not occur). +struct Lut { + t: Vec, +} + +impl Lut { + fn build(codes: &[(u32, u8)]) -> Lut { + let mut t = vec![0u32; 1 << L1]; + // sub-table width per first-level prefix + let mut width = vec![0u32; 1 << L1]; + for &(code, len) in codes { + let len = len as u32; + if len > L1 { + let p = (code >> (len - L1)) as usize; + width[p] = width[p].max(len - L1); + } + } + for (p, &w) in width.iter().enumerate() { + if w > 0 { + t[p] = SUB | (t.len() as u32) << 5 | w; + t.resize(t.len() + (1 << w), 0); + } + } + for (sym, &(code, len)) in codes.iter().enumerate() { + let len = len as u32; + let leaf = len << 24 | sym as u32; + if len <= L1 { + let first = (code << (L1 - len)) as usize; + t[first..first + (1 << (L1 - len))].fill(leaf); + } else { + let e = t[(code >> (len - L1)) as usize]; + let (off, w) = (((e & !SUB) >> 5) as usize, e & 31); + let rest = code & ((1 << (len - L1)) - 1); + let first = off + (rest << (w - (len - L1))) as usize; + t[first..first + (1 << (w - (len - L1)))].fill(leaf); + } + } + Lut { t } + } + + /// The symbol of the codeword at the reader position; `Eof` if the codeword is cut off. + #[inline] + fn decode(&self, br: &mut BitReader) -> Result { + let bits = peek32(br); + let mut e = self.t[(bits >> (32 - L1)) as usize]; + if e & SUB != 0 { + let w = e & 31; + e = self.t[((e & !SUB) >> 5) as usize + ((bits << L1) >> (32 - w)) as usize]; + } + let len = e >> 24; + if len == 0 { + return Err(Error::Bitstream("invalid Huffman codeword")); + } + br.skip(len as usize).map_err(|_| Error::Eof)?; + Ok(e & 0xFF_FFFF) + } +} + +/// The next 32 bits (zero padded past the end), from one 8-byte load when possible. +#[inline] +fn peek32(br: &BitReader) -> u32 { + let pos = br.position(); + let byte = pos >> 3; + match br.data().get(byte..byte.saturating_add(8)).and_then(|b| <[u8; 8]>::try_from(b).ok()) { + Some(b) => ((u64::from_be_bytes(b) << (pos & 7)) >> 32) as u32, + None => br.peek(32), + } +} + +fn luts() -> &'static [Lut; 12] { + static LUTS: OnceLock<[Lut; 12]> = OnceLock::new(); + LUTS.get_or_init(|| std::array::from_fn(|i| if i == 0 { Lut::build(&SCALEFACTOR) } else { Lut::build(BOOKS[i - 1].codes) })) +} + +/// Decode one scalefactor difference (`-60..=60`). +#[inline] +pub fn decode_sf(br: &mut BitReader) -> Result { + Ok(luts()[0].decode(br)? as i32 - 60) +} + +/// Write one scalefactor difference (`-60..=60`). +#[inline] +pub fn write_sf(bw: &mut BitWriter, diff: i32) { + let (code, len) = SCALEFACTOR[(diff + 60).clamp(0, 120) as usize]; + bw.write_bits(code, len as u32); +} + +/// Bits of one scalefactor difference. +#[inline] +pub fn sf_bits(diff: i32) -> u32 { + SCALEFACTOR[(diff + 60).clamp(0, 120) as usize].1 as u32 +} + +/// Decode one codeword of codebook `cb` (1..=11) into `out[..dim]`, including sign bits and escapes. +pub fn decode_spectral(br: &mut BitReader, cb: u8, out: &mut [i32]) -> Result<()> { + let b = book(cb); + let mut idx = luts()[cb as usize].decode(br)?; + for i in (0..b.dim).rev() { + out[i] = (idx % b.modulo) as i32; + idx /= b.modulo; + } + if b.signed { + for v in out[..b.dim].iter_mut() { + *v -= b.lav as i32; + } + return Ok(()); + } + for v in out[..b.dim].iter_mut() { + if *v != 0 && br.read_bits(1)? == 1 { + *v = -*v; + } + } + if cb == ESC_HCB { + for v in out[..b.dim].iter_mut() { + if v.abs() == 16 { + let mut n = 0u32; + while br.read_bits(1)? == 1 { + n += 1; + if n > 8 { + return Err(Error::Bitstream("escape sequence too long")); + } + } + let mag = (1i32 << (n + 4)) + br.read_bits(n + 4)? as i32; + *v = if *v < 0 { -mag } else { mag }; + } + } + } + Ok(()) +} + +/// Codebook index of the (sign-stripped for unsigned books) values. +#[inline] +fn index_of(b: &Book, vals: &[i32]) -> usize { + let mut idx = 0u32; + for &v in &vals[..b.dim] { + let d = if b.signed { (v + b.lav as i32) as u32 } else { v.unsigned_abs().min(b.lav) }; + idx = idx * b.modulo + d; + } + idx as usize +} + +#[inline] +fn escape_bits(mag: u32) -> u32 { + // mag >= 16: N ones, a zero, N+4 bits where 2^(N+4) <= mag < 2^(N+5). + let n = 31 - mag.leading_zeros() - 4; + 2 * n + 5 +} + +/// Bits needed to code `vals` (a multiple of the codebook dimension) with codebook `cb` (1..=11). +/// The values must fit the codebook (`|v| <= lav`, or `<= 8191` for codebook 11). +pub fn count_bits(cb: u8, vals: &[i32]) -> u32 { + let b = book(cb); + let mut bits = 0u32; + for chunk in vals.chunks_exact(b.dim) { + bits += b.codes[index_of(b, chunk)].1 as u32; + if !b.signed { + for &v in chunk { + if v != 0 { + bits += 1; + if cb == ESC_HCB && v.unsigned_abs() >= 16 { + bits += escape_bits(v.unsigned_abs()); + } + } + } + } + } + bits +} + +/// Bits for codebooks `cb` and `cb + 1` in one pass. `cb` must be odd and below 11: each such pair +/// shares dimension, modulus and signedness, so the index is computed once and only the length table +/// differs. +pub fn count_bits_pair(cb: u8, vals: &[i32]) -> (u32, u32) { + debug_assert!(cb % 2 == 1 && cb < ESC_HCB); + let (b, b2) = (book(cb), book(cb + 1)); + let (mut x, mut y) = (0u32, 0u32); + for chunk in vals.chunks_exact(b.dim) { + let i = index_of(b, chunk); + x += b.codes[i].1 as u32; + y += b2.codes[i].1 as u32; + if !b.signed { + // sign bits (no escapes below codebook 11) + let nz = chunk.iter().filter(|&&v| v != 0).count() as u32; + x += nz; + y += nz; + } + } + (x, y) +} + +/// Write `vals` with codebook `cb` (1..=11). +pub fn write_spectral(bw: &mut BitWriter, cb: u8, vals: &[i32]) { + let b = book(cb); + for chunk in vals.chunks_exact(b.dim) { + let (code, len) = b.codes[index_of(b, chunk)]; + bw.write_bits(code, len as u32); + if b.signed { + continue; + } + for &v in chunk { + if v != 0 { + bw.write_bits((v < 0) as u32, 1); + } + } + if cb == ESC_HCB { + for &v in chunk { + let mag = v.unsigned_abs(); + if mag >= 16 { + let n = 31 - mag.leading_zeros() - 4; + for _ in 0..n { + bw.write_bits(1, 1); + } + bw.write_bits(0, 1); + bw.write_bits(mag - (1 << (n + 4)), n + 4); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn check_complete(codes: &[(u32, u8)]) { + // Kraft sum exactly one and prefix-free. + let max = codes.iter().map(|c| c.1).max().unwrap() as u32; + let mut kraft = 0u64; + for &(_, l) in codes { + kraft += 1u64 << (max - l as u32); + } + assert_eq!(kraft, 1u64 << max); + for (i, &(ci, li)) in codes.iter().enumerate() { + for (j, &(cj, lj)) in codes.iter().enumerate() { + if i != j && li <= lj { + assert_ne!(cj >> (lj - li), ci, "codeword {i} is a prefix of {j}"); + } + } + } + } + + #[test] + fn codebooks_are_complete_prefix_codes() { + check_complete(&SCALEFACTOR); + assert_eq!(SCALEFACTOR[60], (0, 1)); + for b in &BOOKS { + assert_eq!(b.codes.len() as u32, b.modulo.pow(b.dim as u32)); + check_complete(b.codes); + } + } + + /// Every codeword, followed by arbitrary bits, decodes to its own symbol and length. + #[test] + fn lut_decodes_every_codeword() { + let mut seed = 7u32; + for (i, lut) in luts().iter().enumerate() { + let codes: &[(u32, u8)] = if i == 0 { &SCALEFACTOR } else { BOOKS[i - 1].codes }; + for (sym, &(code, len)) in codes.iter().enumerate() { + for _ in 0..3 { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + let mut bw = BitWriter::new(); + bw.write_bits(code, len as u32); + bw.write_bits(seed >> 8, 24); + let data = bw.finish(); + let mut br = BitReader::new(&data); + assert_eq!(lut.decode(&mut br).unwrap(), sym as u32, "book {i}"); + assert_eq!(br.position(), len as usize); + } + } + } + } + + /// A codeword cut off by the end of the data is `Eof`, also at an unaligned position. + #[test] + fn truncated_codeword_is_eof() { + for (i, lut) in luts().iter().enumerate() { + let codes: &[(u32, u8)] = if i == 0 { &SCALEFACTOR } else { BOOKS[i - 1].codes }; + let &(code, len) = codes.iter().max_by_key(|c| c.1).unwrap(); + for lead in 0..8u32 { + let mut bw = BitWriter::new(); + bw.write_bits(0, lead); + bw.write_bits(code, len as u32); + let total = lead as usize + len as usize; + let data = bw.finish(); + // drop the last byte if that cuts the codeword, else the codeword is whole + let short = &data[..data.len() - 1]; + let mut br = BitReader::new(short); + br.skip(lead as usize).unwrap(); + let r = lut.decode(&mut br); + if short.len() * 8 < total { + assert!(matches!(r, Err(Error::Eof)), "book {i} lead {lead}"); + } else { + assert!(r.is_ok()); + } + } + } + } + + #[test] + fn spectral_roundtrip_every_book() { + for cb in 1..=11u8 { + let b = book(cb); + let lim = if cb == 11 { 8191 } else { b.lav as i32 }; + let mut vals = Vec::new(); + let mut seed = 12345u32; + for _ in 0..400 { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + let r = (seed >> 8) as i32; + let v = if cb == 11 && r % 3 == 0 { r % (lim + 1) } else { r % (b.lav.min(16) as i32 + 1) }; + let v = v.min(lim); + vals.push(if b.signed || r & 1 == 0 { v } else { -v }); + } + if b.signed { + for v in vals.iter_mut() { + *v = (*v).clamp(-(b.lav as i32), b.lav as i32); + } + } + let mut bw = BitWriter::new(); + write_spectral(&mut bw, cb, &vals); + assert_eq!(bw.bit_len() as u32, count_bits(cb, &vals)); + let data = bw.finish(); + let mut br = BitReader::new(&data); + let mut out = vec![0i32; vals.len()]; + for c in out.chunks_mut(b.dim) { + decode_spectral(&mut br, cb, c).unwrap(); + } + assert_eq!(out, vals, "codebook {cb}"); + } + } + + #[test] + fn pair_count_matches_single_count() { + let mut seed = 99u32; + for cb in (1..11u8).step_by(2) { + let b = book(cb); + for round in 0..40 { + let mut vals = Vec::new(); + for _ in 0..(8 * b.dim) { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + let m = ((seed >> 8) % (b.lav + 1)) as i32; + let m = if round % 4 == 0 { 0 } else { m }; + let neg = if b.signed { seed & 0x8000_0000 != 0 } else { seed & 1 == 1 }; + vals.push(if neg { -m } else { m }); + } + assert_eq!(count_bits_pair(cb, &vals), (count_bits(cb, &vals), count_bits(cb + 1, &vals)), "codebook {cb}"); + } + } + } + + #[test] + fn scalefactor_roundtrip() { + let mut bw = BitWriter::new(); + for d in -60..=60 { + write_sf(&mut bw, d); + } + let data = bw.finish(); + let mut br = BitReader::new(&data); + for d in -60..=60 { + assert_eq!(decode_sf(&mut br).unwrap(), d); + } + } +} diff --git a/crates/aac/src/huffman_tables.rs b/crates/aac/src/huffman_tables.rs new file mode 100644 index 0000000000..e48f5fadcb --- /dev/null +++ b/crates/aac/src/huffman_tables.rs @@ -0,0 +1,1404 @@ +//! AAC Huffman codebooks (ISO/IEC 14496-3 §4.A.1, spectrum codebooks 1–11 and the scalefactor codebook). +//! +//! Each table is indexed by the codebook index defined in the standard and stores `(codeword, length)`. +//! Generated data: derived by black-box probing of an external decoder used purely as a test oracle +//! (every table is a complete prefix code — Kraft sum exactly 1 — covering every index once; see +//! `tests::codebooks_are_complete_prefix_codes`). + +pub(crate) static SCALEFACTOR: [(u32, u8); 121] = [ + (0x3ffe8, 18), + (0x3ffe6, 18), + (0x3ffe7, 18), + (0x3ffe5, 18), + (0x7fff5, 19), + (0x7fff1, 19), + (0x7ffed, 19), + (0x7fff6, 19), + (0x7ffee, 19), + (0x7ffef, 19), + (0x7fff0, 19), + (0x7fffc, 19), + (0x7fffd, 19), + (0x7ffff, 19), + (0x7fffe, 19), + (0x7fff7, 19), + (0x7fff8, 19), + (0x7fffb, 19), + (0x7fff9, 19), + (0x3ffe4, 18), + (0x7fffa, 19), + (0x3ffe3, 18), + (0x1ffef, 17), + (0x1fff0, 17), + (0xfff5, 16), + (0x1ffee, 17), + (0xfff2, 16), + (0xfff3, 16), + (0xfff4, 16), + (0xfff1, 16), + (0x7ff6, 15), + (0x7ff7, 15), + (0x3ff9, 14), + (0x3ff5, 14), + (0x3ff7, 14), + (0x3ff3, 14), + (0x3ff6, 14), + (0x3ff2, 14), + (0x1ff7, 13), + (0x1ff5, 13), + (0xff9, 12), + (0xff7, 12), + (0xff6, 12), + (0x7f9, 11), + (0xff4, 12), + (0x7f8, 11), + (0x3f9, 10), + (0x3f7, 10), + (0x3f5, 10), + (0x1f8, 9), + (0x1f7, 9), + (0xfa, 8), + (0xf8, 8), + (0xf6, 8), + (0x79, 7), + (0x3a, 6), + (0x38, 6), + (0x1a, 5), + (0xb, 4), + (0x4, 3), + (0x0, 1), + (0xa, 4), + (0xc, 4), + (0x1b, 5), + (0x39, 6), + (0x3b, 6), + (0x78, 7), + (0x7a, 7), + (0xf7, 8), + (0xf9, 8), + (0x1f6, 9), + (0x1f9, 9), + (0x3f4, 10), + (0x3f6, 10), + (0x3f8, 10), + (0x7f5, 11), + (0x7f4, 11), + (0x7f6, 11), + (0x7f7, 11), + (0xff5, 12), + (0xff8, 12), + (0x1ff4, 13), + (0x1ff6, 13), + (0x1ff8, 13), + (0x3ff8, 14), + (0x3ff4, 14), + (0xfff0, 16), + (0x7ff4, 15), + (0xfff6, 16), + (0x7ff5, 15), + (0x3ffe2, 18), + (0x7ffd9, 19), + (0x7ffda, 19), + (0x7ffdb, 19), + (0x7ffdc, 19), + (0x7ffdd, 19), + (0x7ffde, 19), + (0x7ffd8, 19), + (0x7ffd2, 19), + (0x7ffd3, 19), + (0x7ffd4, 19), + (0x7ffd5, 19), + (0x7ffd6, 19), + (0x7fff2, 19), + (0x7ffdf, 19), + (0x7ffe7, 19), + (0x7ffe8, 19), + (0x7ffe9, 19), + (0x7ffea, 19), + (0x7ffeb, 19), + (0x7ffe6, 19), + (0x7ffe0, 19), + (0x7ffe1, 19), + (0x7ffe2, 19), + (0x7ffe3, 19), + (0x7ffe4, 19), + (0x7ffe5, 19), + (0x7ffd7, 19), + (0x7ffec, 19), + (0x7fff4, 19), + (0x7fff3, 19), +]; + +pub(crate) static SPECTRUM_1: [(u32, u8); 81] = [ + (0x7f8, 11), + (0x1f1, 9), + (0x7fd, 11), + (0x3f5, 10), + (0x68, 7), + (0x3f0, 10), + (0x7f7, 11), + (0x1ec, 9), + (0x7f5, 11), + (0x3f1, 10), + (0x72, 7), + (0x3f4, 10), + (0x74, 7), + (0x11, 5), + (0x76, 7), + (0x1eb, 9), + (0x6c, 7), + (0x3f6, 10), + (0x7fc, 11), + (0x1e1, 9), + (0x7f1, 11), + (0x1f0, 9), + (0x61, 7), + (0x1f6, 9), + (0x7f2, 11), + (0x1ea, 9), + (0x7fb, 11), + (0x1f2, 9), + (0x69, 7), + (0x1ed, 9), + (0x77, 7), + (0x17, 5), + (0x6f, 7), + (0x1e6, 9), + (0x64, 7), + (0x1e5, 9), + (0x67, 7), + (0x15, 5), + (0x62, 7), + (0x12, 5), + (0x0, 1), + (0x14, 5), + (0x65, 7), + (0x16, 5), + (0x6d, 7), + (0x1e9, 9), + (0x63, 7), + (0x1e4, 9), + (0x6b, 7), + (0x13, 5), + (0x71, 7), + (0x1e3, 9), + (0x70, 7), + (0x1f3, 9), + (0x7fe, 11), + (0x1e7, 9), + (0x7f3, 11), + (0x1ef, 9), + (0x60, 7), + (0x1ee, 9), + (0x7f0, 11), + (0x1e2, 9), + (0x7fa, 11), + (0x3f3, 10), + (0x6a, 7), + (0x1e8, 9), + (0x75, 7), + (0x10, 5), + (0x73, 7), + (0x1f4, 9), + (0x6e, 7), + (0x3f7, 10), + (0x7f6, 11), + (0x1e0, 9), + (0x7f9, 11), + (0x3f2, 10), + (0x66, 7), + (0x1f5, 9), + (0x7ff, 11), + (0x1f7, 9), + (0x7f4, 11), +]; + +pub(crate) static SPECTRUM_2: [(u32, u8); 81] = [ + (0x1f3, 9), + (0x6f, 7), + (0x1fd, 9), + (0xeb, 8), + (0x23, 6), + (0xea, 8), + (0x1f7, 9), + (0xe8, 8), + (0x1fa, 9), + (0xf2, 8), + (0x2d, 6), + (0x70, 7), + (0x20, 6), + (0x6, 5), + (0x2b, 6), + (0x6e, 7), + (0x28, 6), + (0xe9, 8), + (0x1f9, 9), + (0x66, 7), + (0xf8, 8), + (0xe7, 8), + (0x1b, 6), + (0xf1, 8), + (0x1f4, 9), + (0x6b, 7), + (0x1f5, 9), + (0xec, 8), + (0x2a, 6), + (0x6c, 7), + (0x2c, 6), + (0xa, 5), + (0x27, 6), + (0x67, 7), + (0x1a, 6), + (0xf5, 8), + (0x24, 6), + (0x8, 5), + (0x1f, 6), + (0x9, 5), + (0x0, 3), + (0x7, 5), + (0x1d, 6), + (0xb, 5), + (0x30, 6), + (0xef, 8), + (0x1c, 6), + (0x64, 7), + (0x1e, 6), + (0xc, 5), + (0x29, 6), + (0xf3, 8), + (0x2f, 6), + (0xf0, 8), + (0x1fc, 9), + (0x71, 7), + (0x1f2, 9), + (0xf4, 8), + (0x21, 6), + (0xe6, 8), + (0xf7, 8), + (0x68, 7), + (0x1f8, 9), + (0xee, 8), + (0x22, 6), + (0x65, 7), + (0x31, 6), + (0x2, 4), + (0x26, 6), + (0xed, 8), + (0x25, 6), + (0x6a, 7), + (0x1fb, 9), + (0x72, 7), + (0x1fe, 9), + (0x69, 7), + (0x2e, 6), + (0xf6, 8), + (0x1ff, 9), + (0x6d, 7), + (0x1f6, 9), +]; + +pub(crate) static SPECTRUM_3: [(u32, u8); 81] = [ + (0x0, 1), + (0x9, 4), + (0xef, 8), + (0xb, 4), + (0x19, 5), + (0xf0, 8), + (0x1eb, 9), + (0x1e6, 9), + (0x3f2, 10), + (0xa, 4), + (0x35, 6), + (0x1ef, 9), + (0x34, 6), + (0x37, 6), + (0x1e9, 9), + (0x1ed, 9), + (0x1e7, 9), + (0x3f3, 10), + (0x1ee, 9), + (0x3ed, 10), + (0x1ffa, 13), + (0x1ec, 9), + (0x1f2, 9), + (0x7f9, 11), + (0x7f8, 11), + (0x3f8, 10), + (0xff8, 12), + (0x8, 4), + (0x38, 6), + (0x3f6, 10), + (0x36, 6), + (0x75, 7), + (0x3f1, 10), + (0x3eb, 10), + (0x3ec, 10), + (0xff4, 12), + (0x18, 5), + (0x76, 7), + (0x7f4, 11), + (0x39, 6), + (0x74, 7), + (0x3ef, 10), + (0x1f3, 9), + (0x1f4, 9), + (0x7f6, 11), + (0x1e8, 9), + (0x3ea, 10), + (0x1ffc, 13), + (0xf2, 8), + (0x1f1, 9), + (0xffb, 12), + (0x3f5, 10), + (0x7f3, 11), + (0xffc, 12), + (0xee, 8), + (0x3f7, 10), + (0x7ffe, 15), + (0x1f0, 9), + (0x7f5, 11), + (0x7ffd, 15), + (0x1ffb, 13), + (0x3ffa, 14), + (0xffff, 16), + (0xf1, 8), + (0x3f0, 10), + (0x3ffc, 14), + (0x1ea, 9), + (0x3ee, 10), + (0x3ffb, 14), + (0xff6, 12), + (0xffa, 12), + (0x7ffc, 15), + (0x7f2, 11), + (0xff5, 12), + (0xfffe, 16), + (0x3f4, 10), + (0x7f7, 11), + (0x7ffb, 15), + (0xff7, 12), + (0xff9, 12), + (0x7ffa, 15), +]; + +pub(crate) static SPECTRUM_4: [(u32, u8); 81] = [ + (0x7, 4), + (0x16, 5), + (0xf6, 8), + (0x18, 5), + (0x8, 4), + (0xef, 8), + (0x1ef, 9), + (0xf3, 8), + (0x7f8, 11), + (0x19, 5), + (0x17, 5), + (0xed, 8), + (0x15, 5), + (0x1, 4), + (0xe2, 8), + (0xf0, 8), + (0x70, 7), + (0x3f0, 10), + (0x1ee, 9), + (0xf1, 8), + (0x7fa, 11), + (0xee, 8), + (0xe4, 8), + (0x3f2, 10), + (0x7f6, 11), + (0x3ef, 10), + (0x7fd, 11), + (0x5, 4), + (0x14, 5), + (0xf2, 8), + (0x9, 4), + (0x4, 4), + (0xe5, 8), + (0xf4, 8), + (0xe8, 8), + (0x3f4, 10), + (0x6, 4), + (0x2, 4), + (0xe7, 8), + (0x3, 4), + (0x0, 4), + (0x6b, 7), + (0xe3, 8), + (0x69, 7), + (0x1f3, 9), + (0xeb, 8), + (0xe6, 8), + (0x3f6, 10), + (0x6e, 7), + (0x6a, 7), + (0x1f4, 9), + (0x3ec, 10), + (0x1f0, 9), + (0x3f9, 10), + (0xf5, 8), + (0xec, 8), + (0x7fb, 11), + (0xea, 8), + (0x6f, 7), + (0x3f7, 10), + (0x7f9, 11), + (0x3f3, 10), + (0xfff, 12), + (0xe9, 8), + (0x6d, 7), + (0x3f8, 10), + (0x6c, 7), + (0x68, 7), + (0x1f5, 9), + (0x3ee, 10), + (0x1f2, 9), + (0x7f4, 11), + (0x7f7, 11), + (0x3f1, 10), + (0xffe, 12), + (0x3ed, 10), + (0x1f1, 9), + (0x7f5, 11), + (0x7fe, 11), + (0x3f5, 10), + (0x7fc, 11), +]; + +pub(crate) static SPECTRUM_5: [(u32, u8); 81] = [ + (0x1fff, 13), + (0xff7, 12), + (0x7f4, 11), + (0x7e8, 11), + (0x3f1, 10), + (0x7ee, 11), + (0x7f9, 11), + (0xff8, 12), + (0x1ffd, 13), + (0xffd, 12), + (0x7f1, 11), + (0x3e8, 10), + (0x1e8, 9), + (0xf0, 8), + (0x1ec, 9), + (0x3ee, 10), + (0x7f2, 11), + (0xffa, 12), + (0xff4, 12), + (0x3ef, 10), + (0x1f2, 9), + (0xe8, 8), + (0x70, 7), + (0xec, 8), + (0x1f0, 9), + (0x3ea, 10), + (0x7f3, 11), + (0x7eb, 11), + (0x1eb, 9), + (0xea, 8), + (0x1a, 5), + (0x8, 4), + (0x19, 5), + (0xee, 8), + (0x1ef, 9), + (0x7ed, 11), + (0x3f0, 10), + (0xf2, 8), + (0x73, 7), + (0xb, 4), + (0x0, 1), + (0xa, 4), + (0x71, 7), + (0xf3, 8), + (0x7e9, 11), + (0x7ef, 11), + (0x1ee, 9), + (0xef, 8), + (0x18, 5), + (0x9, 4), + (0x1b, 5), + (0xeb, 8), + (0x1e9, 9), + (0x7ec, 11), + (0x7f6, 11), + (0x3eb, 10), + (0x1f3, 9), + (0xed, 8), + (0x72, 7), + (0xe9, 8), + (0x1f1, 9), + (0x3ed, 10), + (0x7f7, 11), + (0xff6, 12), + (0x7f0, 11), + (0x3e9, 10), + (0x1ed, 9), + (0xf1, 8), + (0x1ea, 9), + (0x3ec, 10), + (0x7f8, 11), + (0xff9, 12), + (0x1ffc, 13), + (0xffc, 12), + (0xff5, 12), + (0x7ea, 11), + (0x3f3, 10), + (0x3f2, 10), + (0x7f5, 11), + (0xffb, 12), + (0x1ffe, 13), +]; + +pub(crate) static SPECTRUM_6: [(u32, u8); 81] = [ + (0x7fe, 11), + (0x3fd, 10), + (0x1f1, 9), + (0x1eb, 9), + (0x1f4, 9), + (0x1ea, 9), + (0x1f0, 9), + (0x3fc, 10), + (0x7fd, 11), + (0x3f6, 10), + (0x1e5, 9), + (0xea, 8), + (0x6c, 7), + (0x71, 7), + (0x68, 7), + (0xf0, 8), + (0x1e6, 9), + (0x3f7, 10), + (0x1f3, 9), + (0xef, 8), + (0x32, 6), + (0x27, 6), + (0x28, 6), + (0x26, 6), + (0x31, 6), + (0xeb, 8), + (0x1f7, 9), + (0x1e8, 9), + (0x6f, 7), + (0x2e, 6), + (0x8, 4), + (0x4, 4), + (0x6, 4), + (0x29, 6), + (0x6b, 7), + (0x1ee, 9), + (0x1ef, 9), + (0x72, 7), + (0x2d, 6), + (0x2, 4), + (0x0, 4), + (0x3, 4), + (0x2f, 6), + (0x73, 7), + (0x1fa, 9), + (0x1e7, 9), + (0x6e, 7), + (0x2b, 6), + (0x7, 4), + (0x1, 4), + (0x5, 4), + (0x2c, 6), + (0x6d, 7), + (0x1ec, 9), + (0x1f9, 9), + (0xee, 8), + (0x30, 6), + (0x24, 6), + (0x2a, 6), + (0x25, 6), + (0x33, 6), + (0xec, 8), + (0x1f2, 9), + (0x3f8, 10), + (0x1e4, 9), + (0xed, 8), + (0x6a, 7), + (0x70, 7), + (0x69, 7), + (0x74, 7), + (0xf1, 8), + (0x3fa, 10), + (0x7ff, 11), + (0x3f9, 10), + (0x1f6, 9), + (0x1ed, 9), + (0x1f8, 9), + (0x1e9, 9), + (0x1f5, 9), + (0x3fb, 10), + (0x7fc, 11), +]; + +pub(crate) static SPECTRUM_7: [(u32, u8); 64] = [ + (0x0, 1), + (0x5, 3), + (0x37, 6), + (0x74, 7), + (0xf2, 8), + (0x1eb, 9), + (0x3ed, 10), + (0x7f7, 11), + (0x4, 3), + (0xc, 4), + (0x35, 6), + (0x71, 7), + (0xec, 8), + (0xee, 8), + (0x1ee, 9), + (0x1f5, 9), + (0x36, 6), + (0x34, 6), + (0x72, 7), + (0xea, 8), + (0xf1, 8), + (0x1e9, 9), + (0x1f3, 9), + (0x3f5, 10), + (0x73, 7), + (0x70, 7), + (0xeb, 8), + (0xf0, 8), + (0x1f1, 9), + (0x1f0, 9), + (0x3ec, 10), + (0x3fa, 10), + (0xf3, 8), + (0xed, 8), + (0x1e8, 9), + (0x1ef, 9), + (0x3ef, 10), + (0x3f1, 10), + (0x3f9, 10), + (0x7fb, 11), + (0x1ed, 9), + (0xef, 8), + (0x1ea, 9), + (0x1f2, 9), + (0x3f3, 10), + (0x3f8, 10), + (0x7f9, 11), + (0x7fc, 11), + (0x3ee, 10), + (0x1ec, 9), + (0x1f4, 9), + (0x3f4, 10), + (0x3f7, 10), + (0x7f8, 11), + (0xffd, 12), + (0xffe, 12), + (0x7f6, 11), + (0x3f0, 10), + (0x3f2, 10), + (0x3f6, 10), + (0x7fa, 11), + (0x7fd, 11), + (0xffc, 12), + (0xfff, 12), +]; + +pub(crate) static SPECTRUM_8: [(u32, u8); 64] = [ + (0xe, 5), + (0x5, 4), + (0x10, 5), + (0x30, 6), + (0x6f, 7), + (0xf1, 8), + (0x1fa, 9), + (0x3fe, 10), + (0x3, 4), + (0x0, 3), + (0x4, 4), + (0x12, 5), + (0x2c, 6), + (0x6a, 7), + (0x75, 7), + (0xf8, 8), + (0xf, 5), + (0x2, 4), + (0x6, 4), + (0x14, 5), + (0x2e, 6), + (0x69, 7), + (0x72, 7), + (0xf5, 8), + (0x2f, 6), + (0x11, 5), + (0x13, 5), + (0x2a, 6), + (0x32, 6), + (0x6c, 7), + (0xec, 8), + (0xfa, 8), + (0x71, 7), + (0x2b, 6), + (0x2d, 6), + (0x31, 6), + (0x6d, 7), + (0x70, 7), + (0xf2, 8), + (0x1f9, 9), + (0xef, 8), + (0x68, 7), + (0x33, 6), + (0x6b, 7), + (0x6e, 7), + (0xee, 8), + (0xf9, 8), + (0x3fc, 10), + (0x1f8, 9), + (0x74, 7), + (0x73, 7), + (0xed, 8), + (0xf0, 8), + (0xf6, 8), + (0x1f6, 9), + (0x1fd, 9), + (0x3fd, 10), + (0xf3, 8), + (0xf4, 8), + (0xf7, 8), + (0x1f7, 9), + (0x1fb, 9), + (0x1fc, 9), + (0x3ff, 10), +]; + +pub(crate) static SPECTRUM_9: [(u32, u8); 169] = [ + (0x0, 1), + (0x5, 3), + (0x37, 6), + (0xe7, 8), + (0x1de, 9), + (0x3ce, 10), + (0x3d9, 10), + (0x7c8, 11), + (0x7cd, 11), + (0xfc8, 12), + (0xfdd, 12), + (0x1fe4, 13), + (0x1fec, 13), + (0x4, 3), + (0xc, 4), + (0x35, 6), + (0x72, 7), + (0xea, 8), + (0xed, 8), + (0x1e2, 9), + (0x3d1, 10), + (0x3d3, 10), + (0x3e0, 10), + (0x7d8, 11), + (0xfcf, 12), + (0xfd5, 12), + (0x36, 6), + (0x34, 6), + (0x71, 7), + (0xe8, 8), + (0xec, 8), + (0x1e1, 9), + (0x3cf, 10), + (0x3dd, 10), + (0x3db, 10), + (0x7d0, 11), + (0xfc7, 12), + (0xfd4, 12), + (0xfe4, 12), + (0xe6, 8), + (0x70, 7), + (0xe9, 8), + (0x1dd, 9), + (0x1e3, 9), + (0x3d2, 10), + (0x3dc, 10), + (0x7cc, 11), + (0x7ca, 11), + (0x7de, 11), + (0xfd8, 12), + (0xfea, 12), + (0x1fdb, 13), + (0x1df, 9), + (0xeb, 8), + (0x1dc, 9), + (0x1e6, 9), + (0x3d5, 10), + (0x3de, 10), + (0x7cb, 11), + (0x7dd, 11), + (0x7dc, 11), + (0xfcd, 12), + (0xfe2, 12), + (0xfe7, 12), + (0x1fe1, 13), + (0x3d0, 10), + (0x1e0, 9), + (0x1e4, 9), + (0x3d6, 10), + (0x7c5, 11), + (0x7d1, 11), + (0x7db, 11), + (0xfd2, 12), + (0x7e0, 11), + (0xfd9, 12), + (0xfeb, 12), + (0x1fe3, 13), + (0x1fe9, 13), + (0x7c4, 11), + (0x1e5, 9), + (0x3d7, 10), + (0x7c6, 11), + (0x7cf, 11), + (0x7da, 11), + (0xfcb, 12), + (0xfda, 12), + (0xfe3, 12), + (0xfe9, 12), + (0x1fe6, 13), + (0x1ff3, 13), + (0x1ff7, 13), + (0x7d3, 11), + (0x3d8, 10), + (0x3e1, 10), + (0x7d4, 11), + (0x7d9, 11), + (0xfd3, 12), + (0xfde, 12), + (0x1fdd, 13), + (0x1fd9, 13), + (0x1fe2, 13), + (0x1fea, 13), + (0x1ff1, 13), + (0x1ff6, 13), + (0x7d2, 11), + (0x3d4, 10), + (0x3da, 10), + (0x7c7, 11), + (0x7d7, 11), + (0x7e2, 11), + (0xfce, 12), + (0xfdb, 12), + (0x1fd8, 13), + (0x1fee, 13), + (0x3ff0, 14), + (0x1ff4, 13), + (0x3ff2, 14), + (0x7e1, 11), + (0x3df, 10), + (0x7c9, 11), + (0x7d6, 11), + (0xfca, 12), + (0xfd0, 12), + (0xfe5, 12), + (0xfe6, 12), + (0x1feb, 13), + (0x1fef, 13), + (0x3ff3, 14), + (0x3ff4, 14), + (0x3ff5, 14), + (0xfe0, 12), + (0x7ce, 11), + (0x7d5, 11), + (0xfc6, 12), + (0xfd1, 12), + (0xfe1, 12), + (0x1fe0, 13), + (0x1fe8, 13), + (0x1ff0, 13), + (0x3ff1, 14), + (0x3ff8, 14), + (0x3ff6, 14), + (0x7ffc, 15), + (0xfe8, 12), + (0x7df, 11), + (0xfc9, 12), + (0xfd7, 12), + (0xfdc, 12), + (0x1fdc, 13), + (0x1fdf, 13), + (0x1fed, 13), + (0x1ff5, 13), + (0x3ff9, 14), + (0x3ffb, 14), + (0x7ffd, 15), + (0x7ffe, 15), + (0x1fe7, 13), + (0xfcc, 12), + (0xfd6, 12), + (0xfdf, 12), + (0x1fde, 13), + (0x1fda, 13), + (0x1fe5, 13), + (0x1ff2, 13), + (0x3ffa, 14), + (0x3ff7, 14), + (0x3ffc, 14), + (0x3ffd, 14), + (0x7fff, 15), +]; + +pub(crate) static SPECTRUM_10: [(u32, u8); 169] = [ + (0x22, 6), + (0x8, 5), + (0x1d, 6), + (0x26, 6), + (0x5f, 7), + (0xd3, 8), + (0x1cf, 9), + (0x3d0, 10), + (0x3d7, 10), + (0x3ed, 10), + (0x7f0, 11), + (0x7f6, 11), + (0xffd, 12), + (0x7, 5), + (0x0, 4), + (0x1, 4), + (0x9, 5), + (0x20, 6), + (0x54, 7), + (0x60, 7), + (0xd5, 8), + (0xdc, 8), + (0x1d4, 9), + (0x3cd, 10), + (0x3de, 10), + (0x7e7, 11), + (0x1c, 6), + (0x2, 4), + (0x6, 5), + (0xc, 5), + (0x1e, 6), + (0x28, 6), + (0x5b, 7), + (0xcd, 8), + (0xd9, 8), + (0x1ce, 9), + (0x1dc, 9), + (0x3d9, 10), + (0x3f1, 10), + (0x25, 6), + (0xb, 5), + (0xa, 5), + (0xd, 5), + (0x24, 6), + (0x57, 7), + (0x61, 7), + (0xcc, 8), + (0xdd, 8), + (0x1cc, 9), + (0x1de, 9), + (0x3d3, 10), + (0x3e7, 10), + (0x5d, 7), + (0x21, 6), + (0x1f, 6), + (0x23, 6), + (0x27, 6), + (0x59, 7), + (0x64, 7), + (0xd8, 8), + (0xdf, 8), + (0x1d2, 9), + (0x1e2, 9), + (0x3dd, 10), + (0x3ee, 10), + (0xd1, 8), + (0x55, 7), + (0x29, 6), + (0x56, 7), + (0x58, 7), + (0x62, 7), + (0xce, 8), + (0xe0, 8), + (0xe2, 8), + (0x1da, 9), + (0x3d4, 10), + (0x3e3, 10), + (0x7eb, 11), + (0x1c9, 9), + (0x5e, 7), + (0x5a, 7), + (0x5c, 7), + (0x63, 7), + (0xca, 8), + (0xda, 8), + (0x1c7, 9), + (0x1ca, 9), + (0x1e0, 9), + (0x3db, 10), + (0x3e8, 10), + (0x7ec, 11), + (0x1e3, 9), + (0xd2, 8), + (0xcb, 8), + (0xd0, 8), + (0xd7, 8), + (0xdb, 8), + (0x1c6, 9), + (0x1d5, 9), + (0x1d8, 9), + (0x3ca, 10), + (0x3da, 10), + (0x7ea, 11), + (0x7f1, 11), + (0x1e1, 9), + (0xd4, 8), + (0xcf, 8), + (0xd6, 8), + (0xde, 8), + (0xe1, 8), + (0x1d0, 9), + (0x1d6, 9), + (0x3d1, 10), + (0x3d5, 10), + (0x3f2, 10), + (0x7ee, 11), + (0x7fb, 11), + (0x3e9, 10), + (0x1cd, 9), + (0x1c8, 9), + (0x1cb, 9), + (0x1d1, 9), + (0x1d7, 9), + (0x1df, 9), + (0x3cf, 10), + (0x3e0, 10), + (0x3ef, 10), + (0x7e6, 11), + (0x7f8, 11), + (0xffa, 12), + (0x3eb, 10), + (0x1dd, 9), + (0x1d3, 9), + (0x1d9, 9), + (0x1db, 9), + (0x3d2, 10), + (0x3cc, 10), + (0x3dc, 10), + (0x3ea, 10), + (0x7ed, 11), + (0x7f3, 11), + (0x7f9, 11), + (0xff9, 12), + (0x7f2, 11), + (0x3ce, 10), + (0x1e4, 9), + (0x3cb, 10), + (0x3d8, 10), + (0x3d6, 10), + (0x3e2, 10), + (0x3e5, 10), + (0x7e8, 11), + (0x7f4, 11), + (0x7f5, 11), + (0x7f7, 11), + (0xffb, 12), + (0x7fa, 11), + (0x3ec, 10), + (0x3df, 10), + (0x3e1, 10), + (0x3e4, 10), + (0x3e6, 10), + (0x3f0, 10), + (0x7e9, 11), + (0x7ef, 11), + (0xff8, 12), + (0xffe, 12), + (0xffc, 12), + (0xfff, 12), +]; + +pub(crate) static SPECTRUM_11: [(u32, u8); 289] = [ + (0x0, 4), + (0x6, 5), + (0x19, 6), + (0x3d, 7), + (0x9c, 8), + (0xc6, 8), + (0x1a7, 9), + (0x390, 10), + (0x3c2, 10), + (0x3df, 10), + (0x7e6, 11), + (0x7f3, 11), + (0xffb, 12), + (0x7ec, 11), + (0xffa, 12), + (0xffe, 12), + (0x38e, 10), + (0x5, 5), + (0x1, 4), + (0x8, 5), + (0x14, 6), + (0x37, 7), + (0x42, 7), + (0x92, 8), + (0xaf, 8), + (0x191, 9), + (0x1a5, 9), + (0x1b5, 9), + (0x39e, 10), + (0x3c0, 10), + (0x3a2, 10), + (0x3cd, 10), + (0x7d6, 11), + (0xae, 8), + (0x17, 6), + (0x7, 5), + (0x9, 5), + (0x18, 6), + (0x39, 7), + (0x40, 7), + (0x8e, 8), + (0xa3, 8), + (0xb8, 8), + (0x199, 9), + (0x1ac, 9), + (0x1c1, 9), + (0x3b1, 10), + (0x396, 10), + (0x3be, 10), + (0x3ca, 10), + (0x9d, 8), + (0x3c, 7), + (0x15, 6), + (0x16, 6), + (0x1a, 6), + (0x3b, 7), + (0x44, 7), + (0x91, 8), + (0xa5, 8), + (0xbe, 8), + (0x196, 9), + (0x1ae, 9), + (0x1b9, 9), + (0x3a1, 10), + (0x391, 10), + (0x3a5, 10), + (0x3d5, 10), + (0x94, 8), + (0x9a, 8), + (0x36, 7), + (0x38, 7), + (0x3a, 7), + (0x41, 7), + (0x8c, 8), + (0x9b, 8), + (0xb0, 8), + (0xc3, 8), + (0x19e, 9), + (0x1ab, 9), + (0x1bc, 9), + (0x39f, 10), + (0x38f, 10), + (0x3a9, 10), + (0x3cf, 10), + (0x93, 8), + (0xbf, 8), + (0x3e, 7), + (0x3f, 7), + (0x43, 7), + (0x45, 7), + (0x9e, 8), + (0xa7, 8), + (0xb9, 8), + (0x194, 9), + (0x1a2, 9), + (0x1ba, 9), + (0x1c3, 9), + (0x3a6, 10), + (0x3a7, 10), + (0x3bb, 10), + (0x3d4, 10), + (0x9f, 8), + (0x1a0, 9), + (0x8f, 8), + (0x8d, 8), + (0x90, 8), + (0x98, 8), + (0xa6, 8), + (0xb6, 8), + (0xc4, 8), + (0x19f, 9), + (0x1af, 9), + (0x1bf, 9), + (0x399, 10), + (0x3bf, 10), + (0x3b4, 10), + (0x3c9, 10), + (0x3e7, 10), + (0xa8, 8), + (0x1b6, 9), + (0xab, 8), + (0xa4, 8), + (0xaa, 8), + (0xb2, 8), + (0xc2, 8), + (0xc5, 8), + (0x198, 9), + (0x1a4, 9), + (0x1b8, 9), + (0x38c, 10), + (0x3a4, 10), + (0x3c4, 10), + (0x3c6, 10), + (0x3dd, 10), + (0x3e8, 10), + (0xad, 8), + (0x3af, 10), + (0x192, 9), + (0xbd, 8), + (0xbc, 8), + (0x18e, 9), + (0x197, 9), + (0x19a, 9), + (0x1a3, 9), + (0x1b1, 9), + (0x38d, 10), + (0x398, 10), + (0x3b7, 10), + (0x3d3, 10), + (0x3d1, 10), + (0x3db, 10), + (0x7dd, 11), + (0xb4, 8), + (0x3de, 10), + (0x1a9, 9), + (0x19b, 9), + (0x19c, 9), + (0x1a1, 9), + (0x1aa, 9), + (0x1ad, 9), + (0x1b3, 9), + (0x38b, 10), + (0x3b2, 10), + (0x3b8, 10), + (0x3ce, 10), + (0x3e1, 10), + (0x3e0, 10), + (0x7d2, 11), + (0x7e5, 11), + (0xb7, 8), + (0x7e3, 11), + (0x1bb, 9), + (0x1a8, 9), + (0x1a6, 9), + (0x1b0, 9), + (0x1b2, 9), + (0x1b7, 9), + (0x39b, 10), + (0x39a, 10), + (0x3ba, 10), + (0x3b5, 10), + (0x3d6, 10), + (0x7d7, 11), + (0x3e4, 10), + (0x7d8, 11), + (0x7ea, 11), + (0xba, 8), + (0x7e8, 11), + (0x3a0, 10), + (0x1bd, 9), + (0x1b4, 9), + (0x38a, 10), + (0x1c4, 9), + (0x392, 10), + (0x3aa, 10), + (0x3b0, 10), + (0x3bc, 10), + (0x3d7, 10), + (0x7d4, 11), + (0x7dc, 11), + (0x7db, 11), + (0x7d5, 11), + (0x7f0, 11), + (0xc1, 8), + (0x7fb, 11), + (0x3c8, 10), + (0x3a3, 10), + (0x395, 10), + (0x39d, 10), + (0x3ac, 10), + (0x3ae, 10), + (0x3c5, 10), + (0x3d8, 10), + (0x3e2, 10), + (0x3e6, 10), + (0x7e4, 11), + (0x7e7, 11), + (0x7e0, 11), + (0x7e9, 11), + (0x7f7, 11), + (0x190, 9), + (0x7f2, 11), + (0x393, 10), + (0x1be, 9), + (0x1c0, 9), + (0x394, 10), + (0x397, 10), + (0x3ad, 10), + (0x3c3, 10), + (0x3c1, 10), + (0x3d2, 10), + (0x7da, 11), + (0x7d9, 11), + (0x7df, 11), + (0x7eb, 11), + (0x7f4, 11), + (0x7fa, 11), + (0x195, 9), + (0x7f8, 11), + (0x3bd, 10), + (0x39c, 10), + (0x3ab, 10), + (0x3a8, 10), + (0x3b3, 10), + (0x3b9, 10), + (0x3d0, 10), + (0x3e3, 10), + (0x3e5, 10), + (0x7e2, 11), + (0x7de, 11), + (0x7ed, 11), + (0x7f1, 11), + (0x7f9, 11), + (0x7fc, 11), + (0x193, 9), + (0xffd, 12), + (0x3dc, 10), + (0x3b6, 10), + (0x3c7, 10), + (0x3cc, 10), + (0x3cb, 10), + (0x3d9, 10), + (0x3da, 10), + (0x7d3, 11), + (0x7e1, 11), + (0x7ee, 11), + (0x7ef, 11), + (0x7f5, 11), + (0x7f6, 11), + (0xffc, 12), + (0xfff, 12), + (0x19d, 9), + (0x1c2, 9), + (0xb5, 8), + (0xa1, 8), + (0x96, 8), + (0x97, 8), + (0x95, 8), + (0x99, 8), + (0xa0, 8), + (0xa2, 8), + (0xac, 8), + (0xa9, 8), + (0xb1, 8), + (0xb3, 8), + (0xbb, 8), + (0xc0, 8), + (0x18f, 9), + (0x4, 5), +]; diff --git a/crates/aac/src/ics.rs b/crates/aac/src/ics.rs new file mode 100644 index 0000000000..a5d7c2a25e --- /dev/null +++ b/crates/aac/src/ics.rs @@ -0,0 +1,151 @@ +//! `ics_info` and helpers shared by the encoder and decoder (ISO/IEC 14496-3 §4.4.2.1, §4.5.2.3). + +use filmcraft_bitstream::{BitReader, BitWriter}; + +use crate::mdct::WindowShape; +use crate::tables::{swb_offsets_long, swb_offsets_short}; +use crate::{Error, Result}; + +/// `window_sequence`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum WindowSequence { + #[default] + OnlyLong = 0, + LongStart = 1, + EightShort = 2, + LongStop = 3, +} + +impl WindowSequence { + pub fn from_bits(v: u32) -> Self { + match v & 3 { + 0 => WindowSequence::OnlyLong, + 1 => WindowSequence::LongStart, + 2 => WindowSequence::EightShort, + _ => WindowSequence::LongStop, + } + } +} + +/// Decoded/encoded `ics_info`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct IcsInfo { + pub window_sequence: WindowSequence, + pub window_shape: WindowShape, + pub max_sfb: usize, + pub num_groups: usize, + /// Windows per group (only the first `num_groups` entries are used; long windows: `[1]`). + pub group_len: [u8; 8], +} + +impl Default for IcsInfo { + fn default() -> Self { + IcsInfo { + window_sequence: WindowSequence::OnlyLong, + window_shape: WindowShape::Sine, + max_sfb: 0, + num_groups: 1, + group_len: [1, 0, 0, 0, 0, 0, 0, 0], + } + } +} + +impl IcsInfo { + #[inline] + pub fn is_short(&self) -> bool { + self.window_sequence == WindowSequence::EightShort + } + #[inline] + pub fn num_windows(&self) -> usize { + if self.is_short() { 8 } else { 1 } + } + /// Band offsets of one window. + pub fn swb_offsets(&self, sf_index: u8) -> &'static [u16] { + if self.is_short() { swb_offsets_short(sf_index) } else { swb_offsets_long(sf_index) } + } + /// Number of bands for this window length (`num_swb`). + pub fn num_swb(&self, sf_index: u8) -> usize { + self.swb_offsets(sf_index).len() - 1 + } + /// First window of group `g`. + pub fn group_start(&self, g: usize) -> usize { + self.group_len[..g].iter().map(|&l| l as usize).sum() + } + + pub fn parse(br: &mut BitReader, sf_index: u8) -> Result { + if br.read_bits(1)? != 0 { + // ics_reserved_bit; tolerated by most decoders + } + let window_sequence = WindowSequence::from_bits(br.read_bits(2)?); + let window_shape = WindowShape::from_bit(br.read_bits(1)? == 1); + let mut info = IcsInfo { window_sequence, window_shape, ..Default::default() }; + if info.is_short() { + info.max_sfb = br.read_bits(4)? as usize; + let grouping = br.read_bits(7)?; + info.num_groups = 1; + info.group_len = [1, 0, 0, 0, 0, 0, 0, 0]; + for w in 1..8 { + if grouping & (1 << (6 - (w - 1))) != 0 { + info.group_len[info.num_groups - 1] += 1; + } else { + info.num_groups += 1; + info.group_len[info.num_groups - 1] = 1; + } + } + } else { + info.max_sfb = br.read_bits(6)? as usize; + if br.read_bits(1)? == 1 { + return Err(Error::Unsupported("predictor data (AAC Main / LTP)")); + } + } + if info.max_sfb > info.num_swb(sf_index) { + return Err(Error::Bitstream("max_sfb exceeds the number of scalefactor bands")); + } + Ok(info) + } + + pub fn write(&self, bw: &mut BitWriter) { + bw.write_bits(0, 1); + bw.write_bits(self.window_sequence as u32, 2); + bw.write_bits(self.window_shape.bit(), 1); + if self.is_short() { + bw.write_bits(self.max_sfb as u32, 4); + let mut grouping = 0u32; + let mut w = 0; + for g in 0..self.num_groups { + for i in 0..self.group_len[g] as usize { + if w > 0 && i > 0 { + grouping |= 1 << (6 - (w - 1)); + } + w += 1; + } + } + bw.write_bits(grouping, 7); + } else { + bw.write_bits(self.max_sfb as u32, 6); + bw.write_bits(0, 1); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn grouping_roundtrip() { + let info = IcsInfo { + window_sequence: WindowSequence::EightShort, + window_shape: WindowShape::Kbd, + max_sfb: 12, + num_groups: 3, + group_len: [3, 1, 4, 0, 0, 0, 0, 0], + }; + let mut bw = BitWriter::new(); + info.write(&mut bw); + let d = bw.finish(); + let p = IcsInfo::parse(&mut BitReader::new(&d), 4).unwrap(); + assert_eq!(p, info); + assert_eq!(p.group_start(2), 4); + } +} diff --git a/crates/aac/src/lib.rs b/crates/aac/src/lib.rs new file mode 100644 index 0000000000..c2c3c88783 --- /dev/null +++ b/crates/aac/src/lib.rs @@ -0,0 +1,74 @@ +//! Clean-room AAC-LC encoder and decoder (ISO/IEC 14496-3, MPEG-4 Audio, Low Complexity profile). +//! +//! - [`Encoder`]: planar f32 → raw access units (+ [`Encoder::audio_specific_config`] for MP4 `esds`, +//! optional ADTS framing). Psychoacoustic model, window switching, M/S, TNS, bit reservoir, CBR/VBR, +//! 1–8 channels. +//! - [`Decoder`]: `AudioSpecificConfig` + raw access units → planar f32. Supports M/S, intensity +//! stereo, PNS, TNS, pulse data, both window shapes and every window sequence. +//! +//! HE-AAC (SBR/PS), Main/SSR/LTP profiles, coupling channels and 960-sample frames are not supported; +//! for explicitly signalled HE-AAC only the AAC-LC core is decoded (at the core rate). +//! +//! Layer L0: depends only on `filmcraft-bitstream` (+ `thiserror`, optional `rayon`); no `unsafe`; +//! builds for `wasm32-unknown-unknown`. +//! +//! ``` +//! use craft_aac::{Decoder, Encoder, EncoderConfig}; +//! +//! let mut enc = Encoder::new(EncoderConfig::cbr(48_000, 2, 128_000))?; +//! let left: Vec = (0..4800).map(|i| (i as f32 * 0.05).sin() * 0.5).collect(); +//! let right = left.clone(); +//! let mut aus = enc.encode(&[&left, &right]); // 0..n access units +//! aus.extend(enc.flush()); +//! let asc = enc.audio_specific_config(); // for the MP4 `esds` +//! assert_eq!(enc.priming_samples(), 1024); // MP4 edit list media_time +//! +//! let mut dec = Decoder::new(&asc)?; +//! for au in &aus { +//! let pcm = dec.decode(au)?; // planar, 1024 samples per channel +//! assert_eq!(pcm.len(), 2); +//! } +//! # Ok::<(), craft_aac::Error>(()) +//! ``` + +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +mod config; +mod decoder; +mod encoder; +mod huffman; +mod huffman_tables; +mod ics; +mod mdct; +mod tables; +mod tns; + +pub use config::{AdtsHeader, AudioSpecificConfig, ElementType, ProgramConfig, config_layout, split_adts}; +pub use decoder::Decoder; +pub use encoder::{BitrateMode, Encoder, EncoderConfig, EncoderStats}; +pub use mdct::WindowShape; +pub use tables::{SAMPLE_RATES, sample_rate_index}; + +/// Samples per AAC-LC frame (per channel). +pub const FRAME_LEN: usize = 1024; + +/// Errors from configuration parsing, encoding and decoding. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("invalid configuration: {0}")] + InvalidConfig(&'static str), + #[error("unsupported feature: {0}")] + Unsupported(&'static str), + #[error("invalid bitstream: {0}")] + Bitstream(&'static str), + #[error("unexpected end of access unit")] + Eof, +} + +impl From for Error { + fn from(_: filmcraft_bitstream::BitError) -> Self { + Error::Eof + } +} + +pub type Result = std::result::Result; diff --git a/crates/aac/src/mdct.rs b/crates/aac/src/mdct.rs new file mode 100644 index 0000000000..a69d1a6d60 --- /dev/null +++ b/crates/aac/src/mdct.rs @@ -0,0 +1,361 @@ +//! MDCT / IMDCT (ISO/IEC 14496-3 §4.6.18) via a DCT-IV computed with an N/4-point complex FFT, +//! plus the sine and Kaiser–Bessel-derived windows. + +use std::f64::consts::PI; +use std::sync::OnceLock; + +/// Radix-2 complex FFT (forward, `e^{-2πi nk/P}`). +struct Fft { + n: usize, + /// Twiddle of the length-2 stage, `e^{-2πi 0/n}`. + w0: [f32; 2], + /// Twiddles of the stages of length 4, 8, ..., n, one contiguous run of `len / 2` per stage. + stage_re: Vec, + stage_im: Vec, + rev: Vec, +} + +impl Fft { + fn new(n: usize) -> Fft { + assert!(n.is_power_of_two()); + let bits = n.trailing_zeros(); + let rev = (0..n as u32).map(|i| if bits == 0 { 0 } else { i.reverse_bits() >> (32 - bits) }).collect(); + let twiddle: Vec<[f32; 2]> = (0..n / 2) + .map(|k| { + let a = -2.0 * PI * k as f64 / n as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + let (mut stage_re, mut stage_im) = (Vec::new(), Vec::new()); + let mut len = 4; + while len <= n { + let step = n / len; + for k in 0..len / 2 { + stage_re.push(twiddle[k * step][0]); + stage_im.push(twiddle[k * step][1]); + } + len <<= 1; + } + Fft { n, w0: twiddle.first().copied().unwrap_or([1.0, 0.0]), stage_re, stage_im, rev } + } + + fn run(&self, re: &mut [f32], im: &mut [f32]) { + let n = self.n; + for i in 0..n { + let j = self.rev[i] as usize; + if j > i { + re.swap(i, j); + im.swap(i, j); + } + } + if n >= 2 { + let [wr, wi] = self.w0; + let ((re2, _), (im2, _)) = (re.as_chunks_mut::<2>(), im.as_chunks_mut::<2>()); + for (r, i) in re2.iter_mut().zip(im2) { + let xr = r[1] * wr - i[1] * wi; + let xi = r[1] * wi + i[1] * wr; + r[1] = r[0] - xr; + i[1] = i[0] - xi; + r[0] += xr; + i[0] += xi; + } + } + let mut len = 4; + let mut off = 0; + // Two radix-2 stages (len, 2 * len) per pass over the data: the same butterflies in the same + // order per element, so the result is bit-identical to running the stages one by one. + while len * 2 <= n { + let h = len / 2; + let (t1r, t1i) = (&self.stage_re[off..off + h], &self.stage_im[off..off + h]); + let (t2r, t2i) = (&self.stage_re[off + h..off + 3 * h], &self.stage_im[off + h..off + 3 * h]); + off += 3 * h; + for (rc, ic) in re.chunks_exact_mut(4 * h).zip(im.chunks_exact_mut(4 * h)) { + let (ra, rest) = rc.split_at_mut(h); + let (rb, rest) = rest.split_at_mut(h); + let (rc2, rd) = rest.split_at_mut(h); + let (ia, rest) = ic.split_at_mut(h); + let (ib, rest) = rest.split_at_mut(h); + let (ic2, id) = rest.split_at_mut(h); + for k in 0..h { + let (w1r, w1i) = (t1r[k], t1i[k]); + // stage `len`: (a, b) and (c, d) + let xr = rb[k] * w1r - ib[k] * w1i; + let xi = rb[k] * w1i + ib[k] * w1r; + let (br, bi) = (ra[k] - xr, ia[k] - xi); + let (ar, ai) = (ra[k] + xr, ia[k] + xi); + let xr = rd[k] * w1r - id[k] * w1i; + let xi = rd[k] * w1i + id[k] * w1r; + let (dr, di) = (rc2[k] - xr, ic2[k] - xi); + let (cr, ci) = (rc2[k] + xr, ic2[k] + xi); + // stage `2 * len`: (a, c) with w2[k], (b, d) with w2[k + h] + let (w2r, w2i) = (t2r[k], t2i[k]); + let xr = cr * w2r - ci * w2i; + let xi = cr * w2i + ci * w2r; + rc2[k] = ar - xr; + ic2[k] = ai - xi; + ra[k] = ar + xr; + ia[k] = ai + xi; + let (w2r, w2i) = (t2r[k + h], t2i[k + h]); + let xr = dr * w2r - di * w2i; + let xi = dr * w2i + di * w2r; + rd[k] = br - xr; + id[k] = bi - xi; + rb[k] = br + xr; + ib[k] = bi + xi; + } + } + len <<= 2; + } + if len <= n { + let half = len / 2; + let (twr, twi) = (&self.stage_re[off..off + half], &self.stage_im[off..off + half]); + for (rc, ic) in re.chunks_exact_mut(len).zip(im.chunks_exact_mut(len)) { + let (ra, rb) = rc.split_at_mut(half); + let (ia, ib) = ic.split_at_mut(half); + for k in 0..half { + let (wr, wi) = (twr[k], twi[k]); + let xr = rb[k] * wr - ib[k] * wi; + let xi = rb[k] * wi + ib[k] * wr; + rb[k] = ra[k] - xr; + ib[k] = ia[k] - xi; + ra[k] += xr; + ia[k] += xi; + } + } + } + } +} + +/// MDCT of `2M` inputs to `M` outputs (unscaled sums): +/// `X[k] = Σ x[n] cos(π/M (n + ½ + M/2)(k + ½))`, and the matching unscaled inverse. +pub struct Mdct { + m: usize, + fft: Fft, + pre: Vec<[f32; 2]>, + post: Vec<[f32; 2]>, +} + +impl Mdct { + pub fn new(m: usize) -> Mdct { + let q = m / 2; + let pre = (0..q) + .map(|n| { + let a = -PI * n as f64 / m as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + let post = (0..q) + .map(|k| { + let a = -PI * (k as f64 + 0.25) / m as f64; + [a.cos() as f32, a.sin() as f32] + }) + .collect(); + Mdct { m, fft: Fft::new(q), pre, post } + } + + /// Working buffers for this transform size. + pub fn scratch(&self) -> MdctScratch { + MdctScratch { re: vec![0.0; self.m / 2], im: vec![0.0; self.m / 2], u: vec![0.0; self.m] } + } + + /// DCT-IV: `out[k] = Σ u[n] cos(π/M (n+½)(k+½))`; `re` and `im` (length M/2) are overwritten. + fn dct4(&self, u: &[f32], out: &mut [f32], re: &mut [f32], im: &mut [f32]) { + let m = self.m; + let q = m / 2; + for n in 0..q { + let (a, b) = (u[2 * n], u[m - 1 - 2 * n]); + let [c, s] = self.pre[n]; + re[n] = a * c - b * s; + im[n] = a * s + b * c; + } + self.fft.run(re, im); + for k in 0..q { + let [c, s] = self.post[k]; + let wr = re[k] * c - im[k] * s; + let wi = re[k] * s + im[k] * c; + out[2 * k] = wr; + out[m - 1 - 2 * k] = -wi; + } + } + + /// Forward MDCT of `x` (length 2M, already windowed) into `out` (length M). + pub fn forward(&self, x: &[f32], out: &mut [f32]) { + let m = self.m; + let h = m / 2; + let MdctScratch { re, im, mut u } = self.scratch(); + for n in 0..h { + // quarters a, b, c, d of length M/2 + u[n] = -x[m + h - 1 - n] - x[m + h + n]; + u[h + n] = x[n] - x[m - 1 - n]; + } + let (mut re, mut im) = (re, im); + self.dct4(&u, out, &mut re, &mut im); + } + + /// Inverse MDCT (unscaled): `y[n] = Σ X[k] cos(π/M (n + ½ + M/2)(k + ½))`, `y` has length 2M. + #[cfg(test)] + pub fn inverse(&self, spec: &[f32], y: &mut [f32]) { + self.inverse_with(spec, y, &mut self.scratch()); + } + + /// [`Self::inverse`] with caller-provided working buffers from [`Self::scratch`]. + pub fn inverse_with(&self, spec: &[f32], y: &mut [f32], sc: &mut MdctScratch) { + let m = self.m; + let h = m / 2; + let MdctScratch { re, im, u } = sc; + self.dct4(spec, u, re, im); + for n in 0..h { + y[n] = u[h + n]; + y[m - 1 - n] = -u[h + n]; + y[m + h - 1 - n] = -u[n]; + y[m + h + n] = -u[n]; + } + } +} + +/// Working buffers of one [`Mdct`] size, reusable across transforms. +pub struct MdctScratch { + re: Vec, + im: Vec, + u: Vec, +} + +/// Cached transforms for 1024 and 128 lines. +pub fn mdct_long() -> &'static Mdct { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| Mdct::new(1024)) +} +pub fn mdct_short() -> &'static Mdct { + static M: OnceLock = OnceLock::new(); + M.get_or_init(|| Mdct::new(128)) +} + +/// Window shape (`window_shape` bit): 0 = sine, 1 = Kaiser–Bessel derived. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum WindowShape { + #[default] + Sine, + Kbd, +} + +impl WindowShape { + pub fn from_bit(b: bool) -> Self { + if b { WindowShape::Kbd } else { WindowShape::Sine } + } + pub fn bit(self) -> u32 { + (self == WindowShape::Kbd) as u32 + } +} + +fn sine_half(m: usize) -> Vec { + let n = 2 * m; + (0..m).map(|i| (PI / n as f64 * (i as f64 + 0.5)).sin() as f32).collect() +} + +fn bessel_i0(x: f64) -> f64 { + let mut sum = 1.0; + let mut term = 1.0; + let y = x * x / 4.0; + for k in 1..60 { + term *= y / (k as f64 * k as f64); + sum += term; + if term < sum * 1e-17 { + break; + } + } + sum +} + +fn kbd_half(m: usize, alpha: f64) -> Vec { + // w[n] = sqrt(Σ_{j<=n} W(j) / Σ_{j<=M} W(j)), W the Kaiser kernel of length M+1. + let kernel: Vec = (0..=m) + .map(|j| { + let r = (j as f64 - m as f64 / 2.0) / (m as f64 / 2.0); + bessel_i0(PI * alpha * (1.0 - r * r).max(0.0).sqrt()) + }) + .collect(); + let total: f64 = kernel.iter().sum(); + let mut acc = 0.0; + (0..m) + .map(|n| { + acc += kernel[n]; + (acc / total).sqrt() as f32 + }) + .collect() +} + +/// Rising half (length N/2) of the window of the given shape; `long` selects N = 2048, else N = 256. +pub fn window(shape: WindowShape, long: bool) -> &'static [f32] { + static W: OnceLock<[Vec; 4]> = OnceLock::new(); + let w = W.get_or_init(|| [sine_half(1024), kbd_half(1024, 4.0), sine_half(128), kbd_half(128, 6.0)]); + let i = match (long, shape) { + (true, WindowShape::Sine) => 0, + (true, WindowShape::Kbd) => 1, + (false, WindowShape::Sine) => 2, + (false, WindowShape::Kbd) => 3, + }; + &w[i] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn direct(x: &[f32]) -> Vec { + let m = x.len() / 2; + (0..m).map(|k| (0..2 * m).map(|n| x[n] as f64 * (PI / m as f64 * (n as f64 + 0.5 + m as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum()).collect() + } + + #[test] + fn forward_matches_definition() { + for m in [128usize, 1024] { + let x: Vec = (0..2 * m).map(|i| ((i * 7919) % 101) as f32 / 50.0 - 1.0).collect(); + let mut out = vec![0f32; m]; + Mdct::new(m).forward(&x, &mut out); + let d = direct(&x); + for k in 0..m { + assert!((out[k] as f64 - d[k]).abs() < 1e-3 * (m as f64).sqrt(), "m={m} k={k} {} {}", out[k], d[k]); + } + } + } + + #[test] + fn inverse_matches_definition() { + let m = 128; + let spec: Vec = (0..m).map(|i| ((i * 31) % 17) as f32 - 8.0).collect(); + let mut y = vec![0f32; 2 * m]; + Mdct::new(m).inverse(&spec, &mut y); + for n in 0..2 * m { + let d: f64 = (0..m).map(|k| spec[k] as f64 * (PI / m as f64 * (n as f64 + 0.5 + m as f64 / 2.0) * (k as f64 + 0.5)).cos()).sum(); + assert!((y[n] as f64 - d).abs() < 1e-3, "n={n}"); + } + } + + #[test] + fn tdac_perfect_reconstruction() { + let m = 1024; + let t = Mdct::new(m); + for shape in [WindowShape::Sine, WindowShape::Kbd] { + let w = window(shape, true); + let win = |n: usize| if n < m { w[n] } else { w[2 * m - 1 - n] }; + let x: Vec = (0..4 * m).map(|i| ((i as f32) * 0.013).sin() + ((i * 13) % 7) as f32 * 0.05).collect(); + let mut out = vec![0f32; 4 * m]; + for f in 0..3 { + let seg: Vec = (0..2 * m).map(|n| x[f * m + n] * win(n)).collect(); + let mut spec = vec![0f32; m]; + t.forward(&seg, &mut spec); + for v in spec.iter_mut() { + *v *= 2.0; + } + let mut y = vec![0f32; 2 * m]; + t.inverse(&spec, &mut y); + for n in 0..2 * m { + out[f * m + n] += y[n] * win(n) / m as f32; + } + } + for n in m..3 * m { + assert!((out[n] - x[n]).abs() < 1e-4, "{shape:?} n={n} {} {}", out[n], x[n]); + } + } + } +} diff --git a/crates/aac/src/tables.rs b/crates/aac/src/tables.rs new file mode 100644 index 0000000000..ab3a152db3 --- /dev/null +++ b/crates/aac/src/tables.rs @@ -0,0 +1,163 @@ +//! Sampling-rate dependent tables from ISO/IEC 14496-3 §4.5.4 (scalefactor bands) and §4.6.9 (TNS). + +/// Sampling frequencies indexed by `samplingFrequencyIndex`. +pub const SAMPLE_RATES: [u32; 13] = [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350]; + +/// Map an arbitrary sample rate to its `samplingFrequencyIndex`, if it is one of the standard rates. +pub fn sample_rate_index(rate: u32) -> Option { + SAMPLE_RATES.iter().position(|&r| r == rate).map(|i| i as u8) +} + +/// The table index used for scalefactor-band tables when a non-standard rate is signalled explicitly +/// (§4.5.4: rates are mapped to the nearest table by frequency ranges). +pub fn table_index_for_rate(rate: u32) -> u8 { + match rate { + 92017.. => 0, + 75132.. => 1, + 55426.. => 2, + 46009.. => 3, + 37566.. => 4, + 27713.. => 5, + 23004.. => 6, + 18783.. => 7, + 13856.. => 8, + 11502.. => 9, + 9391.. => 10, + _ => 11, + } +} + +const SWB_LONG_96: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 156, 172, 188, 212, 240, 276, 320, 384, 448, + 512, 576, 640, 704, 768, 832, 896, 960, 1024, +]; +const SWB_LONG_64: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 64, 72, 80, 88, 100, 112, 124, 140, 156, 172, 192, 216, 240, 268, 304, 344, 384, 424, + 464, 504, 544, 584, 624, 664, 704, 744, 784, 824, 864, 904, 944, 984, 1024, +]; +const SWB_LONG_48: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 48, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 160, 176, 196, 216, 240, 264, 292, 320, 352, 384, 416, + 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832, 864, 896, 928, 1024, +]; +const SWB_LONG_32: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 48, 56, 64, 72, 80, 88, 96, 108, 120, 132, 144, 160, 176, 196, 216, 240, 264, 292, 320, 352, 384, 416, + 448, 480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832, 864, 896, 928, 960, 992, 1024, +]; +const SWB_LONG_24: &[u16] = &[ + 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 52, 60, 68, 76, 84, 92, 100, 108, 116, 124, 136, 148, 160, 172, 188, 204, 220, 240, 260, 284, 308, + 336, 364, 396, 432, 468, 508, 552, 600, 652, 704, 768, 832, 896, 960, 1024, +]; +const SWB_LONG_16: &[u16] = &[ + 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 100, 112, 124, 136, 148, 160, 172, 184, 196, 212, 228, 244, 260, 280, 300, 320, 344, 368, 396, 424, + 456, 492, 532, 572, 616, 664, 716, 772, 832, 896, 960, 1024, +]; +const SWB_LONG_8: &[u16] = &[ + 0, 12, 24, 36, 48, 60, 72, 84, 96, 108, 120, 132, 144, 156, 172, 188, 204, 220, 236, 252, 268, 288, 308, 328, 348, 372, 396, 420, 448, 476, 508, + 544, 580, 620, 664, 712, 764, 820, 880, 944, 1024, +]; + +const SWB_SHORT_96: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 32, 40, 48, 64, 92, 128]; +const SWB_SHORT_48: &[u16] = &[0, 4, 8, 12, 16, 20, 28, 36, 44, 56, 68, 80, 96, 112, 128]; +const SWB_SHORT_24: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 36, 44, 52, 64, 76, 92, 108, 128]; +const SWB_SHORT_16: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 60, 72, 88, 108, 128]; +const SWB_SHORT_8: &[u16] = &[0, 4, 8, 12, 16, 20, 24, 28, 36, 44, 52, 60, 72, 88, 108, 128]; + +/// Scalefactor band offsets (`swb_offset_long_window`, 1024-line frames); `len() == num_swb + 1`. +pub fn swb_offsets_long(sf_index: u8) -> &'static [u16] { + match sf_index { + 0 | 1 => SWB_LONG_96, + 2 => SWB_LONG_64, + 3 | 4 => SWB_LONG_48, + 5 => SWB_LONG_32, + 6 | 7 => SWB_LONG_24, + 8..=10 => SWB_LONG_16, + _ => SWB_LONG_8, + } +} + +/// Scalefactor band offsets for one short window (128 lines). +pub fn swb_offsets_short(sf_index: u8) -> &'static [u16] { + match sf_index { + 0..=2 => SWB_SHORT_96, + 3..=5 => SWB_SHORT_48, + 6 | 7 => SWB_SHORT_24, + 8..=10 => SWB_SHORT_16, + _ => SWB_SHORT_8, + } +} + +/// `TNS_MAX_BANDS` for AAC-LC (long, short). +pub fn tns_max_bands(sf_index: u8, short: bool) -> usize { + const LONG: [u8; 13] = [31, 31, 34, 40, 42, 51, 46, 46, 42, 42, 42, 39, 39]; + const SHORT: [u8; 13] = [9, 9, 10, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14]; + let i = (sf_index as usize).min(12); + if short { SHORT[i] as usize } else { LONG[i] as usize } +} + +/// `TNS_MAX_ORDER` for AAC-LC. +pub fn tns_max_order(short: bool) -> usize { + if short { 7 } else { 12 } +} + +/// The `x^(4/3)` table for the integer quantised magnitudes 0..=8191. +pub fn pow43_table() -> &'static [f32; 8192] { + use std::sync::OnceLock; + static TABLE: OnceLock> = OnceLock::new(); + TABLE.get_or_init(|| { + let mut t = Box::new([0f32; 8192]); + for (i, v) in t.iter_mut().enumerate() { + *v = (i as f64).powf(4.0 / 3.0) as f32; + } + t + }) +} + +/// `x^(4/3)` for an integer quantised magnitude (table lookup up to 8191). +pub fn pow43(q: u32) -> f32 { + match pow43_table().get(q as usize) { + Some(v) => *v, + None => (q as f64).powf(4.0 / 3.0) as f32, + } +} + +/// `2^(0.25 * (sf - 100))`, the dequantisation gain of scalefactor `sf`. +#[inline] +pub fn sf_gain(sf: i32) -> f32 { + use std::sync::OnceLock; + static TABLE: OnceLock<[f32; 256]> = OnceLock::new(); + let t = TABLE.get_or_init(|| std::array::from_fn(|i| sf_gain_exact(i as i32))); + match usize::try_from(sf).ok().and_then(|i| t.get(i)) { + Some(g) => *g, + None => sf_gain_exact(sf), + } +} + +fn sf_gain_exact(sf: i32) -> f32 { + (0.25 * (sf as f64 - 100.0)).exp2() as f32 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn band_tables_are_monotonic_and_complete() { + for i in 0..13u8 { + let l = swb_offsets_long(i); + assert_eq!(*l.last().unwrap(), 1024); + assert!(l.windows(2).all(|w| w[0] < w[1] && (w[1] - w[0]) % 4 == 0)); + let s = swb_offsets_short(i); + assert_eq!(*s.last().unwrap(), 128); + assert!(s.windows(2).all(|w| w[0] < w[1] && (w[1] - w[0]) % 4 == 0)); + assert!(tns_max_bands(i, false) < l.len()); + assert!(tns_max_bands(i, true) < s.len()); + } + assert_eq!(swb_offsets_long(3).len() - 1, 49); + assert_eq!(swb_offsets_long(5).len() - 1, 51); + assert_eq!(swb_offsets_long(0).len() - 1, 41); + assert_eq!(swb_offsets_long(2).len() - 1, 47); + assert_eq!(swb_offsets_long(6).len() - 1, 47); + assert_eq!(swb_offsets_long(8).len() - 1, 43); + assert_eq!(swb_offsets_long(11).len() - 1, 40); + } +} diff --git a/crates/aac/src/tns.rs b/crates/aac/src/tns.rs new file mode 100644 index 0000000000..c22d52413e --- /dev/null +++ b/crates/aac/src/tns.rs @@ -0,0 +1,313 @@ +//! Temporal noise shaping (ISO/IEC 14496-3 §4.6.9): syntax, coefficient conversion, and the +//! decoder (all-pole) and encoder (all-zero) filters. + +use std::f64::consts::FRAC_PI_2; + +use filmcraft_bitstream::{BitReader, BitWriter}; + +use crate::ics::IcsInfo; +use crate::tables::{tns_max_bands, tns_max_order}; +use crate::{Error, Result}; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsFilter { + /// Length in scalefactor bands. + pub length: u8, + pub order: u8, + /// false = upward, true = downward. + pub direction: bool, + pub coef_compress: bool, + /// Quantised reflection coefficients (signed, `coef_res + 3 - coef_compress` bits each). + pub coef: [i8; 12], +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsWindow { + pub n_filt: u8, + /// false = 3-bit, true = 4-bit coefficient resolution. + pub coef_res: bool, + pub filt: [TnsFilter; 4], +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TnsData { + pub windows: [TnsWindow; 8], +} + +impl TnsData { + pub fn parse(br: &mut BitReader, info: &IcsInfo) -> Result { + let short = info.is_short(); + let mut t = TnsData::default(); + for w in 0..info.num_windows() { + let win = &mut t.windows[w]; + win.n_filt = br.read_bits(if short { 1 } else { 2 })? as u8; + if win.n_filt == 0 { + continue; + } + win.coef_res = br.read_bits(1)? == 1; + for f in 0..win.n_filt as usize { + let filt = &mut win.filt[f]; + filt.length = br.read_bits(if short { 4 } else { 6 })? as u8; + filt.order = br.read_bits(if short { 3 } else { 5 })? as u8; + if filt.order as usize > tns_max_order(short) { + return Err(Error::Bitstream("TNS order too large")); + } + if filt.order > 0 { + filt.direction = br.read_bits(1)? == 1; + filt.coef_compress = br.read_bits(1)? == 1; + let bits = 3 + win.coef_res as u32 - filt.coef_compress as u32; + for i in 0..filt.order as usize { + let v = br.read_bits(bits)? as i32; + let v = if v & (1 << (bits - 1)) != 0 { v - (1 << bits) } else { v }; + filt.coef[i] = v as i8; + } + } + } + } + Ok(t) + } + + pub fn write(&self, bw: &mut BitWriter, info: &IcsInfo) { + let short = info.is_short(); + for w in 0..info.num_windows() { + let win = &self.windows[w]; + bw.write_bits(win.n_filt as u32, if short { 1 } else { 2 }); + if win.n_filt == 0 { + continue; + } + bw.write_bits(win.coef_res as u32, 1); + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + bw.write_bits(filt.length as u32, if short { 4 } else { 6 }); + bw.write_bits(filt.order as u32, if short { 3 } else { 5 }); + if filt.order > 0 { + bw.write_bits(filt.direction as u32, 1); + bw.write_bits(filt.coef_compress as u32, 1); + let bits = 3 + win.coef_res as u32 - filt.coef_compress as u32; + for i in 0..filt.order as usize { + bw.write_bits(filt.coef[i] as i32 as u32 & ((1 << bits) - 1), bits); + } + } + } + } + } + + /// Bits `write` would produce. + pub fn bit_count(&self, info: &IcsInfo) -> u32 { + let short = info.is_short(); + let mut n = 0; + for w in 0..info.num_windows() { + let win = &self.windows[w]; + n += if short { 1 } else { 2 }; + if win.n_filt == 0 { + continue; + } + n += 1; + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + n += if short { 7 } else { 11 }; + if filt.order > 0 { + n += 2 + filt.order as u32 * (3 + win.coef_res as u32 - filt.coef_compress as u32); + } + } + } + n + } +} + +/// Dequantise one reflection coefficient. +pub fn dequant_coef(c: i8, coef_res: bool) -> f64 { + let res = 3 + coef_res as i32; + let iqfac = ((1 << (res - 1)) as f64 - 0.5) / FRAC_PI_2; + let iqfac_m = ((1 << (res - 1)) as f64 + 0.5) / FRAC_PI_2; + let c = c as f64; + (c / if c >= 0.0 { iqfac } else { iqfac_m }).sin() +} + +/// Quantise a reflection coefficient (inverse of [`dequant_coef`], nearest index). +pub fn quant_coef(k: f64, coef_res: bool) -> i8 { + let res = 3 + coef_res as i32; + let iqfac = ((1 << (res - 1)) as f64 - 0.5) / FRAC_PI_2; + let iqfac_m = ((1 << (res - 1)) as f64 + 0.5) / FRAC_PI_2; + let a = k.clamp(-1.0, 1.0).asin(); + let lo = -(1 << (res - 1)); + let hi = (1 << (res - 1)) - 1; + let q = if a >= 0.0 { (a * iqfac).round() } else { (a * iqfac_m).round() }; + (q as i32).clamp(lo, hi) as i8 +} + +/// Reflection coefficients → direct-form LPC `a[0..=order]` (`a[0] = 1`). +pub fn parcor_to_lpc(k: &[f64]) -> [f64; 13] { + let mut a = [0f64; 13]; + a[0] = 1.0; + let mut b = [0f64; 13]; + for m in 1..=k.len() { + for i in 1..m { + b[i] = a[i] + k[m - 1] * a[m - i]; + } + a[1..m].copy_from_slice(&b[1..m]); + a[m] = k[m - 1]; + } + a +} + +/// Filter region `[start, end)` of every TNS filter of window `w`, in spectrum lines of that window. +fn regions(info: &IcsInfo, sf_index: u8, win: &TnsWindow) -> Vec<(usize, usize, usize)> { + let short = info.is_short(); + let swb = info.swb_offsets(sf_index); + let num_swb = swb.len() - 1; + let lim = tns_max_bands(sf_index, short).min(info.max_sfb); + let mut out = Vec::new(); + let mut bottom = num_swb; + for f in 0..win.n_filt as usize { + let filt = &win.filt[f]; + let top = bottom; + bottom = top.saturating_sub(filt.length as usize); + if filt.order == 0 { + continue; + } + let start = swb[bottom.min(lim)] as usize; + let end = swb[top.min(lim)] as usize; + if end > start { + out.push((f, start, end)); + } + } + out +} + +fn lpc_of(filt: &TnsFilter, coef_res: bool) -> [f64; 13] { + let k: Vec = filt.coef[..filt.order as usize].iter().map(|&c| dequant_coef(c, coef_res)).collect(); + parcor_to_lpc(&k) +} + +/// Decoder: apply the all-pole TNS synthesis filters to `spec` (1024 lines; short windows at 128 stride). +pub fn apply_decoder(spec: &mut [f32], info: &IcsInfo, sf_index: u8, tns: &TnsData) { + let wlen = if info.is_short() { 128 } else { 1024 }; + for w in 0..info.num_windows() { + let win = &tns.windows[w]; + for (f, start, end) in regions(info, sf_index, win) { + let filt = &win.filt[f]; + let a = lpc_of(filt, win.coef_res); + let order = filt.order as usize; + let s = &mut spec[w * wlen..(w + 1) * wlen]; + let size = end - start; + let mut state = [0f64; 12]; + for m in 0..size { + let idx = if filt.direction { end - 1 - m } else { start + m }; + let mut y = s[idx] as f64; + for i in 0..order { + y -= a[i + 1] * state[i]; + } + for i in (1..order).rev() { + state[i] = state[i - 1]; + } + if order > 0 { + state[0] = y; + } + s[idx] = y as f32; + } + } + } +} + +/// Encoder: apply the all-zero TNS analysis filters (exact inverse of [`apply_decoder`]). +pub fn apply_encoder(spec: &mut [f32], info: &IcsInfo, sf_index: u8, tns: &TnsData) { + let wlen = if info.is_short() { 128 } else { 1024 }; + for w in 0..info.num_windows() { + let win = &tns.windows[w]; + for (f, start, end) in regions(info, sf_index, win) { + let filt = &win.filt[f]; + let a = lpc_of(filt, win.coef_res); + let order = filt.order as usize; + let s = &mut spec[w * wlen..(w + 1) * wlen]; + let size = end - start; + let mut state = [0f64; 12]; + for m in 0..size { + let idx = if filt.direction { end - 1 - m } else { start + m }; + let x = s[idx] as f64; + let mut y = x; + for i in 0..order { + y += a[i + 1] * state[i]; + } + for i in (1..order).rev() { + state[i] = state[i - 1]; + } + if order > 0 { + state[0] = x; + } + s[idx] = y as f32; + } + } + } +} + +/// Levinson–Durbin on autocorrelation `r[0..=order]`: returns reflection coefficients and the +/// prediction gain `r[0] / error`. +pub fn levinson(r: &[f64], order: usize) -> (Vec, f64) { + let mut a = vec![0f64; order + 1]; + a[0] = 1.0; + let mut err = r[0]; + let mut ks = Vec::with_capacity(order); + if err <= 0.0 { + return (vec![0.0; order], 1.0); + } + for m in 1..=order { + let mut acc = r[m]; + for i in 1..m { + acc += a[i] * r[m - i]; + } + let k = -acc / err; + let prev = a.clone(); + for i in 1..m { + a[i] = prev[i] + k * prev[m - i]; + } + a[m] = k; + err *= 1.0 - k * k; + ks.push(k); + if err <= r[0] * 1e-9 { + ks.resize(order, 0.0); + break; + } + } + (ks, r[0] / err.max(r[0] * 1e-9)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ics::WindowSequence; + use crate::mdct::WindowShape; + + #[test] + fn coef_quant_roundtrip() { + for res in [false, true] { + let n = if res { 8 } else { 4 }; + for c in -n..n { + let k = dequant_coef(c as i8, res); + assert_eq!(quant_coef(k, res), c as i8); + } + } + } + + #[test] + fn encoder_decoder_filters_are_inverse() { + let info = IcsInfo { window_sequence: WindowSequence::OnlyLong, window_shape: WindowShape::Sine, max_sfb: 49, ..Default::default() }; + let mut t = TnsData::default(); + t.windows[0].n_filt = 1; + t.windows[0].coef_res = true; + t.windows[0].filt[0] = TnsFilter { length: 40, order: 4, direction: true, coef_compress: false, coef: [3, -2, 1, 5, 0, 0, 0, 0, 0, 0, 0, 0] }; + let orig: Vec = (0..1024).map(|i| ((i * 37 % 101) as f32 - 50.0) * 0.1).collect(); + let mut s = orig.clone(); + apply_encoder(&mut s, &info, 4, &t); + assert_ne!(s, orig); + apply_decoder(&mut s, &info, 4, &t); + for i in 0..1024 { + assert!((s[i] - orig[i]).abs() < 1e-3); + } + let mut bw = BitWriter::new(); + t.write(&mut bw, &info); + assert_eq!(bw.bit_len() as u32, t.bit_count(&info)); + let d = bw.finish(); + assert_eq!(TnsData::parse(&mut BitReader::new(&d), &info).unwrap(), t); + } +} diff --git a/crates/aac/tools/derive.py b/crates/aac/tools/derive.py new file mode 100644 index 0000000000..656c470291 --- /dev/null +++ b/crates/aac/tools/derive.py @@ -0,0 +1,128 @@ +"""Black-box derivation of the AAC Huffman codebooks by probing the ffmpeg decoder (oracle only).""" +import sys, json, math +from concurrent.futures import ThreadPoolExecutor +from probe import frame, silent, run, fit + +def unit(sf): + return 2.0 ** (0.25 * (sf - 100)) / (32768 * 1024) + +PARAMS = { # cb: (dim, signed, lav) + 1: (4, True, 1), 2: (4, True, 1), 3: (4, False, 2), 4: (4, False, 2), + 5: (2, True, 4), 6: (2, True, 4), 7: (2, False, 7), 8: (2, False, 7), + 9: (2, False, 12), 10: (2, False, 12), 11: (2, False, 16), +} + +def analyse(y, err, sfs): + if err or len(y) < 2048: + return None + x, res, en = fit(y[:2048]) + if en == 0: + return [0] * 12 if res == 0 else None + if res > 1e-6 * en: + return None + out = [] + for i, v in enumerate(x): + u = unit(sfs[i // 4]) + m = abs(v) / u + q = m ** 0.75 + r = round(q) + if abs(q - r) > 0.02 * max(1, r): + return None + out.append(int(r) * (1 if v >= 0 else -1)) + return out + +G = 140 + +def query_spec(cb, S, k, e): + dim, signed, lav = PARAMS[cb] + ncopies = 12 // dim + bits = '' + for c in range(ncopies): + bits += S + ('0' if c % 2 == 0 else '1') * k + '00000' * e + y, err = run([frame(cb, '000', bits, G=G), silent(), silent()]) + vals = analyse(y, err, [G, G, G]) + if vals is None: + return None + vecs = [vals[c * dim:(c + 1) * dim] for c in range(ncopies)] + v = vecs[0] + for c in range(ncopies): + exp = v if (signed or c % 2 == 0) else [-a for a in v] + if vecs[c] != exp: + return None + if signed: + if any(abs(a) > lav for a in v): + return None + mod = 2 * lav + 1 + idx = 0 + for a in v: + idx = idx * mod + (a + lav) + return idx + if any(a < 0 for a in v): + return None + if sum(1 for a in v if a != 0) != k: + return None + if cb == 11: + if sum(1 for a in v if a == 16) != e: + return None + elif e: + return None + if any(a > lav for a in v): + return None + mod = lav + 1 + idx = 0 + for a in v: + idx = idx * mod + a + return idx + +def query_sf(S): + Gs = 128 + # spectral: cb1 (-1,0,0,0) = '10001' in each band + y, err = run([frame(1, S + '0' + S, '10001' * 3, G=Gs), silent(), silent()]) + for d in range(-60, 61): + sfs = [Gs + d, Gs + d, Gs + 2 * d] + if not all(0 <= s <= 255 for s in sfs): + continue + vals = analyse(y, err, sfs) + if vals == [-1, 0, 0, 0] * 3: + return d + 60 + return None + +def hyps(cb): + dim, signed, lav = PARAMS[cb] + if signed: + return [(0, 0)] + hs = [(k, 0) for k in range(dim + 1)] + if cb == 11: + hs += [(1, 1), (2, 1), (2, 2)] + return hs + +def derive(cb, maxlen=20): + leaves = {} + frontier = ['0', '1'] + with ThreadPoolExecutor(14) as ex: + while frontier: + def q(S): + if cb == 0: + return S, query_sf(S) + for (k, e) in hyps(cb): + r = query_spec(cb, S, k, e) + if r is not None: + return S, r + return S, None + nxt = [] + for S, r in ex.map(q, frontier): + if r is not None: + leaves[S] = r + elif len(S) < maxlen: + nxt += [S + '0', S + '1'] + print(f'cb{cb}: depth {len(frontier[0])} frontier {len(frontier)} leaves {len(leaves)}', file=sys.stderr) + frontier = nxt + return leaves + +if __name__ == '__main__': + cb = int(sys.argv[1]) + leaves = derive(cb) + kraft = sum(2.0 ** -len(s) for s in leaves) + idxs = sorted(leaves.values()) + print(f'cb{cb}: {len(leaves)} leaves, kraft {kraft}, unique idx {len(set(idxs))}, range {idxs[0]}..{idxs[-1]}', file=sys.stderr) + json.dump({s: i for s, i in leaves.items()}, open(f'cb{cb}.json', 'w')) diff --git a/crates/aac/tools/gen.py b/crates/aac/tools/gen.py new file mode 100644 index 0000000000..c7aba8df28 --- /dev/null +++ b/crates/aac/tools/gen.py @@ -0,0 +1,29 @@ +import json +out = ['//! AAC Huffman codebooks (ISO/IEC 14496-3 §4.A.1, spectrum codebooks 1–11 and the scalefactor codebook).', + '//!', + '//! Each table is indexed by the codebook index defined in the standard and stores `(codeword, length)`.', + '//! Generated data: derived by black-box probing of an external decoder used purely as a test oracle', + '//! (every table is a complete prefix code — Kraft sum exactly 1 — covering every index once; see', + '//! `tests::codebooks_are_complete_prefix_codes`).', + '', + '#![allow(clippy::unreadable_literal)]', + ''] +for cb in [0] + list(range(1, 12)): + d = json.load(open(f'cb{cb}.json')) + n = len(d) + arr = [None] * n + for s, i in d.items(): + arr[i] = (int(s, 2), len(s)) + name = 'SCALEFACTOR' if cb == 0 else f'SPECTRUM_{cb}' + out.append(f'pub(crate) static {name}: [(u32, u8); {n}] = [') + line = ' ' + for (c, l) in arr: + item = f' (0x{c:x}, {l}),' + if len(line) + len(item) > 100: + out.append(line) + line = ' ' + line += item + out.append(line) + out.append('];') + out.append('') +open('huffman_tables.rs', 'w').write('\n'.join(out)) diff --git a/crates/aac/tools/probe.py b/crates/aac/tools/probe.py new file mode 100644 index 0000000000..3e4c142e19 --- /dev/null +++ b/crates/aac/tools/probe.py @@ -0,0 +1,78 @@ +"""Codebook recovery by black-box probing of an external AAC decoder (ffmpeg binary, oracle only). + +Builds single-frame ADTS streams whose band data repeats a candidate bit string, decodes them with +`ffmpeg -f f32le`, least-squares fits the output to the IMDCT basis functions of the probed lines and +accepts the candidate as a codeword when the fit is exact, the copies agree and the implied sign/escape +bits are consistent. A breadth-first walk over bit strings then recovers each complete prefix code. + +Usage: python3 derive.py <1..11 | 0 for scalefactors> (writes cbN.json) + python3 gen.py (writes huffman_tables.rs) +Needs only the Python standard library and ffmpeg at /opt/homebrew/bin. +""" +import subprocess, struct, math, sys, os, tempfile + +class BW: + def __init__(self): self.bits=[] + def w(self, v, n): + for i in range(n-1,-1,-1): self.bits.append((v>>i)&1) + def s(self, st): + for c in st: self.bits.append(int(c)) + def bytes(self): + b=self.bits+[0]*((-len(self.bits))%8) + return bytes(int(''.join(map(str,b[i:i+8])),2) for i in range(0,len(b),8)) + +def adts(payload, sfi=3, ch=1): + L=len(payload)+7 + w=BW(); w.w(0xFFF,12); w.w(0,1); w.w(0,2); w.w(1,1); w.w(1,2); w.w(sfi,4); w.w(0,1); w.w(ch,3) + w.w(0,4); w.w(L,13); w.w(0x7FF,11); w.w(0,2) + return w.bytes()+payload + +def frame(cb, sfbits, specbits, nbands=3, G=100): + w=BW() + w.w(0,3); w.w(0,4); w.w(G,8) + w.w(0,1); w.w(0,2); w.w(0,1); w.w(nbands,6); w.w(0,1) + if nbands: + w.w(cb,4); w.w(nbands,5) + w.s(sfbits) + w.w(0,1); w.w(0,1); w.w(0,1) + w.s(specbits) + w.w(7,3) + return adts(w.bytes()) + +def silent(): + return frame(0,'','',nbands=0) + +def run(frames): + fd,path=tempfile.mkstemp(suffix='.aac'); os.write(fd,b''.join(frames)); os.close(fd) + p=subprocess.run(['/opt/homebrew/bin/ffmpeg','-v','error','-f','aac','-i',path,'-f','f32le','-'],capture_output=True) + os.unlink(path) + n=len(p.stdout)//4 + return list(struct.unpack('<%df'%n,p.stdout)), p.stderr.decode() + +N=2048 +n0=(N/2+1)/2 +def basis(k): + return [math.sin(math.pi*(n+0.5)/N)*math.cos(2*math.pi/N*(n+n0)*(k+0.5)) for n in range(N)] +B=[basis(k) for k in range(12)] +def solve(A,b): + n=len(A); M=[row[:]+[b[i]] for i,row in enumerate(A)] + for c in range(n): + p=max(range(c,n),key=lambda r:abs(M[r][c])); M[c],M[p]=M[p],M[c] + for r in range(n): + if r!=c: + f=M[r][c]/M[c][c] + for j in range(c,n+1): M[r][j]-=f*M[c][j] + return [M[i][n]/M[i][i] for i in range(n)] +G_=[[sum(B[i][n]*B[j][n] for n in range(N)) for j in range(12)] for i in range(12)] +def fit(y): + rhs=[sum(B[i][n]*y[n] for n in range(N)) for i in range(12)] + x=solve(G_,rhs) + res=sum((y[n]-sum(x[k]*B[k][n] for k in range(12)))**2 for n in range(N)) + en=sum(v*v for v in y) + return x,res,en +if __name__=='__main__': + spec=sys.argv[2]; cb=int(sys.argv[1]) + y,err=run([frame(cb,'000',spec),silent(),silent()]) + print(len(y),repr(err)) + x,res,en=fit(y[:2048]) + print(["%.4g"%v for v in x],res,en) diff --git a/crates/audio-io/Cargo.toml b/crates/audio-io/Cargo.toml new file mode 100644 index 0000000000..018ae2dddc --- /dev/null +++ b/crates/audio-io/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-audio-io" +description = "Universal audio file I/O: WAV/BWF/RF64, AIFF, FLAC encoding, and symphonia-backed decoding for the Craft ecosystem" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[dependencies] +serde = { workspace = true } +thiserror = { workspace = true } +log = { workspace = true } +symphonia = { workspace = true } + +[lints] +workspace = true diff --git a/crates/audio-io/src/aiff.rs b/crates/audio-io/src/aiff.rs new file mode 100644 index 0000000000..77ed6f80f3 --- /dev/null +++ b/crates/audio-io/src/aiff.rs @@ -0,0 +1,283 @@ +//! Native AIFF / AIFF-C reader and writer (Apple "Audio Interchange File Format" 1.3 and the +//! AIFF-C draft: big-endian IFF chunks, 80-bit IEEE extended sample rate). + +use crate::pcm::{PcmKind, Quantizer, be_u16, be_u32, deinterleave, sample_format, tag, validate_buffer, validate_layout, write_interleaved}; +use crate::{AudioBuffer, AudioError, AudioInfo, BitDepth, FileFormat, Result, SampleFormat}; +use std::io::{Cursor, Seek, SeekFrom, Write}; + +/// AIFF-C version 1 timestamp for the FVER chunk. +const AIFC_VERSION_1: u32 = 0xA280_5140; +const MAX_SAMPLE_RATE: f64 = 10_000_000.0; + +pub(crate) struct AiffLayout { + pub info: AudioInfo, + pub kind: PcmKind, + pub data_start: usize, + pub data_len: usize, +} + +pub(crate) enum AiffParse { + Native(AiffLayout), + /// Compressed AIFF-C (ulaw, alaw, ima4, ...): handed to symphonia. + Foreign, +} + +/// True when the bytes start like an AIFF or AIFF-C file. +pub(crate) fn is_aiff(b: &[u8]) -> bool { + tag(b, 0).is_some_and(|t| &t == b"FORM") && tag(b, 8).is_some_and(|t| &t == b"AIFF" || &t == b"AIFC") +} + +/// Decode an 80-bit IEEE 754 extended-precision float. +pub(crate) fn extended_to_f64(b: &[u8]) -> Option { + let se = be_u16(b, 0)?; + let hi = u64::from(be_u32(b, 2)?); + let lo = u64::from(be_u32(b, 6)?); + let mant = (hi << 32) | lo; + if mant == 0 { + return Some(0.0); + } + let exp = i32::from(se & 0x7FFF); + if exp == 0x7FFF { + return None; // inf / NaN + } + let v = mant as f64 * 2f64.powi(exp - 16383 - 63); + Some(if se & 0x8000 != 0 { -v } else { v }) +} + +/// Encode a positive integer as an 80-bit IEEE 754 extended float (exact). +pub(crate) fn u32_to_extended(r: u32) -> [u8; 10] { + let mut out = [0u8; 10]; + if r == 0 { + return out; + } + let e = 31 - r.leading_zeros(); + let exp = (16383 + e) as u16; + let mant = u64::from(r) << (63 - e); + out[..2].copy_from_slice(&exp.to_be_bytes()); + out[2..].copy_from_slice(&mant.to_be_bytes()); + out +} + +pub(crate) fn parse(b: &[u8]) -> Result { + if !is_aiff(b) { + return Err(AudioError::Malformed("not an AIFF/AIFF-C file".into())); + } + let is_aifc = tag(b, 8).is_some_and(|t| &t == b"AIFC"); + let mut comm: Option<(i16, u32, i16, f64, [u8; 4])> = None; + let mut ssnd: Option<(usize, usize)> = None; + + let mut pos: usize = 12; + let mut chunks = 0usize; + while let (Some(id), Some(size32)) = (tag(b, pos), be_u32(b, pos + 4)) { + chunks += 1; + if chunks > 100_000 { + break; + } + let body = pos.checked_add(8).ok_or_else(|| AudioError::Malformed("chunk offset overflow".into()))?; + let remaining = b.len().saturating_sub(body); + let declared = usize::try_from(size32).unwrap_or(usize::MAX); + let len = declared.min(remaining); + let c = b.get(body..body + len).unwrap_or_default(); + match &id { + b"COMM" => { + let short = || AudioError::Malformed("COMM chunk too short".into()); + let channels = be_u16(c, 0).ok_or_else(short)? as i16; + let frames = be_u32(c, 2).ok_or_else(short)?; + let bits = be_u16(c, 6).ok_or_else(short)? as i16; + let rate = extended_to_f64(c.get(8..18).ok_or_else(short)?).ok_or_else(|| AudioError::Malformed("invalid sample rate".into()))?; + let compression = if is_aifc { tag(c, 18).ok_or_else(short)? } else { *b"NONE" }; + comm = Some((channels, frames, bits, rate, compression)); + } + b"SSND" if ssnd.is_none() => { + let offset = usize::try_from(be_u32(c, 0).unwrap_or(0)).unwrap_or(usize::MAX); + let start = body.saturating_add(8).saturating_add(offset).min(b.len()); + let data_len = len.saturating_sub(8).saturating_sub(offset); + ssnd = Some((start, data_len)); + } + _ => {} + } + if declared > remaining { + break; + } + pos = match body.checked_add(declared).and_then(|p| p.checked_add(declared & 1)) { + Some(p) => p, + None => break, + }; + } + + let (channels, frames, bits, rate, compression) = comm.ok_or_else(|| AudioError::Malformed("AIFF has no COMM chunk".into()))?; + if channels <= 0 { + return Err(AudioError::Malformed("AIFF declares zero or negative channels".into())); + } + if !(1.0..=MAX_SAMPLE_RATE).contains(&rate) { + return Err(AudioError::Malformed(format!("AIFF sample rate {rate} out of range"))); + } + let sample_rate = rate.round() as u32; + let int_kind = |big_endian: bool| -> Result<(PcmKind, SampleFormat)> { + if !(1..=32).contains(&bits) { + return Err(AudioError::Unsupported(format!("AIFF sample size {bits}"))); + } + let bytes = (bits as usize).div_ceil(8); + let sf = match bits { + 1..=8 => SampleFormat::Int8, + 9..=16 => SampleFormat::Int16, + 17..=24 => SampleFormat::Int24, + _ => SampleFormat::Int32, + }; + Ok((PcmKind::Int { bytes, big_endian, unsigned: false }, sf)) + }; + let (kind, sample_format) = match &compression { + b"NONE" | b"twos" => int_kind(true)?, + b"sowt" => int_kind(false)?, + b"fl32" | b"FL32" => (PcmKind::Float { bytes: 4, big_endian: true }, SampleFormat::Float32), + b"fl64" | b"FL64" => (PcmKind::Float { bytes: 8, big_endian: true }, SampleFormat::Float64), + _ => return Ok(AiffParse::Foreign), + }; + let (data_start, data_len) = ssnd.unwrap_or((b.len(), 0)); + let block = kind.bytes().saturating_mul(channels as usize).max(1); + let frames = u64::from(frames).min((data_len / block) as u64); + Ok(AiffParse::Native(AiffLayout { + info: AudioInfo { format: FileFormat::Aiff, sample_format, sample_rate, channels: channels as u16, frames, bwf: None }, + kind, + data_start, + data_len, + })) +} + +pub(crate) fn decode(b: &[u8], layout: &AiffLayout) -> Result { + let end = layout.data_start.saturating_add(layout.data_len).min(b.len()); + let data = b.get(layout.data_start..end).unwrap_or_default(); + let channels = deinterleave(data, layout.kind, usize::from(layout.info.channels), layout.info.frames)?; + Ok(AudioBuffer { sample_rate: layout.info.sample_rate, channels }) +} + +fn push_chunk(out: &mut Vec, id: &[u8; 4], body: &[u8]) { + out.extend_from_slice(id); + out.extend_from_slice(&u32::try_from(body.len()).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(body); + if body.len() % 2 == 1 { + out.push(0); + } +} + +/// Encode AIFF (integer depths) or AIFF-C `fl32` (Float32). +pub(crate) fn encode(buf: &AudioBuffer, depth: BitDepth, dither: bool) -> Result> { + validate_buffer(buf)?; + let frames = buf.frames(); + let head_len = header(buf.channels.len(), buf.sample_rate, depth, frames as u64)?.len(); + let data_len = usize::from(sample_format(depth).0 / 8).saturating_mul(buf.channels.len()).saturating_mul(frames); + let total = head_len.saturating_add(data_len).saturating_add(1); + let mut out = Vec::new(); + out.try_reserve_exact(total).map_err(|_| AudioError::TooLarge(format!("cannot allocate {total} bytes for AIFF output")))?; + let mut w = Writer::new(Cursor::new(out), buf.channels.len(), buf.sample_rate, depth, dither)?; + w.write(&buf.channels, frames)?; + Ok(w.finish()?.into_inner()) +} + +/// An AIFF written block by block: a placeholder header first, then the real one (the same +/// length) written over it by [`Writer::finish`]. +pub(crate) struct Writer { + out: W, + start: u64, + channels: usize, + sample_rate: u32, + depth: BitDepth, + quantizers: Vec, + frames: u64, + bytes: Vec, + /// Set when a block write has failed: the sink may hold a prefix of that block while + /// `frames` does not count it, so the writer refuses every later call. + failed: bool, +} + +impl Writer { + pub(crate) fn new(mut out: W, channels: usize, sample_rate: u32, depth: BitDepth, dither: bool) -> Result { + validate_layout(channels, sample_rate)?; + let start = out.stream_position()?; + out.write_all(&header(channels, sample_rate, depth, 0)?)?; + let quantizers = Quantizer::per_channel(u32::from(sample_format(depth).0), dither, channels); + Ok(Writer { out, start, channels, sample_rate, depth, quantizers, frames: 0, bytes: Vec::new(), failed: false }) + } + + /// The frame count after `frames` more, or an error when AIFF can't hold that many (4 GiB). + pub(crate) fn check_room(&self, frames: u64) -> Result { + let total = self.frames.saturating_add(frames); + header(self.channels, self.sample_rate, self.depth, total)?; + Ok(total) + } + + /// Appends `frames` frames of planar audio (missing samples are silence), unless they don't + /// fit (see [`Writer::check_room`]). + pub(crate) fn write(&mut self, block: &[Vec], frames: usize) -> Result<()> { + if self.failed { + return Err(crate::failed_write_error()); + } + let total = self.check_room(frames as u64)?; + if let Err(e) = write_interleaved(&mut self.out, &mut self.bytes, block, frames, &mut self.quantizers, sample_format(self.depth), true) { + self.failed = true; + return Err(AudioError::from(e)); + } + self.frames = total; + Ok(()) + } + + pub(crate) fn finish(mut self) -> Result { + if self.failed { + return Err(crate::failed_write_error()); + } + let data_len = u64::from(sample_format(self.depth).0 / 8).saturating_mul(self.channels as u64).saturating_mul(self.frames); + if data_len % 2 == 1 { + self.out.write_all(&[0])?; + } + let head = header(self.channels, self.sample_rate, self.depth, self.frames)?; + self.out.seek(SeekFrom::Start(self.start))?; + self.out.write_all(&head)?; + self.out.seek(SeekFrom::End(0))?; + self.out.flush()?; + Ok(self.out) + } +} + +/// Everything before the sample data of an AIFF holding `frames` frames; its length doesn't +/// depend on `frames`. +fn header(channels: usize, sample_rate: u32, depth: BitDepth, frames: u64) -> Result> { + let channels_i16 = i16::try_from(channels).map_err(|_| AudioError::Encode("too many channels for AIFF".into()))?; + let frames_u32 = u32::try_from(frames).map_err(|_| AudioError::TooLarge("too many frames for AIFF".into()))?; + let (bits, is_float) = sample_format(depth); + let data_len = u64::from(bits / 8).saturating_mul(channels as u64).saturating_mul(frames); + + let mut comm = Vec::with_capacity(40); + comm.extend_from_slice(&channels_i16.to_be_bytes()); + comm.extend_from_slice(&frames_u32.to_be_bytes()); + comm.extend_from_slice(&bits.to_be_bytes()); + comm.extend_from_slice(&u32_to_extended(sample_rate)); + if is_float { + comm.extend_from_slice(b"fl32"); + let name = b"32-bit float"; + comm.push(name.len() as u8); + comm.extend_from_slice(name); + if (name.len() + 1) % 2 == 1 { + comm.push(0); + } + } + + let mut head = Vec::with_capacity(128); + head.extend_from_slice(b"FORM"); + head.extend_from_slice(&[0, 0, 0, 0]); // patched below + head.extend_from_slice(if is_float { b"AIFC" } else { b"AIFF" }); + if is_float { + push_chunk(&mut head, b"FVER", &AIFC_VERSION_1.to_be_bytes()); + } + push_chunk(&mut head, b"COMM", &comm); + head.extend_from_slice(b"SSND"); + let ssnd_len = u32::try_from(data_len.saturating_add(8)).map_err(|_| AudioError::TooLarge("AIFF data exceeds 4 GiB".into()))?; + head.extend_from_slice(&ssnd_len.to_be_bytes()); + head.extend_from_slice(&[0u8; 8]); // offset, blockSize + + let total = (head.len() as u64).saturating_add(data_len).saturating_add(data_len & 1); + let form_len = u32::try_from(total - 8).map_err(|_| AudioError::TooLarge("AIFF file exceeds 4 GiB".into()))?; + if let Some(slot) = head.get_mut(4..8) { + slot.copy_from_slice(&form_len.to_be_bytes()); + } + Ok(head) +} diff --git a/crates/audio-io/src/buffer.rs b/crates/audio-io/src/buffer.rs new file mode 100644 index 0000000000..78f25af6f3 --- /dev/null +++ b/crates/audio-io/src/buffer.rs @@ -0,0 +1,71 @@ +//! Planar `f32` audio buffer. + +/// Planar f32 audio. +/// +/// Every channel is expected to hold the same number of frames; [`AudioBuffer::frames`] reports the +/// shortest channel so a ragged buffer never causes out-of-bounds access. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct AudioBuffer { + /// Sample rate in Hz. + pub sample_rate: u32, + /// One `Vec` of samples per channel. + pub channels: Vec>, +} + +impl AudioBuffer { + /// A silent buffer with `channels` channels of `frames` frames each. + pub fn new(sample_rate: u32, channels: usize, frames: usize) -> Self { + Self { sample_rate, channels: vec![vec![0.0; frames]; channels] } + } + + /// Number of frames (the length of the shortest channel; 0 when there are no channels). + pub fn frames(&self) -> usize { + self.channels.iter().map(Vec::len).min().unwrap_or(0) + } + + /// Number of channels. + pub fn num_channels(&self) -> usize { + self.channels.len() + } + + /// Duration in seconds (0 when the sample rate is 0). + pub fn duration_secs(&self) -> f64 { + if self.sample_rate == 0 { + return 0.0; + } + self.frames() as f64 / f64::from(self.sample_rate) + } + + /// Interleaved copy of the samples (frame-major: `L0 R0 L1 R1 ...`). + pub fn interleaved(&self) -> Vec { + let frames = self.frames(); + let mut out = Vec::with_capacity(frames.saturating_mul(self.channels.len())); + for f in 0..frames { + for ch in &self.channels { + out.push(ch.get(f).copied().unwrap_or(0.0)); + } + } + out + } + + /// Build a planar buffer from interleaved samples. A trailing partial frame is dropped; + /// `channels == 0` yields an empty buffer. + pub fn from_interleaved(sr: u32, channels: usize, data: &[f32]) -> Self { + if channels == 0 { + return Self { sample_rate: sr, channels: Vec::new() }; + } + let frames = data.len() / channels; + let mut out: Vec> = (0..channels).map(|_| Vec::with_capacity(frames)).collect(); + for frame in data.chunks_exact(channels) { + for (ch, &s) in out.iter_mut().zip(frame) { + ch.push(s); + } + } + Self { sample_rate: sr, channels: out } + } + + /// Absolute peak over all channels (NaN samples are ignored). + pub fn peak(&self) -> f32 { + self.channels.iter().flat_map(|c| c.iter()).fold(0.0f32, |acc, &s| acc.max(s.abs())) + } +} diff --git a/crates/audio-io/src/caf.rs b/crates/audio-io/src/caf.rs new file mode 100644 index 0000000000..9d3d9ccb6f --- /dev/null +++ b/crates/audio-io/src/caf.rs @@ -0,0 +1,98 @@ +//! Size checks on CAF chunks before symphonia reads them: it allocates whatever a magic cookie, +//! a packet table or a packet claims, and a failed allocation aborts the process. + +use crate::pcm::{be_u16, be_u32, be_u64}; +use crate::{AudioError, Result}; + +/// Refuse the CAF stream at `at` if its magic cookie, packet table or packets claim more bytes +/// than the file holds. +pub(crate) fn check(b: &[u8], at: usize) -> Result<()> { + if be_u16(b, at + 4) == Some(1) { chunks(b, at + 8) } else { Ok(()) } +} + +fn malformed(what: &str) -> AudioError { + AudioError::Malformed(format!("CAF {what} is larger than the file")) +} + +/// Walk the chunks the way the CAF reader reads them: it advances by the bytes it parses, not by +/// the declared size, and after a `data` chunk of unknown size it reads on into the audio. +fn chunks(b: &[u8], mut pos: usize) -> Result<()> { + // Whether the packet table lists each packet's size and frame count (variable per packet). + let (mut lists_sizes, mut lists_frames, mut described) = (false, false, false); + while let (Some(kind), Some(size)) = (b.get(pos..pos + 4), be_u64(b, pos + 4)) { + let body = pos + 12; + let used = match kind { + b"desc" if size == 32 => { + lists_sizes = be_u32(b, body + 16) == Some(0); + lists_frames = be_u32(b, body + 20) == Some(0); + described = true; + Some(32) + } + b"data" if size == u64::MAX => Some(4), + b"chan" if size >= 12 => be_u32(b, body + 8).map(|n| 12 + u64::from(n) * 20), + b"pakt" if size >= 24 && described => packet_table(b, body, lists_sizes, lists_frames)?, + b"kuki" if size > b.len().saturating_sub(body) as u64 => return Err(malformed("magic cookie")), + b"desc" | b"pakt" | b"chan" => None, + _ => Some(size), + }; + match used.filter(|_| described).and_then(|n| usize::try_from(n).ok()).and_then(|n| body.checked_add(n)) { + Some(next) if next < b.len() => pos = next, + _ => return Ok(()), + } + } + Ok(()) +} + +/// Check a packet table's count and packet sizes; returns the bytes symphonia reads for it, or +/// None where symphonia stops. It reserves room for every packet before reading any, so a table +/// it would refuse after that, or that runs past the file, is refused first. +fn packet_table(b: &[u8], body: usize, lists_sizes: bool, lists_frames: bool) -> Result> { + let valid = |at: usize| be_u64(b, at).filter(|&n| n as i64 >= 0); + let (Some(packets), Some(_)) = (valid(body), valid(body + 8)) else { return Ok(None) }; + let mut at = body + 24; + if packets > b.len().saturating_sub(at) as u64 { + return Err(malformed("packet table")); + } + if !lists_sizes && !lists_frames { + return if packets == 0 { Ok(Some(24)) } else { Err(AudioError::Malformed("a constant bit rate CAF lists packets".into())) }; + } + for _ in 0..packets { + if lists_sizes { + let Some((size, len)) = varint(b, at) else { return Err(malformed("packet table")) }; + if size > b.len() as u64 { + return Err(malformed("packet")); + } + at += len; + } + if lists_frames { + let Some((_, len)) = varint(b, at) else { return Err(malformed("packet table")) }; + at += len; + } + } + Ok(Some((at - body) as u64)) +} + +/// A CAF variable-length integer at `at` and its length in bytes. +fn varint(b: &[u8], at: usize) -> Option<(u64, usize)> { + let mut v = 0u64; + for i in 0..9 { + let x = *b.get(at + i)?; + v |= u64::from(x & 0x7f); + if x & 0x80 == 0 { + return Some((v, i + 1)); + } + v <<= 7; + } + None +} + +#[cfg(test)] +mod tests { + #[test] + fn rejects_cookie_size_above_signed_range() { + let mut bytes = b"caff\0\x01\0\0kuki".to_vec(); + bytes.extend_from_slice(&(1u64 << 63).to_be_bytes()); + + assert!(super::check(&bytes, 0).is_err()); + } +} diff --git a/crates/audio-io/src/flac.rs b/crates/audio-io/src/flac.rs new file mode 100644 index 0000000000..8f48904764 --- /dev/null +++ b/crates/audio-io/src/flac.rs @@ -0,0 +1,563 @@ +//! A small pure-Rust FLAC encoder (RFC 9639): fixed blocksize, independent channels, and per +//! subframe the cheapest of CONSTANT, VERBATIM or FIXED (orders 0–4) prediction with +//! partitioned Rice-coded residuals. Decoding is symphonia's, after [`check_metadata`]. + +use crate::pcm::{Quantizer, be_u32, le_u32, validate_buffer}; +use crate::{AudioBuffer, AudioError, AudioInfo, BitDepth, FileFormat, Result, SampleFormat}; +use std::io::{Cursor, Seek, SeekFrom, Write}; + +const BLOCK_SIZE: usize = 4096; +const MAX_PARTITION_ORDER: u32 = 6; + +struct BitWriter { + out: Vec, + acc: u64, + nbits: u32, +} + +impl BitWriter { + fn new() -> Self { + Self { out: Vec::new(), acc: 0, nbits: 0 } + } + + /// Write the low `n` (≤ 32) bits of `v`, MSB first. + fn write(&mut self, v: u64, n: u32) { + if n == 0 { + return; + } + let n = n.min(32); + let mask = (1u64 << n) - 1; + self.acc = (self.acc << n) | (v & mask); + self.nbits += n; + while self.nbits >= 8 { + self.nbits -= 8; + self.out.push((self.acc >> self.nbits) as u8); + } + self.acc &= (1u64 << self.nbits) - 1; + } + + fn write_signed(&mut self, v: i64, n: u32) { + self.write(v as u64, n); + } + + /// `q` zero bits followed by a one bit. + fn write_unary(&mut self, mut q: u64) { + while q >= 32 { + self.write(0, 32); + q -= 32; + } + self.write(1, (q as u32) + 1); + } + + fn align(&mut self) { + if self.nbits > 0 { + let pad = 8 - self.nbits; + self.write(0, pad); + } + } + + fn into_bytes(mut self) -> Vec { + self.align(); + self.out + } +} + +fn crc8(data: &[u8]) -> u8 { + let mut crc: u8 = 0; + for &b in data { + crc ^= b; + for _ in 0..8 { + crc = if crc & 0x80 != 0 { (crc << 1) ^ 0x07 } else { crc << 1 }; + } + } + crc +} + +fn crc16(data: &[u8]) -> u16 { + let mut crc: u16 = 0; + for &b in data { + crc ^= u16::from(b) << 8; + for _ in 0..8 { + crc = if crc & 0x8000 != 0 { (crc << 1) ^ 0x8005 } else { crc << 1 }; + } + } + crc +} + +/// FLAC's UTF-8-like variable-length integer coding (up to 36 bits). +fn write_utf8_number(w: &mut BitWriter, v: u64) { + if v < 0x80 { + w.write(v, 8); + return; + } + // Payload bits available for an n-byte coding: 5n + 1 (n = 2..=6), 36 for n = 7. + let n: u32 = (2..=7).find(|&n| if n == 7 { true } else { v < (1u64 << (5 * n + 1)) }).unwrap_or(7); + let cont_bits = 6 * (n - 1); + let lead_payload = if n == 7 { 0 } else { 7 - n }; + let prefix: u64 = (0xFFu64 << (8 - n)) & 0xFF; + let lead = prefix | ((v >> cont_bits) & ((1u64 << lead_payload) - 1)); + w.write(lead, 8); + for i in (0..n - 1).rev() { + w.write(0x80 | ((v >> (6 * i)) & 0x3F), 8); + } +} + +fn sample_rate_code(r: u32) -> (u64, Option<(u64, u32)>) { + match r { + 88_200 => (1, None), + 176_400 => (2, None), + 192_000 => (3, None), + 8_000 => (4, None), + 16_000 => (5, None), + 22_050 => (6, None), + 24_000 => (7, None), + 32_000 => (8, None), + 44_100 => (9, None), + 48_000 => (10, None), + 96_000 => (11, None), + r if r % 1000 == 0 && r / 1000 <= 255 => (12, Some((u64::from(r / 1000), 8))), + r if r <= 65_535 => (13, Some((u64::from(r), 16))), + r if r % 10 == 0 && r / 10 <= 65_535 => (14, Some((u64::from(r / 10), 16))), + _ => (0, None), + } +} + +fn bps_code(bps: u32) -> u64 { + match bps { + 8 => 1, + 12 => 2, + 16 => 4, + 20 => 5, + 24 => 6, + _ => 0, // from STREAMINFO + } +} + +fn zigzag(r: i64) -> u64 { + ((r << 1) ^ (r >> 63)) as u64 +} + +/// Fixed-predictor residuals for `order`, or `None` when any residual leaves the i32 range. +fn fixed_residual(x: &[i64], order: usize) -> Option> { + let mut res = Vec::with_capacity(x.len().saturating_sub(order)); + for n in order..x.len() { + let g = |k: usize| x.get(n - k).copied().unwrap_or(0); + let r = match order { + 0 => g(0), + 1 => g(0) - g(1), + 2 => g(0) - 2 * g(1) + g(2), + 3 => g(0) - 3 * g(1) + 3 * g(2) - g(3), + _ => g(0) - 4 * g(1) + 6 * g(2) - 4 * g(3) + g(4), + }; + if r < i64::from(i32::MIN) || r > i64::from(i32::MAX) { + return None; + } + res.push(r); + } + Some(res) +} + +/// Best Rice parameter and its cost in bits for one partition. +fn best_rice(part: &[u64]) -> (u32, u64) { + if part.is_empty() { + return (0, 0); + } + let sum: u64 = part.iter().sum(); + let mean = sum / part.len() as u64; + let guess = if mean == 0 { 0 } else { 63 - mean.leading_zeros() }; + let mut best = (0u32, u64::MAX); + for k in guess.saturating_sub(1)..=(guess + 1).min(30) { + let cost = part.len() as u64 * u64::from(k + 1) + part.iter().map(|&u| u >> k).sum::(); + if cost < best.1 { + best = (k, cost); + } + } + best +} + +/// Residual coding plan: partition order, per-partition parameters, total bits. +struct RicePlan { + order: u32, + params: Vec, + bits: u64, + wide: bool, +} + +fn plan_residual(u: &[u64], block_len: usize, pred_order: usize) -> RicePlan { + let mut best: Option = None; + for p in 0..=MAX_PARTITION_ORDER { + let parts = 1usize << p; + if !block_len.is_multiple_of(parts) || block_len / parts <= pred_order { + break; + } + let per = block_len / parts; + let mut params = Vec::with_capacity(parts); + let mut bits = 0u64; + let mut start = 0usize; + for i in 0..parts { + let len = if i == 0 { per - pred_order } else { per }; + let part = u.get(start..start + len).unwrap_or_default(); + start += len; + let (k, cost) = best_rice(part); + params.push(k); + bits += cost; + } + let wide = params.iter().any(|&k| k > 14); + bits += 6 + parts as u64 * if wide { 5 } else { 4 }; + if best.as_ref().is_none_or(|b| bits < b.bits) { + best = Some(RicePlan { order: p, params, bits, wide }); + } + } + best.unwrap_or(RicePlan { order: 0, params: vec![0], bits: u64::MAX, wide: false }) +} + +fn write_residual(w: &mut BitWriter, u: &[u64], block_len: usize, pred_order: usize, plan: &RicePlan) { + w.write(u64::from(plan.wide), 2); + w.write(u64::from(plan.order), 4); + let parts = 1usize << plan.order; + let per = block_len / parts; + let mut start = 0usize; + for (i, &k) in plan.params.iter().enumerate() { + let len = if i == 0 { per - pred_order } else { per }; + w.write(u64::from(k), if plan.wide { 5 } else { 4 }); + for &v in u.get(start..start + len).unwrap_or_default() { + w.write_unary(v >> k); + w.write(v, k); + } + start += len; + } +} + +fn write_subframe(w: &mut BitWriter, x: &[i64], bps: u32) { + let n = x.len(); + let first = x.first().copied().unwrap_or(0); + if x.iter().all(|&s| s == first) { + w.write(0, 1); + w.write(0, 6); + w.write(0, 1); + w.write_signed(first, bps); + return; + } + let verbatim_bits = n as u64 * u64::from(bps); + let mut best: Option<(usize, Vec, RicePlan)> = None; + for order in 0..=4usize.min(n.saturating_sub(1)) { + let Some(res) = fixed_residual(x, order) else { continue }; + let u: Vec = res.into_iter().map(zigzag).collect(); + let plan = plan_residual(&u, n, order); + let bits = plan.bits.saturating_add(order as u64 * u64::from(bps)); + if bits < verbatim_bits && best.as_ref().is_none_or(|b| bits < b.2.bits.saturating_add(b.0 as u64 * u64::from(bps))) { + best = Some((order, u, plan)); + } + } + match best { + Some((order, u, plan)) => { + w.write(0, 1); + w.write(8 + order as u64, 6); + w.write(0, 1); + for &s in x.iter().take(order) { + w.write_signed(s, bps); + } + write_residual(w, &u, n, order, &plan); + } + None => { + w.write(0, 1); + w.write(1, 6); + w.write(0, 1); + for &s in x { + w.write_signed(s, bps); + } + } + } +} + +/// Checks that FLAC can hold the layout; returns the bits per sample. +fn check(channels: usize, sample_rate: u32, depth: BitDepth) -> Result { + crate::pcm::validate_layout(channels, sample_rate)?; + if channels > 8 { + return Err(AudioError::Unsupported(format!("FLAC supports at most 8 channels, got {channels}"))); + } + if sample_rate > 655_350 { + return Err(AudioError::Unsupported(format!("FLAC sample rate {sample_rate} exceeds 655350 Hz"))); + } + match depth { + BitDepth::Int16 => Ok(16), + BitDepth::Int24 => Ok(24), + BitDepth::Int32 => Ok(32), + BitDepth::Float32 => Err(AudioError::Unsupported("FLAC cannot store 32-bit float samples".into())), + } +} + +fn stream_info(channels: usize, sample_rate: u32, bps: u32, total: u64) -> Vec { + let mut si = BitWriter::new(); + si.write(1, 1); // last metadata block + si.write(0, 7); // STREAMINFO + si.write(34, 24); + si.write(BLOCK_SIZE as u64, 16); + si.write(BLOCK_SIZE as u64, 16); + si.write(0, 24); + si.write(0, 24); + si.write(u64::from(sample_rate), 20); + si.write(channels as u64 - 1, 3); + si.write(u64::from(bps - 1), 5); + si.write(total >> 32, 4); + si.write(total & 0xFFFF_FFFF, 32); + for _ in 0..4 { + si.write(0, 32); // MD5 unknown + } + si.into_bytes() +} + +/// A FLAC stream written block by block; [`Writer::finish`] fills in the total sample count. +pub(crate) struct Writer { + out: W, + start: u64, + channels: usize, + sample_rate: u32, + bps: u32, + quantizers: Vec, + pending: Vec>, + frames: u64, + frame_no: u64, + /// Set when a frame write has failed: `write_frame` clears the pending samples before + /// writing, so a failed frame is lost while `frames` still counts it; the writer refuses + /// every later call. + failed: bool, +} + +impl Writer { + pub(crate) fn new(mut out: W, channels: usize, sample_rate: u32, depth: BitDepth, dither: bool) -> Result { + let bps = check(channels, sample_rate, depth)?; + let start = out.stream_position()?; + out.write_all(b"fLaC")?; + out.write_all(&stream_info(channels, sample_rate, bps, 0))?; + Ok(Writer { + out, + start, + channels, + sample_rate, + bps, + quantizers: Quantizer::per_channel(bps, dither, channels), + pending: (0..channels).map(|_| Vec::with_capacity(BLOCK_SIZE)).collect(), + frames: 0, + frame_no: 0, + failed: false, + }) + } + + /// The frame count after `frames` more, or an error past what FLAC can count (2^36 frames). + pub(crate) fn check_room(&self, frames: u64) -> Result { + let total = self.frames.saturating_add(frames); + if total >= 1u64 << 36 { + return Err(AudioError::TooLarge("too many frames for FLAC".into())); + } + Ok(total) + } + + /// Appends `frames` frames of planar audio (missing samples are silence), unless they don't + /// fit (see [`Writer::check_room`]). + pub(crate) fn write(&mut self, block: &[Vec], frames: usize) -> Result<()> { + if self.failed { + return Err(crate::failed_write_error()); + } + let total = self.check_room(frames as u64)?; + let mut done = 0; + while done < frames { + let held = self.pending.first().map_or(0, Vec::len); + let n = (BLOCK_SIZE - held).min(frames - done); + for (c, (p, q)) in self.pending.iter_mut().zip(self.quantizers.iter_mut()).enumerate() { + let src = block.get(c).and_then(|ch| ch.get(done..)).unwrap_or_default(); + p.extend((0..n).map(|i| i64::from(q.quantize(src.get(i).copied().unwrap_or(0.0))))); + } + done += n; + if held + n == BLOCK_SIZE + && let Err(e) = self.write_frame() + { + self.failed = true; + return Err(e); + } + } + self.frames = total; + Ok(()) + } + + fn write_frame(&mut self) -> Result<()> { + let len = self.pending.first().map_or(0, Vec::len); + if len == 0 { + return Ok(()); + } + let (sr_code, sr_extra) = sample_rate_code(self.sample_rate); + let mut w = BitWriter::new(); + w.write(0x3FFE, 14); + w.write(0, 1); + w.write(0, 1); // fixed blocksize stream + w.write(7, 4); // 16-bit (blocksize-1) follows + w.write(sr_code, 4); + w.write(self.channels as u64 - 1, 4); + w.write(bps_code(self.bps), 3); + w.write(0, 1); + write_utf8_number(&mut w, self.frame_no); + w.write(len as u64 - 1, 16); + if let Some((v, n)) = sr_extra { + w.write(v, n); + } + let header = w.into_bytes(); + let mut w = BitWriter::new(); + for &b in &header { + w.write(u64::from(b), 8); + } + w.write(u64::from(crc8(&header)), 8); + for p in &mut self.pending { + write_subframe(&mut w, p, self.bps); + p.clear(); + } + let mut frame = w.into_bytes(); + let crc = crc16(&frame); + frame.extend_from_slice(&crc.to_be_bytes()); + self.out.write_all(&frame)?; + self.frame_no += 1; + Ok(()) + } + + pub(crate) fn finish(mut self) -> Result { + if self.failed { + return Err(crate::failed_write_error()); + } + self.write_frame()?; + let info = stream_info(self.channels, self.sample_rate, self.bps, self.frames); + self.out.seek(SeekFrom::Start(self.start.saturating_add(4)))?; + self.out.write_all(&info)?; + self.out.seek(SeekFrom::End(0))?; + self.out.flush()?; + Ok(self.out) + } +} + +pub(crate) fn encode(buf: &AudioBuffer, depth: BitDepth, dither: bool) -> Result> { + validate_buffer(buf)?; + let bps = check(buf.channels.len(), buf.sample_rate, depth)?; + let frames = buf.frames(); + let mut out = Vec::new(); + out.try_reserve(frames.saturating_mul(buf.channels.len()).saturating_mul(bps as usize / 8) / 2) + .map_err(|_| AudioError::TooLarge("cannot allocate FLAC output".into()))?; + let mut w = Writer::new(Cursor::new(out), buf.channels.len(), buf.sample_rate, depth, dither)?; + w.write(&buf.channels, frames)?; + Ok(w.finish()?.into_inner()) +} + +/// Refuse the FLAC stream at `at` if a comment or picture field claims more bytes than its +/// metadata block holds: symphonia allocates each field's length before reading it. +pub(crate) fn check_metadata(b: &[u8], at: usize) -> Result<()> { + let mut pos = at + 4; + while let Some(head) = be_u32(b, pos) { + let (body, len) = (pos + 4, (head & 0x00ff_ffff) as usize); + let end = (body + len).min(b.len()); + let overruns = match (head >> 24) & 0x7f { + 4 => comments_overrun(b, body, end), + // Picture: type, MIME type, description, four size fields, data. + 6 => overruns(b, body + 4, end, be_u32, [0, 0, 16]), + _ => false, + }; + if overruns { + return Err(AudioError::Malformed("a FLAC metadata field is larger than its block".into())); + } + if head >> 31 == 1 { + break; + } + pos = body + len; + } + Ok(()) +} + +/// The info of a FLAC stream with no audio frames at all: STREAMINFO declares 0 samples and +/// nothing follows the metadata blocks. Symphonia's reader fails on these, as it looks for a first +/// frame, so the caller answers them itself. +pub(crate) fn empty_stream_info(b: &[u8], at: usize) -> Option { + let mut pos = at.checked_add(4)?; + let mut first = true; + loop { + let head = be_u32(b, pos)?; + let (body, len) = (pos.checked_add(4)?, (head & 0x00ff_ffff) as usize); + let end = body.checked_add(len)?; + if end > b.len() { + return None; + } + if first && ((head >> 24) & 0x7f != 0 || len != 34) { + return None; + } + first = false; + if head >> 31 == 1 { + if end != b.len() { + return None; + } + break; + } + pos = end; + } + // STREAMINFO, from the sample rate on: 20 bits rate, 3 channels - 1, 5 bits per sample - 1, 36 bits total. + let si = at.checked_add(4 + 4 + 10)?; + let packed = u64::from_be_bytes(b.get(si..si.checked_add(8)?)?.try_into().ok()?); + let sample_rate = (packed >> 44) as u32; + let channels = ((packed >> 41) & 0x7) as u16 + 1; + let bps = ((packed >> 36) & 0x1f) as u32 + 1; + if packed & 0xf_ffff_ffff != 0 || sample_rate == 0 { + return None; + } + let sample_format = match bps { + 1..=8 => SampleFormat::Int8, + 9..=16 => SampleFormat::Int16, + 17..=24 => SampleFormat::Int24, + _ => SampleFormat::Int32, + }; + Some(AudioInfo { format: FileFormat::Flac, sample_format, sample_rate, channels, frames: 0, bwf: None }) +} + +/// Whether a vendor string or comment in a (little-endian) Vorbis comment block claims more than +/// the block holds. +fn comments_overrun(b: &[u8], body: usize, end: usize) -> bool { + let Some(vendor) = le_u32(b, body).filter(|_| body.saturating_add(4) <= end) else { return false }; + if vendor as usize > end - (body + 4) { + return true; + } + let at = (body + 4).saturating_add(vendor as usize); + let Some(count) = le_u32(b, at).filter(|_| at.saturating_add(4) <= end) else { return false }; + overruns(b, at + 4, end, le_u32, std::iter::repeat_n(0, count as usize)) +} + +/// Whether one of the length-prefixed fields from `at` claims more than `end` leaves; `gaps` are +/// the bytes skipped before each length, which `read` reads. +fn overruns(b: &[u8], mut at: usize, end: usize, read: fn(&[u8], usize) -> Option, gaps: impl IntoIterator) -> bool { + for gap in gaps { + at = at.saturating_add(gap); + let Some(n) = read(b, at).filter(|_| at.saturating_add(4) <= end) else { return false }; + at += 4; + if n as usize > end - at { + return true; + } + at += n as usize; + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn crc_known_values() { + // CRC-8/SMBUS and CRC-16/UMTS (FLAC's polynomials) of "123456789". + assert_eq!(crc8(b"123456789"), 0xF4); + assert_eq!(crc16(b"123456789"), 0xFEE8); + } + + #[test] + fn utf8_numbers() { + let enc = |v| { + let mut w = BitWriter::new(); + write_utf8_number(&mut w, v); + w.into_bytes() + }; + assert_eq!(enc(0x7F), vec![0x7F]); + assert_eq!(enc(0x80), vec![0xC2, 0x80]); + assert_eq!(enc(0x800), vec![0xE0, 0xA0, 0x80]); + } +} diff --git a/crates/audio-io/src/lib.rs b/crates/audio-io/src/lib.rs new file mode 100644 index 0000000000..bdb9d35277 --- /dev/null +++ b/crates/audio-io/src/lib.rs @@ -0,0 +1,459 @@ +//! SoundCraft audio file I/O. +//! +//! Bytes in, bytes out (no filesystem access, so the crate also builds for `wasm32`): +//! +//! * [`detect_format`] / [`probe`] / [`decode`] read WAV (PCM 8/16/24/32, float 32/64, +//! `WAVE_FORMAT_EXTENSIBLE`, RF64/BW64, BWF `bext`) and AIFF/AIFF-C (big-endian, `sowt`, +//! `fl32`/`fl64`) with native readers, and everything else (FLAC, MP3, Ogg Vorbis, AAC/ALAC in +//! MP4, CAF, Matroska, ADPCM/A-law/µ-law WAV, ...) through `symphonia`. +//! * [`encode`] writes WAV/BWF (RF64 above 4 GiB), AIFF / AIFF-C `fl32`, and FLAC (built-in +//! encoder), with optional TPDF dither. +//! * [`wav_float_header`] heads a float WAV written while it grows (recording). +//! * [`peaks`] builds multi-resolution min/max overviews for waveform drawing. +//! * [`FileEncoder`] writes the same formats block by block into any `Write + Seek`, so a file +//! of any length never has to fit in memory. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod aiff; +mod buffer; +mod caf; +mod flac; +mod mp4; +mod pcm; +pub mod peaks; +mod symph; +mod wav; + +pub use buffer::AudioBuffer; +use std::io::{Seek, Write}; + +/// Errors from probing, decoding and encoding. +#[derive(Debug, thiserror::Error)] +pub enum AudioError { + /// The format, codec or option is valid but not supported. + #[error("unsupported audio: {0}")] + Unsupported(String), + /// The input is corrupt or not what it claims to be. + #[error("malformed audio: {0}")] + Malformed(String), + /// The input (or requested output) exceeds a size limit. + #[error("audio too large: {0}")] + TooLarge(String), + /// Encoding failed (e.g. an empty buffer). + #[error("audio encode failed: {0}")] + Encode(String), + /// Writing the output failed. + #[error("audio write failed: {0}")] + Io(#[from] std::io::Error), +} + +/// Result alias for this crate. +pub type Result = std::result::Result; + +/// Container / codec family of an audio file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum FileFormat { + Wav, + Aiff, + Flac, + Mp3, + Ogg, + Aac, + Alac, + Caf, + /// Any other container `symphonia` reads: Matroska and WebM by name. + Other, +} + +impl FileFormat { + /// The formats [`decode`] reads, found by extension. Contents decide when they disagree. + pub const READ: &'static [FileFormat] = + &[FileFormat::Wav, FileFormat::Aiff, FileFormat::Flac, FileFormat::Mp3, FileFormat::Ogg, FileFormat::Aac, FileFormat::Caf, FileFormat::Other]; + /// The formats [`encode`] writes. + pub const WRITE: &'static [FileFormat] = &[FileFormat::Wav, FileFormat::Aiff, FileFormat::Flac]; + + /// The extensions that name this format, lower case and without the dot; the first is the one + /// a new file gets. ALAC shares AAC's MP4 container, so a `.m4a` name reads as AAC until the + /// contents say otherwise. + pub fn extensions(self) -> &'static [&'static str] { + match self { + FileFormat::Wav => &["wav", "wave", "bwf", "rf64"], + FileFormat::Aiff => &["aif", "aiff", "aifc"], + FileFormat::Flac => &["flac"], + FileFormat::Mp3 => &["mp3", "mp2", "mp1"], + FileFormat::Ogg => &["ogg", "oga"], + FileFormat::Aac => &["m4a", "m4b", "aac", "mp4", "adts"], + FileFormat::Alac => &["m4a"], + FileFormat::Caf => &["caf"], + FileFormat::Other => &["mkv", "webm"], + } + } + + /// The name people know the format by, as menus show it. + pub fn name(self) -> &'static str { + match self { + FileFormat::Wav => "WAV", + FileFormat::Aiff => "AIFF", + FileFormat::Flac => "FLAC", + FileFormat::Mp3 => "MP3", + FileFormat::Ogg => "Ogg Vorbis", + FileFormat::Aac => "AAC", + FileFormat::Alac => "ALAC", + FileFormat::Caf => "CAF", + FileFormat::Other => "Other", + } + } + + /// The media type a file of this format is served or downloaded as. + pub fn mime_type(self) -> &'static str { + match self { + FileFormat::Wav => "audio/wav", + FileFormat::Aiff => "audio/aiff", + FileFormat::Flac => "audio/flac", + FileFormat::Mp3 => "audio/mpeg", + FileFormat::Ogg => "audio/ogg", + FileFormat::Aac | FileFormat::Alac => "audio/mp4", + FileFormat::Caf => "audio/x-caf", + FileFormat::Other => "application/octet-stream", + } + } + + /// The extension a new file of this format gets. + pub fn default_extension(self) -> &'static str { + self.extensions().first().copied().unwrap_or("bin") + } + + /// The readable format an extension or format name names (`wav`, `.AIFF`; ASCII + /// case-insensitive), or `None` when nothing reads it. + pub fn from_extension(ext: &str) -> Option { + let ext = ext.trim_start_matches('.'); + Self::READ.iter().copied().find(|f| f.extensions().iter().any(|x| x.eq_ignore_ascii_case(ext))) + } + + /// The readable format `path`'s extension names, or `None` without a known extension. + pub fn from_path(path: impl AsRef) -> Option { + path.as_ref().extension().and_then(std::ffi::OsStr::to_str).and_then(Self::from_extension) + } + + /// The extensions [`encode`] writes this format under: the names whose readers expect what it + /// writes. RF64 and AIFF-C names are left out because `encode` writes plain RIFF and AIFF + /// headers for them. + pub fn write_extensions(self) -> &'static [&'static str] { + match self { + FileFormat::Wav => &["wav", "wave", "bwf"], + FileFormat::Aiff => &["aif", "aiff"], + FileFormat::Flac => &["flac"], + _ => &[], + } + } + + /// The format [`encode`] writes for an extension or format name (ASCII case-insensitive), or + /// `None` when it writes none. + pub fn from_write_extension(ext: &str) -> Option { + let ext = ext.trim_start_matches('.'); + Self::WRITE.iter().copied().find(|f| f.write_extensions().iter().any(|x| x.eq_ignore_ascii_case(ext))) + } + + /// Every extension of [`FileFormat::READ`], in order: what file pickers and browsers filter by. + pub fn readable_extensions() -> Vec<&'static str> { + Self::READ.iter().flat_map(|f| f.extensions().iter().copied()).collect() + } + + /// Every extension of [`FileFormat::WRITE`], in order: what save dialogs, help and errors list. + pub fn writable_extensions() -> Vec<&'static str> { + Self::WRITE.iter().flat_map(|f| f.write_extensions().iter().copied()).collect() + } +} + +/// Stored sample representation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum SampleFormat { + Int8, + Int16, + Int24, + Int32, + Float32, + Float64, + /// A lossy codec (MP3, AAC, Vorbis, ...): no fixed bit depth. + Compressed, +} + +/// Broadcast Wave (`bext`) metadata. +#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct BwfInfo { + pub description: String, + pub originator: String, + pub originator_reference: String, + /// `yyyy-mm-dd`. + pub origination_date: String, + /// `hh:mm:ss`. + pub origination_time: String, + /// Sample count since midnight of the first sample (the timestamp Pro Tools spots to). + pub time_reference: u64, +} + +/// What [`probe`] learns about a file without decoding all of it. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct AudioInfo { + pub format: FileFormat, + pub sample_format: SampleFormat, + pub sample_rate: u32, + pub channels: u16, + pub frames: u64, + pub bwf: Option, +} + +impl AudioInfo { + /// Duration in seconds. + pub fn duration_secs(&self) -> f64 { + if self.sample_rate == 0 { 0.0 } else { self.frames as f64 / f64::from(self.sample_rate) } + } +} + +/// Bit depth for [`encode`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum BitDepth { + Int16, + Int24, + Int32, + /// Preserve finite samples, including values above full scale; map NaN to 0 and infinities to ±1. + Float32, +} + +/// Options for [`encode`]. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct EncodeOptions { + /// `Wav`, `Aiff` or `Flac`; anything else is [`AudioError::Unsupported`]. + pub format: FileFormat, + pub bit_depth: BitDepth, + /// Add TPDF dither when quantizing to an integer depth. + pub dither: bool, + /// Write a BWF `bext` chunk (WAV only; ignored for other formats). + pub bwf: Option, +} + +impl Default for EncodeOptions { + fn default() -> Self { + Self { format: FileFormat::Wav, bit_depth: BitDepth::Int24, dither: false, bwf: None } + } +} + +/// Size of an ID3v2 tag at the start of `b`, if there is one. The 10-byte footer flag (`0x10`) +/// exists only in ID3v2.4; in v2.2/v2.3 that bit is reserved and must not extend the skip. +fn id3v2_len(b: &[u8]) -> Option { + if b.get(..3)? != b"ID3" { + return None; + } + let version = *b.get(3)?; + let flags = *b.get(5)?; + let size = b.get(6..10)?.iter().fold(0usize, |acc, &x| (acc << 7) | usize::from(x & 0x7F)); + let footer = version == 4 && flags & 0x10 != 0; + Some(10 + size + if footer { 10 } else { 0 }) +} + +fn detect_by_magic(b: &[u8], depth: u32) -> Option { + let head = |n: usize| b.get(..n); + if wav::is_wav(b) { + return Some(FileFormat::Wav); + } + if aiff::is_aiff(b) { + return Some(FileFormat::Aiff); + } + match head(4)? { + b"fLaC" => return Some(FileFormat::Flac), + b"OggS" => return Some(FileFormat::Ogg), + b"caff" => return Some(FileFormat::Caf), + [0x1A, 0x45, 0xDF, 0xA3] => return Some(FileFormat::Other), + _ => {} + } + if b.get(4..8) == Some(b"ftyp") { + // MP4 / M4A: the sample entry inside `stsd` names the codec. + let found = b.windows(4).position(|w| w == b"stsd"); + let codec = found.and_then(|p| b.get(p.checked_add(16)?..p.checked_add(20)?)); + return Some(if codec == Some(b"alac") { FileFormat::Alac } else { FileFormat::Aac }); + } + if let Some(len) = id3v2_len(b) { + if depth == 0 + && let Some(f) = b.get(len..).and_then(|rest| detect_by_magic(rest, 1)) + { + return Some(f); + } + return Some(FileFormat::Mp3); + } + let (b0, b1) = (*b.first()?, *b.get(1)?); + if b0 == 0xFF && b1 & 0xE0 == 0xE0 { + let layer = (b1 >> 1) & 0x3; + if b1 & 0xF0 == 0xF0 && layer == 0 { + return Some(FileFormat::Aac); // ADTS + } + if layer != 0 { + return Some(FileFormat::Mp3); + } + } + None +} + +/// Identify a file by its magic bytes, falling back to the extension hint (`"wav"`, `".flac"`). +pub fn detect_format(bytes: &[u8], ext_hint: Option<&str>) -> FileFormat { + detect_by_magic(bytes, 0).unwrap_or_else(|| ext_hint.and_then(FileFormat::from_extension).unwrap_or(FileFormat::Other)) +} + +/// Read a file's format, sample rate, channel count, length and BWF metadata. +pub fn probe(bytes: &[u8], ext_hint: Option<&str>) -> Result { + let format = detect_format(bytes, ext_hint); + match format { + FileFormat::Wav if wav::is_wav(bytes) => match wav::parse(bytes)? { + wav::WavParse::Native(l) => Ok(l.info), + wav::WavParse::Foreign { bwf } => { + let mut info = symph::probe(bytes, format)?; + info.bwf = bwf; + Ok(info) + } + }, + FileFormat::Aiff if aiff::is_aiff(bytes) => match aiff::parse(bytes)? { + aiff::AiffParse::Native(l) => Ok(l.info), + aiff::AiffParse::Foreign => symph::probe(bytes, format), + }, + _ => symph::probe(bytes, format), + } +} + +/// Decode a whole file to planar `f32`. +pub fn decode(bytes: &[u8], ext_hint: Option<&str>) -> Result<(AudioInfo, AudioBuffer)> { + let format = detect_format(bytes, ext_hint); + match format { + FileFormat::Wav if wav::is_wav(bytes) => match wav::parse(bytes)? { + wav::WavParse::Native(l) => { + let buf = wav::decode(bytes, &l)?; + let mut info = l.info; + info.frames = buf.frames() as u64; + Ok((info, buf)) + } + wav::WavParse::Foreign { bwf } => { + let (mut info, buf) = symph::decode(bytes, format)?; + info.bwf = bwf; + Ok((info, buf)) + } + }, + FileFormat::Aiff if aiff::is_aiff(bytes) => match aiff::parse(bytes)? { + aiff::AiffParse::Native(l) => { + let buf = aiff::decode(bytes, &l)?; + let mut info = l.info; + info.frames = buf.frames() as u64; + Ok((info, buf)) + } + aiff::AiffParse::Foreign => symph::decode(bytes, format), + }, + _ => symph::decode(bytes, format), + } +} + +/// Encode `buf` as WAV/BWF, AIFF/AIFF-C or FLAC. Samples are clamped to [-1, 1]. +pub fn encode(buf: &AudioBuffer, opts: &EncodeOptions) -> Result> { + match opts.format { + FileFormat::Wav => wav::encode(buf, opts.bit_depth, opts.dither, opts.bwf.as_ref(), false, None), + FileFormat::Aiff => aiff::encode(buf, opts.bit_depth, opts.dither), + FileFormat::Flac => flac::encode(buf, opts.bit_depth, opts.dither), + other => Err(AudioError::Unsupported(format!("encoding {other:?} is not supported"))), + } +} + +/// Encode a WAV file in the RF64 layout regardless of size (normally used only above 4 GiB). +pub fn encode_wav_rf64(buf: &AudioBuffer, opts: &EncodeOptions) -> Result> { + wav::encode(buf, opts.bit_depth, opts.dither, opts.bwf.as_ref(), true, None) +} + +/// Like [`encode`], with an explicit WAV speaker mask (`dwChannelMask`, e.g. 0x60F) for layouts +/// a channel count alone cannot identify (7.0 vs 6.1, 5.1.4 vs 7.1.2, Ambisonics = 0). AIFF and +/// FLAC have fixed channel orders and ignore it. +pub fn encode_with_channel_mask(buf: &AudioBuffer, opts: &EncodeOptions, mask: u32) -> Result> { + match opts.format { + FileFormat::Wav => wav::encode(buf, opts.bit_depth, opts.dither, opts.bwf.as_ref(), false, Some(mask)), + _ => encode(buf, opts), + } +} + +/// The error a stream encoder returns for every call after one of its writes has failed: the +/// sink may already hold a prefix of the failed block, so the encoder's counters can no longer +/// describe the bytes on the sink, and later writes or `finish` must not claim success. +pub(crate) fn failed_write_error() -> AudioError { + AudioError::Encode("the stream encoder already failed on an earlier write; its output is incomplete".into()) +} + +/// Writes WAV/BWF (RF64 above 4 GiB), AIFF / AIFF-C `fl32` or FLAC block by block, producing +/// the same bytes [`encode_with_channel_mask`] would for the whole buffer. +pub struct FileEncoder(Writer); + +enum Writer { + Wav(wav::Writer), + Aiff(aiff::Writer), + Flac(flac::Writer), +} + +impl FileEncoder { + /// Starts a file at the current position of `out`. `mask` overrides the WAV speaker mask + /// derived from the channel count. + pub fn new(out: W, opts: &EncodeOptions, channels: usize, sample_rate: u32, mask: Option) -> Result { + let (depth, dither) = (opts.bit_depth, opts.dither); + Ok(FileEncoder(match opts.format { + FileFormat::Wav => Writer::Wav(wav::Writer::new( + out, + channels, + sample_rate, + depth, + dither, + wav::HeaderExtras { bwf: opts.bwf.as_ref(), mask, force_rf64: false }, + )?), + FileFormat::Aiff => Writer::Aiff(aiff::Writer::new(out, channels, sample_rate, depth, dither)?), + FileFormat::Flac => Writer::Flac(flac::Writer::new(out, channels, sample_rate, depth, dither)?), + other => return Err(AudioError::Unsupported(format!("encoding {other:?} is not supported"))), + })) + } + + /// Fails, without writing anything, when `frames` more frames would not fit the format (an + /// AIFF holds 4 GiB, a FLAC 2^36 frames). + pub fn check_room(&self, frames: u64) -> Result<()> { + match &self.0 { + Writer::Wav(w) => w.check_room(frames), + Writer::Aiff(w) => w.check_room(frames), + Writer::Flac(w) => w.check_room(frames), + } + .map(drop) + } + + /// Appends the first `frames` frames of `block`, one `Vec` per channel (missing samples are + /// silence), unless they don't fit (see [`FileEncoder::check_room`]). Once a write has + /// failed, the encoder is unusable: the sink may hold a prefix of the failed block, so + /// every later `write` and `finish` fails instead of completing an inconsistent file. + pub fn write(&mut self, block: &[Vec], frames: usize) -> Result<()> { + match &mut self.0 { + Writer::Wav(w) => w.write(block, frames), + Writer::Aiff(w) => w.write(block, frames), + Writer::Flac(w) => w.write(block, frames), + } + } + + /// Completes the headers and hands back `out`, positioned at the end of the file. + pub fn finish(self) -> Result { + match self.0 { + Writer::Wav(w) => w.finish(), + Writer::Aiff(w) => w.finish(), + Writer::Flac(w) => w.finish(), + } + } +} + +/// The WAVE_FORMAT_EXTENSIBLE speaker mask of a WAV file, if it has one. +pub fn wav_channel_mask(bytes: &[u8]) -> Option { + wav::read_channel_mask(bytes) +} + +/// The header of a 32-bit float WAV holding `frames` frames (little-endian samples follow it). Its +/// length doesn't depend on `frames`, so a recorder can rewrite it in place as the file grows; it +/// switches to RF64 above 4 GiB. +pub fn wav_float_header(channels: usize, sample_rate: u32, frames: u64) -> Result> { + if channels == 0 || sample_rate == 0 { + return Err(AudioError::Encode("a WAV needs channels and a sample rate".into())); + } + wav::header(channels, sample_rate, (32, true), None, None, frames, false) +} diff --git a/crates/audio-io/src/mp4.rs b/crates/audio-io/src/mp4.rs new file mode 100644 index 0000000000..06e68ddc9c --- /dev/null +++ b/crates/audio-io/src/mp4.rs @@ -0,0 +1,106 @@ +//! Size checks on MP4 atoms before symphonia reads them: it allocates whatever an atom, a sample +//! table or a sample claims, and a failed allocation aborts the process. + +use crate::pcm::{be_u32, be_u64}; +use crate::{AudioError, Result}; + +#[derive(Clone, Copy)] +enum Level { + Top, + Moov, + Trak, + Mdia, + Minf, + Stbl, + Udta, + Meta, + Ilst, + Tag, +} + +/// Refuse the MP4 file whose top-level atoms start at `at` if an atom symphonia reads whole, a +/// sample table or a sample claims more bytes than the file holds. +pub(crate) fn check(b: &[u8], at: usize) -> Result<()> { + atoms(b, at, b.len(), Level::Top).map(|_| ()) +} + +fn malformed(what: &str) -> AudioError { + AudioError::Malformed(format!("MP4 {what} is larger than the file")) +} + +/// Walk the atoms in `pos..end` that symphonia descends into, as it reads them. False where +/// symphonia stops reading: at a header it refuses, or after an atom that runs past `end`. +fn atoms(b: &[u8], mut pos: usize, end: usize, level: Level) -> Result { + // A header that starts inside the container is read whole, even past its end. + while pos < end { + let (Some(size), Some(kind)) = (be_u32(b, pos), b.get(pos + 4..pos + 8)) else { return Ok(false) }; + let head = if size == 1 { 16 } else { 8 }; + // Size 0: no data, and the next atom starts at the container's end. + let len = match size { + 0 => head, + 1 => be_u64(b, pos + 8).unwrap_or(0), + n => u64::from(n), + }; + if len < head { + return Ok(false); + } + let body = pos + head as usize; + let data_end = usize::try_from(len).ok().and_then(|n| pos.checked_add(n)); + let next = if size == 0 { Some(end) } else { data_end }; + let inner = data_end.map_or(b.len(), |n| n.min(b.len())); + let more = match (level, kind) { + (Level::Top, b"moov") => atoms(b, body, inner, Level::Moov)?, + (Level::Top | Level::Udta, b"meta") => atoms(b, body + 4, inner, Level::Meta)?, + (Level::Moov, b"trak") => atoms(b, body, inner, Level::Trak)?, + (Level::Moov, b"udta") => atoms(b, body, inner, Level::Udta)?, + (Level::Trak, b"mdia") => atoms(b, body, inner, Level::Mdia)?, + (Level::Mdia, b"minf") => atoms(b, body, inner, Level::Minf)?, + (Level::Minf, b"stbl") => atoms(b, body, inner, Level::Stbl)?, + (Level::Meta, b"ilst") => atoms(b, body, inner, Level::Ilst)?, + (Level::Ilst, _) => atoms(b, body, inner, Level::Tag)?, + (Level::Stbl, b"stts" | b"co64") => table(b, body + 4, 8, inner)?, + (Level::Stbl, b"stsc") => table(b, body + 4, 12, inner)?, + (Level::Stbl, b"stco") => table(b, body + 4, 4, inner)?, + (Level::Stbl, b"stsz") => sample_sizes(b, body, inner)?, + (Level::Mdia, b"hdlr") | (Level::Tag, b"data" | b"name" | b"mean") if len > (b.len() - pos) as u64 => { + return Err(malformed(&format!("`{}` atom", String::from_utf8_lossy(kind)))); + } + _ => true, + }; + match next { + Some(n) if more && n <= end => pos = n, + _ => return Ok(false), + } + } + Ok(true) +} + +/// A sample table's entries, counted at `count`, must fit in the file: symphonia reserves room for +/// all of them first. False if they run past the table's atom (`end`), where symphonia stops. +fn table(b: &[u8], count: usize, entry: u64, end: usize) -> Result { + let Some(n) = be_u32(b, count) else { return Ok(false) }; + let (first, bytes) = (count + 4, u64::from(n) * entry); + if bytes > b.len().saturating_sub(first) as u64 { + return Err(malformed("sample table")); + } + Ok(first as u64 + bytes <= end as u64) +} + +/// Every sample's size must fit in the file: symphonia reads each one into a buffer of that size. +/// Sizes past the atom are never used, as symphonia stops before reading a packet. +fn sample_sizes(b: &[u8], body: usize, end: usize) -> Result { + let (Some(size), Some(count)) = (be_u32(b, body + 4), be_u32(b, body + 8)) else { return Ok(false) }; + let file = b.len() as u64; + if u64::from(size) > file { + return Err(malformed("sample")); + } + if size != 0 { + return Ok(true); + } + let fits = table(b, body + 8, 4, end)?; + let (sizes, _) = b.get(body + 12..end).unwrap_or_default().as_chunks::<4>(); + if sizes.iter().take(count as usize).any(|s| u64::from(u32::from_be_bytes(*s)) > file) { + return Err(malformed("sample")); + } + Ok(fits) +} diff --git a/crates/audio-io/src/pcm.rs b/crates/audio-io/src/pcm.rs new file mode 100644 index 0000000000..6bd4358d91 --- /dev/null +++ b/crates/audio-io/src/pcm.rs @@ -0,0 +1,279 @@ +//! Byte-level PCM helpers shared by the WAV, AIFF and FLAC code: bounded readers, sample +//! decoding to `f32`, and quantization (with optional TPDF dither) for encoding. + +use crate::{AudioError, BitDepth, Result}; +use std::io::Write; + +/// Maximum number of decoded `f32` samples (all channels) we will produce: 4 GiB of output. +pub(crate) const MAX_DECODE_SAMPLES: u64 = (4u64 << 30) / 4; + +/// Little-endian `u16` at `pos`. +pub(crate) fn le_u16(b: &[u8], pos: usize) -> Option { + let s = b.get(pos..pos.checked_add(2)?)?; + Some(u16::from_le_bytes([*s.first()?, *s.get(1)?])) +} + +/// Little-endian `u32` at `pos`. +pub(crate) fn le_u32(b: &[u8], pos: usize) -> Option { + let s: [u8; 4] = b.get(pos..pos.checked_add(4)?)?.try_into().ok()?; + Some(u32::from_le_bytes(s)) +} + +/// Little-endian `u64` at `pos`. +pub(crate) fn le_u64(b: &[u8], pos: usize) -> Option { + let s: [u8; 8] = b.get(pos..pos.checked_add(8)?)?.try_into().ok()?; + Some(u64::from_le_bytes(s)) +} + +/// Big-endian `u16` at `pos`. +pub(crate) fn be_u16(b: &[u8], pos: usize) -> Option { + let s: [u8; 2] = b.get(pos..pos.checked_add(2)?)?.try_into().ok()?; + Some(u16::from_be_bytes(s)) +} + +/// Big-endian `u32` at `pos`. +pub(crate) fn be_u32(b: &[u8], pos: usize) -> Option { + let s: [u8; 4] = b.get(pos..pos.checked_add(4)?)?.try_into().ok()?; + Some(u32::from_be_bytes(s)) +} + +/// Big-endian `u64` at `pos`. +pub(crate) fn be_u64(b: &[u8], pos: usize) -> Option { + let s: [u8; 8] = b.get(pos..pos.checked_add(8)?)?.try_into().ok()?; + Some(u64::from_be_bytes(s)) +} + +/// Four-byte tag at `pos`. +pub(crate) fn tag(b: &[u8], pos: usize) -> Option<[u8; 4]> { + b.get(pos..pos.checked_add(4)?)?.try_into().ok() +} + +/// How the samples in a PCM byte stream are encoded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PcmKind { + /// Signed (or, for 1-byte WAV, unsigned) integers occupying `bytes` bytes each, left-justified. + Int { bytes: usize, big_endian: bool, unsigned: bool }, + /// IEEE float, 4 or 8 bytes. + Float { bytes: usize, big_endian: bool }, +} + +impl PcmKind { + pub(crate) fn bytes(self) -> usize { + match self { + PcmKind::Int { bytes, .. } | PcmKind::Float { bytes, .. } => bytes, + } + } +} + +/// Decode one sample from exactly `kind.bytes()` bytes. Returns 0 for a short slice. +fn decode_sample(s: &[u8], kind: PcmKind) -> f32 { + match kind { + PcmKind::Int { bytes, big_endian, unsigned } => { + if bytes == 0 || bytes > 4 || s.len() < bytes { + return 0.0; + } + let mut acc: u32 = 0; + for i in 0..bytes { + let idx = if big_endian { i } else { bytes - 1 - i }; + let mut byte = s.get(idx).copied().unwrap_or(0); + if unsigned && i == 0 { + byte ^= 0x80; + } + acc = (acc << 8) | u32::from(byte); + } + let shifted = acc << (32 - 8 * bytes as u32); + (f64::from(shifted as i32) / 2_147_483_648.0) as f32 + } + PcmKind::Float { bytes, big_endian } => { + let v = match bytes { + 4 => { + let Some(a) = s.get(..4).and_then(|x| <[u8; 4]>::try_from(x).ok()) else { return 0.0 }; + if big_endian { f32::from_be_bytes(a) } else { f32::from_le_bytes(a) } + } + 8 => { + let Some(a) = s.get(..8).and_then(|x| <[u8; 8]>::try_from(x).ok()) else { return 0.0 }; + (if big_endian { f64::from_be_bytes(a) } else { f64::from_le_bytes(a) }) as f32 + } + _ => 0.0, + }; + if v.is_finite() { v } else { 0.0 } + } + } +} + +/// Deinterleave `frames` frames of `channels`-channel PCM from `data` into planar `f32`. +pub(crate) fn deinterleave(data: &[u8], kind: PcmKind, channels: usize, frames: u64) -> Result>> { + let bytes = kind.bytes(); + let block = bytes.checked_mul(channels).filter(|&b| b > 0).ok_or_else(|| AudioError::Malformed("zero-sized sample frame".into()))?; + let available = (data.len() / block) as u64; + let frames = frames.min(available); + let total = frames.checked_mul(channels as u64).ok_or_else(|| AudioError::TooLarge("sample count overflows".into()))?; + if total > MAX_DECODE_SAMPLES { + return Err(AudioError::TooLarge(format!("{total} samples exceeds the {MAX_DECODE_SAMPLES}-sample decode limit"))); + } + let frames = usize::try_from(frames).map_err(|_| AudioError::TooLarge("frame count exceeds address space".into()))?; + let mut out: Vec> = (0..channels).map(|_| Vec::with_capacity(frames)).collect(); + for frame in data.chunks_exact(block).take(frames) { + for (ch, s) in out.iter_mut().zip(frame.chunks_exact(bytes)) { + ch.push(decode_sample(s, kind)); + } + } + Ok(out) +} + +/// Converts `f32` samples to integers of a given bit depth, clamping to [-1, 1] and optionally +/// adding triangular (TPDF) dither of ±1 LSB. +pub(crate) struct Quantizer { + bits: u32, + dither: bool, + rng: u64, +} + +impl Quantizer { + /// A quantizer for one channel; each channel gets an independent dither sequence. + pub(crate) fn new(bits: u32, dither: bool, channel: usize) -> Self { + let rng = 0x9E37_79B9_7F4A_7C15u64 ^ (channel as u64).wrapping_add(1).wrapping_mul(0xD1B5_4A32_D192_ED03); + Self { bits: bits.clamp(2, 32), dither, rng: if rng == 0 { 1 } else { rng } } + } + + /// One quantizer per channel. + pub(crate) fn per_channel(bits: u32, dither: bool, channels: usize) -> Vec { + (0..channels).map(|c| Self::new(bits, dither, c)).collect() + } + + /// Uniform in [0, 1) from a xorshift64* generator (deterministic, no OS entropy needed). + fn uniform(&mut self) -> f64 { + let mut x = self.rng; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.rng = x; + (x.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 11) as f64 / (1u64 << 53) as f64 + } + + pub(crate) fn quantize(&mut self, x: f32) -> i32 { + let x = if x.is_nan() { 0.0 } else { x.clamp(-1.0, 1.0) }; + let scale = (1u64 << (self.bits - 1)) as f64; + let mut v = f64::from(x) * scale; + if self.dither { + v += self.uniform() - self.uniform(); + } + let max = scale - 1.0; + round_half_away(v.clamp(-scale, max)) as i32 + } +} + +#[inline] +fn round_half_away(v: f64) -> f64 { + let t = v as i64; + let f = v - t as f64; + (t + i64::from(f >= 0.5) - i64::from(f <= -0.5)) as f64 +} + +/// Bits per sample at `depth`, and whether they are float. +pub(crate) fn sample_format(depth: BitDepth) -> (u16, bool) { + match depth { + BitDepth::Int16 => (16, false), + BitDepth::Int24 => (24, false), + BitDepth::Int32 => (32, false), + BitDepth::Float32 => (32, true), + } +} + +/// Interleaves `frames` frames of planar audio (missing samples are silence) as `bits`-bit +/// samples and writes them to `out`, a few thousand frames at a time through `scratch`. +pub(crate) fn write_interleaved( + out: &mut impl Write, + scratch: &mut Vec, + block: &[Vec], + frames: usize, + quantizers: &mut [Quantizer], + (bits, is_float): (u16, bool), + big_endian: bool, +) -> std::io::Result<()> { + let size = usize::from(bits / 8); + let mut start = 0; + while start < frames { + let end = frames.min(start + 4096); + scratch.clear(); + for f in start..end { + for (c, q) in quantizers.iter_mut().enumerate() { + let s = block.get(c).and_then(|ch| ch.get(f)).copied().unwrap_or(0.0); + match (is_float, big_endian) { + (true, false) => scratch.extend_from_slice(&sanitize_float(s).to_le_bytes()), + (true, true) => scratch.extend_from_slice(&sanitize_float(s).to_be_bytes()), + (false, false) => scratch.extend_from_slice(q.quantize(s).to_le_bytes().get(..size).unwrap_or_default()), + (false, true) => scratch.extend_from_slice(q.quantize(s).to_be_bytes().get(4 - size..).unwrap_or_default()), + } + } + } + out.write_all(scratch)?; + start = end; + } + Ok(()) +} + +/// Preserve finite float samples, mapping NaN to 0 and infinities to their sign (±1). +fn sanitize_float(x: f32) -> f32 { + if x.is_finite() { + x + } else if x.is_nan() { + 0.0 + } else { + x.signum() + } +} + +/// Check that a buffer is encodable: at least one channel and a non-zero sample rate. +pub(crate) fn validate_buffer(buf: &crate::AudioBuffer) -> Result<()> { + validate_layout(buf.channels.len(), buf.sample_rate) +} + +/// Check that a stream is encodable: at least one channel and a non-zero sample rate. +pub(crate) fn validate_layout(channels: usize, sample_rate: u32) -> Result<()> { + if channels == 0 { + return Err(AudioError::Encode("buffer has no channels".into())); + } + if channels > usize::from(u16::MAX) { + return Err(AudioError::Encode("too many channels".into())); + } + if sample_rate == 0 { + return Err(AudioError::Encode("sample rate is 0".into())); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_half_away_matches_f64_round_on_the_quantizer_range() { + let edges = [ + 0.0, + -0.0, + 0.5, + -0.5, + 1.5, + -1.5, + 2.5, + 0.499_999_999_999_999_94, + -0.499_999_999_999_999_94, + 8_388_607.5, + -8_388_608.0, + 2_147_483_647.0, + -2_147_483_648.0, + ]; + let mut x = 0x2545_f491_4f6c_dd1du64; + let spread = (0..10_000).map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 11) as f64 / (1u64 << 53) as f64 * 4_294_967_295.0 - 2_147_483_648.0 + }); + for v in edges.into_iter().chain(spread) { + assert!(round_half_away(v) == v.round(), "{v}"); + assert_eq!(round_half_away(v) as i32, v.round() as i32, "{v}"); + } + } +} diff --git a/crates/audio-io/src/peaks.rs b/crates/audio-io/src/peaks.rs new file mode 100644 index 0000000000..6cd8f32170 --- /dev/null +++ b/crates/audio-io/src/peaks.rs @@ -0,0 +1,280 @@ +//! Multi-resolution overview for waveform drawing. +//! +//! Level 0 sums up each block of [`BLOCK`] samples (min, max and sum of squares); each following +//! level merges [`FANOUT`] blocks of the level below, until a level has a single block. Range +//! queries combine the coarsest whole blocks that fit inside the range and read the partial blocks +//! at its edges from the samples, so a query is exact and costs O(log n) regardless of zoom. + +/// Samples per level-0 block. +pub const BLOCK: usize = 32; +/// Blocks of level `n` merged into one block of level `n + 1`. +pub const FANOUT: usize = 4; +/// Upper bound on the number of spans [`Peaks::spans`] will produce. +pub const MAX_COLUMNS: usize = 1 << 20; + +/// Multi-resolution overview of one channel for waveform drawing. +#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Peaks { + len: usize, + levels: Vec>, +} + +/// What a run of samples sums up to; NaN samples count for nothing. +#[derive(Debug, Clone, Copy, PartialEq, serde::Serialize, serde::Deserialize)] +struct Block { + min: f32, + max: f32, + squares: f64, +} + +impl Block { + const EMPTY: Block = Block { min: f32::INFINITY, max: f32::NEG_INFINITY, squares: 0.0 }; + + fn point(s: f32) -> Block { + Block { min: s, max: s, squares: if s.is_finite() { f64::from(s) * f64::from(s) } else { 0.0 } } + } + + fn of(samples: &[f32]) -> Block { + samples.iter().fold(Block::EMPTY, |b, &s| b.merge(Block::point(s))) + } + + fn merge(self, o: Block) -> Block { + Block { min: self.min.min(o.min), max: self.max.max(o.max), squares: self.squares + o.squares } + } + + /// (min, max), or (0, 0) when nothing was summed up. + fn range(self) -> (f32, f32) { + if self.min > self.max { (0.0, 0.0) } else { (self.min, self.max) } + } +} + +impl Peaks { + /// Build the overview of `samples` (one channel). NaN samples are ignored. + pub fn build(samples: &[f32]) -> Peaks { + if samples.is_empty() { + return Peaks::default(); + } + let mut levels: Vec> = vec![samples.chunks(BLOCK).map(Block::of).collect()]; + while let Some(last) = levels.last().filter(|l| l.len() > 1) { + let next = last.chunks(FANOUT).map(|c| c.iter().fold(Block::EMPTY, |a, &b| a.merge(b))).collect(); + levels.push(next); + } + Peaks { len: samples.len(), levels } + } + + /// Number of samples the overview was built from. + pub fn len_samples(&self) -> usize { + self.len + } + + /// Number of resolution levels (0 for an empty overview). + pub fn num_levels(&self) -> usize { + self.levels.len() + } + + /// (min, max) over the sample range `[start, end)`, clamped to the data. `samples` must be the + /// data the overview was built from. Returns `(0.0, 0.0)` for an empty range. + pub fn minmax(&self, samples: &[f32], start: usize, end: usize) -> (f32, f32) { + self.sum(samples, start, end).range() + } + + fn sum(&self, samples: &[f32], start: usize, end: usize) -> Block { + let end = end.min(self.len).min(samples.len()); + let (first, last) = (start.div_ceil(BLOCK), end / BLOCK); + let raw = |a: usize, b: usize| Block::of(samples.get(a..b).unwrap_or_default()); + if first >= last { + return raw(start, end); + } + raw(start, first * BLOCK).merge(raw(last * BLOCK, end)).merge(self.blocks(first, last)) + } + + /// The level-0 blocks `[lo, hi)`, summed from the coarsest blocks that fit. + fn blocks(&self, mut lo: usize, mut hi: usize) -> Block { + let mut acc = Block::EMPTY; + for (lvl, level) in self.levels.iter().enumerate() { + let mut add = |i: usize| acc = acc.merge(level.get(i).copied().unwrap_or(Block::EMPTY)); + if lvl + 1 == self.levels.len() || hi.saturating_sub(lo) < 2 * FANOUT { + (lo..hi).for_each(add); + break; + } + while !lo.is_multiple_of(FANOUT) { + add(lo); + lo += 1; + } + while !hi.is_multiple_of(FANOUT) { + hi -= 1; + add(hi); + } + lo /= FANOUT; + hi /= FANOUT; + } + acc + } + + /// The spans of sample positions `[start + c * step, start + (c + 1) * step + overlap]` that + /// reach the data, clamped to it. + fn windows(&self, samples: &[f32], start: f64, step: f64, overlap: f64, cols: usize) -> impl Iterator> { + let n = self.len.min(samples.len()); + let valid = n > 0 && start.is_finite() && step.is_finite() && step > 0.0 && overlap.is_finite() && overlap >= 0.0; + let last = n.saturating_sub(1) as f64; + (0..cols.min(MAX_COLUMNS)).map(move |c| { + let (a, b) = (start + step * c as f64, start + step * (c + 1) as f64 + overlap); + (valid && b >= 0.0 && a <= last).then(|| (a.max(0.0), b.min(last))) + }) + } + + /// (min, max) of the line through `samples` over each of `cols` spans of sample positions + /// `[start + c * step, start + (c + 1) * step + overlap]`; fractional positions interpolate + /// between samples. Neighbouring spans share at least their boundary, so a waveform drawn from + /// them stays connected at any zoom. `None` for spans outside the data. + pub fn spans(&self, samples: &[f32], start: f64, step: f64, overlap: f64, cols: usize) -> Vec> { + let at = |t: f64| { + let i = t.floor(); + let a = samples.get(i as usize).copied().unwrap_or(0.0); + let b = samples.get(i as usize + 1).copied().unwrap_or(a); + Block::point(a + (b - a) * (t - i) as f32) + }; + self.windows(samples, start, step, overlap, cols) + .map(|w| w.map(|(a, b)| self.sum(samples, a.ceil() as usize, b.floor() as usize + 1).merge(at(a)).merge(at(b)).range())) + .collect() + } + + /// The RMS of `samples` over each span [`Peaks::spans`] gives, including the samples just + /// outside a span's ends. + pub fn rms_spans(&self, samples: &[f32], start: f64, step: f64, overlap: f64, cols: usize) -> Vec> { + self.windows(samples, start, step, overlap, cols) + .map(|w| { + w.map(|(a, b)| { + let (s, e) = (a.floor() as usize, (b.ceil() as usize).saturating_add(1).min(self.len).min(samples.len())); + (self.sum(samples, s, e).squares / e.saturating_sub(s).max(1) as f64).sqrt() as f32 + }) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn brute(samples: &[f32], start: usize, end: usize) -> (f32, f32) { + let s = &samples[start..end]; + (s.iter().copied().fold(f32::INFINITY, f32::min), s.iter().copied().fold(f32::NEG_INFINITY, f32::max)) + } + + fn signal(n: usize) -> Vec { + (0..n).map(|i| ((i as f32 * 0.037).sin() * (i as f32 * 0.0011).cos()) + if i % 997 == 0 { 0.5 } else { 0.0 }).collect() + } + + #[test] + fn every_range_is_exact() { + let x = signal(100_003); + let p = Peaks::build(&x); + for &(a, b) in &[(0, 32), (0, 100_003), (32, 64_000), (3200, 3232), (96, 99_968), (64, 100_003), (5, 6), (31, 33), (1000, 1031)] { + assert_eq!(p.minmax(&x, a, b), brute(&x, a, b), "range {a}..{b}"); + } + let mut seed = 12345u64; + for _ in 0..2000 { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + let a = (seed >> 33) as usize % 100_003; + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + let b = a + 1 + (seed >> 33) as usize % (100_003 - a); + assert_eq!(p.minmax(&x, a, b), brute(&x, a, b), "range {a}..{b}"); + } + } + + #[test] + fn degenerate_inputs() { + let p = Peaks::build(&[]); + assert_eq!(p.len_samples(), 0); + assert_eq!(p.minmax(&[], 0, 10), (0.0, 0.0)); + assert_eq!(p.spans(&[], 0.0, 1.0, 0.0, 4), vec![None; 4]); + let x = [0.5, -0.25]; + let p = Peaks::build(&x); + assert_eq!(p.minmax(&x, 0, 2), (-0.25, 0.5)); + assert_eq!(p.minmax(&x, 5, 3), (0.0, 0.0)); + assert_eq!(p.minmax(&x, 0, usize::MAX), (-0.25, 0.5)); + assert_eq!(p.minmax(&x[..1], 0, 2), (0.5, 0.5)); + for (start, step, overlap) in [ + (f64::NAN, 1.0, 0.0), + (0.0, 0.0, 0.0), + (0.0, -1.0, 0.0), + (0.0, f64::INFINITY, 0.0), + (f64::NEG_INFINITY, 1.0, 0.0), + (0.0, 1.0, -0.5), + (0.0, 1.0, f64::NAN), + ] { + assert_eq!(p.spans(&x, start, step, overlap, 3), vec![None; 3]); + } + assert_eq!(p.spans(&x, 0.0, 1.0, 0.0, 0).len(), 0); + assert_eq!(p.spans(&x, 0.0, 1.0, 0.0, usize::MAX).len(), MAX_COLUMNS); + let x = [f32::NAN, 1.0]; + let p = Peaks::build(&x); + assert_eq!(p.minmax(&x, 0, 2), (1.0, 1.0)); + assert_eq!(p.spans(&x, 0.0, 1.0, 0.0, 1), vec![Some((1.0, 1.0))]); + assert!(p.rms_spans(&x, 0.0, 1.0, 0.0, 1)[0].is_some_and(f32::is_finite)); + } + + #[test] + fn finite_large_samples_keep_finite_rms() { + let samples = vec![1e20_f32; BLOCK]; + let peaks = Peaks::build(&samples); + assert_eq!(peaks.rms_spans(&samples, 0.0, BLOCK as f64, 0.0, 1), vec![Some(1e20)]); + } + + #[test] + fn rms_spans_match_the_samples() { + let x = signal(20_000); + let p = Peaks::build(&x); + let rms = |s: usize, e: usize| (x[s..e].iter().map(|&v| f64::from(v) * f64::from(v)).sum::() / (e - s) as f64).sqrt() as f32; + for (step, overlap) in [(1.5, 0.0), (37.0, 37.0), (1000.0, 0.0), (4096.0, 512.0)] { + let cols = ((19_999.0 - overlap) / step) as usize; + for (c, got) in p.rms_spans(&x, 0.0, step, overlap, cols).into_iter().enumerate() { + let (a, b) = (c as f64 * step, ((c + 1) as f64 * step + overlap).min(19_999.0)); + let (got, want) = (got.unwrap(), rms(a.floor() as usize, (b.ceil() as usize + 1).min(20_000))); + assert!((got - want).abs() <= want * 1e-4 + 1e-6, "span {c} at step {step}: {got} vs {want}"); + } + } + let sine: Vec = (0..48_000).map(|i| (i as f32 * 0.01).sin()).collect(); + let p = Peaks::build(&sine); + let r = p.rms_spans(&sine, 0.0, 47_000.0, 0.0, 1)[0].unwrap(); + assert!((r - std::f32::consts::FRAC_1_SQRT_2).abs() < 1e-3, "{r}"); + assert_eq!(p.rms_spans(&sine, f64::NAN, 1.0, 0.0, 2), vec![None, None]); + assert_eq!(p.rms_spans(&sine, 50_000.0, 1.0, 0.0, 1), vec![None]); + } + + #[test] + fn spans_cover_their_samples_and_touch_their_neighbours() { + let x = signal(10_000); + let p = Peaks::build(&x); + for step in [0.3, 1.0, 7.5, 100.0, 333.3] { + let cols = (9_999.0 / step) as usize; + let spans = p.spans(&x, 0.0, step, 0.0, cols); + for (c, s) in spans.iter().enumerate() { + let (lo, hi) = s.unwrap_or_else(|| panic!("span {c} at step {step}")); + let (a, b) = ((c as f64 * step).ceil() as usize, ((c + 1) as f64 * step).floor() as usize); + if a <= b { + let truth = brute(&x, a, b + 1); + assert!(lo <= truth.0 && hi >= truth.1, "span {c} at step {step}"); + } + if let Some(Some((nlo, nhi))) = spans.get(c + 1) { + assert!(*nlo <= hi && *nhi >= lo, "spans {c} and {} at step {step} don't touch", c + 1); + } + } + let all = spans.iter().flatten().fold((f32::INFINITY, f32::NEG_INFINITY), |a, s| (a.0.min(s.0), a.1.max(s.1))); + assert_eq!(all, brute(&x, 0, ((cols as f64 * step).floor() as usize + 1).min(10_000)), "step {step}"); + } + } + + #[test] + fn spans_follow_the_line_between_samples() { + let x = [0.0, 1.0, -1.0, 0.5]; + let p = Peaks::build(&x); + assert_eq!(p.spans(&x, 0.0, 0.25, 0.0, 4), vec![Some((0.0, 0.25)), Some((0.25, 0.5)), Some((0.5, 0.75)), Some((0.75, 1.0))]); + assert_eq!(p.spans(&x, 0.5, 1.0, 0.0, 1), vec![Some((0.0, 1.0))]); + let spans = vec![None, Some((0.0, 0.0)), Some((0.0, 1.0)), Some((-1.0, 1.0)), Some((-1.0, 0.5)), Some((0.5, 0.5)), None]; + assert_eq!(p.spans(&x, -2.0, 1.0, 0.0, 7), spans); + assert_eq!(p.spans(&x, 0.0, 1.0, 1.0, 3), vec![Some((-1.0, 1.0)), Some((-1.0, 1.0)), Some((-1.0, 0.5))]); + assert_eq!(p.spans(&x, 0.0, 1.0, 0.5, 3), vec![Some((0.0, 1.0)), Some((-1.0, 1.0)), Some((-1.0, 0.5))]); + } +} diff --git a/crates/audio-io/src/symph.rs b/crates/audio-io/src/symph.rs new file mode 100644 index 0000000000..ca96b96486 --- /dev/null +++ b/crates/audio-io/src/symph.rs @@ -0,0 +1,751 @@ +//! Decode path for everything that isn't native PCM WAV/AIFF, backed by `symphonia`. + +use std::io::{Cursor, Seek, SeekFrom}; +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use symphonia::core::codecs::audio::well_known::{ + CODEC_ID_AAC, CODEC_ID_ADPCM_G722, CODEC_ID_ADPCM_G726, CODEC_ID_ADPCM_G726LE, CODEC_ID_ADPCM_IMA_QT, CODEC_ID_ADPCM_IMA_WAV, CODEC_ID_ADPCM_MS, + CODEC_ID_ALAC, CODEC_ID_FLAC, CODEC_ID_MP1, CODEC_ID_MP2, CODEC_ID_MP3, CODEC_ID_OPUS, CODEC_ID_PCM_ALAW, CODEC_ID_PCM_F32BE, CODEC_ID_PCM_F32LE, + CODEC_ID_PCM_F64BE, CODEC_ID_PCM_F64LE, CODEC_ID_PCM_MULAW, CODEC_ID_VORBIS, +}; +use symphonia::core::codecs::audio::{AudioCodecId, AudioCodecParameters, AudioDecoderOptions, CODEC_ID_NULL_AUDIO}; +use symphonia::core::errors::Error as SymError; +use symphonia::core::formats::probe::{Hint, ProbeOptions, ProbeableFormat, Score, Scoreable}; +use symphonia::core::formats::{FormatOptions, FormatReader, Track}; +use symphonia::core::io::{MediaSourceStream, MediaSourceStreamOptions, ScopedStream}; +use symphonia::core::meta::MetadataOptions; + +use crate::pcm::{MAX_DECODE_SAMPLES, be_u32}; +use crate::{AudioBuffer, AudioError, AudioInfo, FileFormat, Result, SampleFormat}; + +/// The most frames a packet may declare: decoders allocate a packet's frames up front, and ALAC +/// takes its frame length from the file (encoders use 4096). +const MAX_PACKET_FRAMES: u64 = 1 << 20; + +/// How far [`first_marker`] looks for the first format marker: the same 1 MiB a default symphonia +/// probe scans. +const MAX_PREFIX_SCAN: usize = 1 << 20; + +/// How many MPEG/ADTS candidates [`first_marker`] may hand to the readers' expensive `Scoreable` +/// implementations. A crafted 1 MiB prefix of repeated sync markers otherwise asks for hundreds +/// of thousands of scores, each allowed up to symphonia's 16 KiB score depth. Once the budget is +/// spent, the scan stops at the next audio candidate: a container-like byte string later in an +/// unscored audio payload must not become authoritative. A container marker reached first is still +/// checked; an unchecked one the probe selects is refused after probing. +const MAX_AUDIO_SCORE_ATTEMPTS: usize = 256; + +/// A per-scan budget for `Scoreable::score` calls. Taking a slot consumes it even when the score +/// rejects the candidate, so a stream of false syncs cannot multiply the work. +struct ScoreBudget { + used: usize, +} + +impl ScoreBudget { + /// Start a scan with no expensive score attempts spent. + fn new() -> Self { + Self { used: 0 } + } + + /// Spend one score attempt, returning false once the per-scan budget is exhausted. + fn take(&mut self) -> bool { + if self.exhausted() { + return false; + } + self.used = self.used.saturating_add(1); + true + } + + /// Whether no attempt is left for this scan. + fn exhausted(&self) -> bool { + self.used >= MAX_AUDIO_SCORE_ATTEMPTS + } +} + +/// Turn a symphonia error into the audio layer's error, keeping "unsupported" and "too large" +/// distinct from a malformed file. +fn map_err(e: SymError) -> AudioError { + match e { + SymError::Unsupported(m) => AudioError::Unsupported(m.to_string()), + SymError::LimitError(m) => AudioError::TooLarge(m.to_string()), + other => AudioError::Malformed(other.to_string()), + } +} + +/// Run `f`, turning a panic inside the third-party decoder into an error. +fn guarded(f: impl FnOnce() -> Result) -> Result { + catch_unwind(AssertUnwindSafe(f)).unwrap_or_else(|_| Err(AudioError::Malformed("decoder failed on malformed input".into()))) +} + +/// What [`open`] found: a reader, or a stream that holds no audio frames and has no reader. +enum Opened { + Reader(Box), + Empty(AudioInfo), +} + +/// Decoder options for every compressed format. Gapless playback used to be a +/// `FormatOptions` flag in symphonia 0.5; in 0.6 the decoder option (on by default) trims the +/// encoder delay and padding that the demuxer reports on each packet. +fn decoder_options() -> AudioDecoderOptions { + AudioDecoderOptions::default().gapless(true) +} + +/// The first byte after any ID3v2 tags at the start of `b`. Symphonia's tag reader allocates +/// whatever a frame claims, so the tags are skipped, not read: they are not used here. +fn skip_id3v2(b: &[u8]) -> usize { + let mut at = 0usize; + loop { + match b.get(at..).and_then(crate::id3v2_len) { + Some(n) if n > 0 => at = at.saturating_add(n), + _ => return at, + } + } +} + +/// The first format marker the probe can reach from `start`, within its 1 MiB scan limit. +#[derive(Clone, Copy)] +enum FirstMarker { + Flac(usize), + Caf(usize), + Mp4(usize), + Wav(usize), + /// MPEG audio, Ogg, Matroska or AIFF: formats with no separate container size check. + Audio, +} + +/// What the preflight container checks covered before the probe built a reader. +enum Precheck { + /// A zero-frame FLAC: returned without a reader, because Symphonia 0.6's reader needs a first + /// audio frame to resynchronize to. + Empty(AudioInfo), + /// The size checks ran for the container the probe reports under this short name. + Checked(&'static str), + /// No container size check was needed. + Unchecked, +} + +/// Probe short names whose demuxer allocates from sizes in the file, so the matching container +/// check has to run before the stream is decoded. +const SIZE_CHECKED_PROBE_FORMATS: &[&str] = &["caf", "flac", "isomp4", "wave"]; + +/// The container markers to check for the family `detected` names. `None` is a raw audio stream +/// (MP3, Ogg, AIFF), whose payload may contain container-like bytes: the unrestricted scan below +/// finds a container only when one starts the stream, before any audio marker. +fn container_markers(detected: FileFormat) -> Option<&'static [&'static [u8; 4]]> { + match detected { + // RF64 and BW64 are the >4 GiB RIFF forms. + FileFormat::Wav => Some(&[b"RIFF", b"RF64", b"BW64"]), + FileFormat::Flac => Some(&[b"fLaC"]), + FileFormat::Caf => Some(&[b"caff"]), + // MP4/M4A put `ftyp` after the first atom's size field. + FileFormat::Aac | FileFormat::Alac => Some(&[b"ftyp"]), + // Unknown files may carry junk before the container; symphonia's probe scans for markers. + FileFormat::Other => Some(&[b"caff", b"ftyp", b"fLaC", b"RIFF"]), + FileFormat::Aiff | FileFormat::Mp3 | FileFormat::Ogg => None, + } +} + +/// The probe short name a container marker stands for. +fn expected_probe_format(marker: &[u8; 4]) -> &'static str { + match marker { + b"caff" => "caf", + // The isomp4 reader reads the `ftyp` atom. + b"ftyp" => "isomp4", + b"fLaC" => "flac", + // RIFF, RF64 and BW64. + _ => "wave", + } +} + +/// Find the first marker of `markers` at or after `at`, within symphonia's probe depth. A RIFF +/// form other than WAVE is skipped, as the probe skips it and reads a WAVE behind it. +fn find_container_marker<'a>(bytes: &[u8], at: usize, markers: &'a [&'static [u8; 4]]) -> Option<(usize, &'a [u8; 4])> { + let end = bytes.len().min(at.saturating_add(MAX_PREFIX_SCAN)); + for pos in at..end.saturating_sub(3) { + let window = bytes.get(pos..pos + 4)?; + if let Some(marker) = markers.iter().find(|marker| window == marker.as_slice()) { + if matches!(*marker, b"RIFF" | b"RF64" | b"BW64") && bytes.get(pos + 8..pos + 12) != Some(&b"WAVE"[..]) { + continue; + } + return Some((pos, *marker)); + } + } + None +} + +/// The scan variant for a marker found by [`find_container_marker`], with the probe short name its +/// size check stands for. `ftyp` sits four bytes into its atom, so the MP4 check starts there. +fn marker_kind(at: usize, marker: &[u8; 4]) -> (FirstMarker, &'static str) { + match marker { + b"fLaC" => (FirstMarker::Flac(at), expected_probe_format(marker)), + b"caff" => (FirstMarker::Caf(at), expected_probe_format(marker)), + b"ftyp" => (FirstMarker::Mp4(at.saturating_sub(4)), expected_probe_format(marker)), + _ => (FirstMarker::Wav(at), expected_probe_format(marker)), + } +} + +/// The probe short name a container scan variant stands for. +fn marker_name(marker: FirstMarker) -> &'static str { + match marker { + FirstMarker::Flac(_) => "flac", + FirstMarker::Caf(_) => "caf", + FirstMarker::Mp4(_) => "isomp4", + FirstMarker::Wav(_) => "wave", + FirstMarker::Audio => "", + } +} + +/// Whether `rest` starts with one of `R`'s exact registered probe markers. +fn has_probe_marker>(rest: &[u8]) -> bool { + R::probe_data().iter().any(|data| data.spec.markers.iter().any(|marker| rest.starts_with(marker))) +} + +/// Score `R` at `at` exactly as symphonia's private probe does: `Supported` wins, EOF and every +/// other scoring error mean "not this format", while a real I/O error aborts the probe. +fn scores_supported(stream: &mut MediaSourceStream<'_>, at: usize, depth: u16) -> Result { + stream.seek(SeekFrom::Start(u64::try_from(at).unwrap_or(u64::MAX)))?; + match R::score(ScopedStream::new(stream, u64::from(depth))) { + Ok(Score::Supported(_)) => Ok(true), + Err(SymError::IoError(e)) if e.kind() != std::io::ErrorKind::UnexpectedEof => Err(AudioError::Malformed(e.to_string())), + _ => Ok(false), + } +} + +/// Whether `b[at..]` is an MPEG/ADTS candidate the probe would actually select: an exact +/// `probe_data()` marker and a `Supported` score from the same reader. A loose `0xFFEx` pair is +/// not enough: `0xFFE0` is neither a registered marker nor a valid frame, so it must not stop the +/// scan before a later FLAC/CAF/MP4 candidate. +fn audio_sync_at<'a>(b: &'a [u8], at: usize, stream: &mut Option>, budget: &mut ScoreBudget) -> Result { + let rest = b.get(at..).unwrap_or_default(); + let adts = has_probe_marker::>(rest); + let mpa = has_probe_marker::>(rest); + if !adts && !mpa { + return Ok(false); + } + if stream.is_none() { + *stream = Some(MediaSourceStream::new(Box::new(Cursor::new(b)), MediaSourceStreamOptions::default())); + } + let Some(mss) = stream.as_mut() else { return Ok(false) }; + let depth = ProbeOptions::default().max_score_depth; + if adts && budget.take() && scores_supported::>(mss, at, depth)? { + return Ok(true); + } + if mpa && budget.take() && scores_supported::>(mss, at, depth)? { + return Ok(true); + } + Ok(false) +} + +/// Find the first marker that can start a container or audio stream. Symphonia's `Probe::probe` +/// constructs the reader itself and exposes no pre-construction hook, while checking every magic +/// in the scan window would validate bytes the probe never reaches (for example a valid MP3's +/// audio payload). Stopping at the first marker avoids that: it is the candidate the probe selects +/// for FLAC, CAF, MP4, Ogg and Matroska, and for WAV/AIFF once their full form is verified below. +/// MPEG/ADTS candidates are checked with the readers' own `probe_data()`/`Scoreable` APIs, so a +/// false sync byte pair cannot stop the scan. If that scoring budget runs out, the scan stops at +/// the next audio candidate instead of trusting a container-like byte string later in the unscored +/// audio payload. A container marker reached before any such candidate is still preflighted; if +/// the probe selects a size-checked container the scan did not check, `open` refuses it. +/// +/// WAV candidates include RF64/BW64 so `check_info` can run, although symphonia 0.6 registers only +/// the RIFF marker. Native PCM RF64/BW64 is parsed by `wav::parse` before this path; a foreign +/// RF64/BW64 stream therefore remains unsupported by the probe rather than misdetected. +fn first_marker(b: &[u8], start: usize) -> Result> { + first_marker_with_budget(b, start, &mut ScoreBudget::new()) +} + +/// [`first_marker`] with an explicit score budget, so a test can prove a prefix crowded with +/// false sync markers performs only the bounded number of reader scores. +fn first_marker_with_budget(b: &[u8], start: usize, budget: &mut ScoreBudget) -> Result> { + let end = b.len().min(start.saturating_add(MAX_PREFIX_SCAN)); + let mut at = start; + let mut audio_probe = None; + while at.saturating_add(4) <= end { + let rest = b.get(at..).unwrap_or_default(); + if rest.get(..4) == Some(b"fLaC".as_slice()) { + return Ok(Some(FirstMarker::Flac(at))); + } + if rest.get(..4) == Some(b"caff".as_slice()) { + return Ok(Some(FirstMarker::Caf(at))); + } + if rest.get(..4) == Some(b"OggS".as_slice()) || rest.get(..4) == Some(b"\x1A\x45\xDF\xA3".as_slice()) || crate::aiff::is_aiff(rest) { + return Ok(Some(FirstMarker::Audio)); + } + if crate::wav::is_wav(rest) { + return Ok(Some(FirstMarker::Wav(at))); + } + if let (Some(&b0), Some(&b1)) = (b.get(at), b.get(at + 1)) + && b0 == 0xFF + && b1 & 0xE0 == 0xE0 + { + // A candidate after the cap cannot be scored. Stop rather than let a container-like + // byte string later in an unscored audio payload become preflight-authoritative; the + // probe will select the reader, and `open` refuses an unchecked size-checked one. + if budget.exhausted() { + return Ok(None); + } + if audio_sync_at(b, at, &mut audio_probe, budget)? { + return Ok(Some(FirstMarker::Audio)); + } + } + if rest.get(4..8) == Some(b"ftyp".as_slice()) { + return Ok(Some(FirstMarker::Mp4(at))); + } + at += 1; + } + Ok(None) +} + +/// Run the early size checks on the container the probe can select. `start` is the first byte +/// after leading ID3v2 tags. A zero-frame FLAC is returned as an empty stream because Symphonia +/// 0.6's reader requires a first audio frame to resynchronize to. +fn precheck(b: &[u8], start: usize, detected: FileFormat) -> Result { + let any_marker = |b: &[u8], start: usize| -> Result> { + Ok(match first_marker(b, start)? { + // Stop at the first marker of any kind, so container magic incidental to an audio + // payload is not mistaken for the stream's container. + Some(FirstMarker::Audio) | None => None, + Some(marker) => Some((marker, marker_name(marker))), + }) + }; + let found = match container_markers(detected) { + // Follow the detection: check the first marker of the family it names, so a misleading + // marker cannot let a differently sized container reach the probe unchecked. A detection + // is only a hint, though: when the stream holds none of its markers (a FLAC behind a + // `.mp3` name), the probe can still select another container, which is checked here. + Some(markers) => match find_container_marker(b, start, markers) { + Some((at, marker)) => Some(marker_kind(at, marker)), + None => any_marker(b, start)?, + }, + None => any_marker(b, start)?, + }; + let Some((marker, expected)) = found else { return Ok(Precheck::Unchecked) }; + match marker { + FirstMarker::Flac(at) => { + crate::flac::check_metadata(b, at)?; + if let Some(info) = crate::flac::empty_stream_info(b, at) { + return Ok(Precheck::Empty(info)); + } + Ok(Precheck::Checked(expected)) + } + FirstMarker::Caf(at) => { + crate::caf::check(b, at)?; + Ok(Precheck::Checked(expected)) + } + FirstMarker::Mp4(at) => { + crate::mp4::check(b, at)?; + Ok(Precheck::Checked(expected)) + } + FirstMarker::Wav(at) => { + crate::wav::check_info(b, at)?; + Ok(Precheck::Checked(expected)) + } + FirstMarker::Audio => Ok(Precheck::Unchecked), + } +} + +/// Skip leading ID3v2 tags, run the container size preflight, then probe with symphonia; a zero-frame FLAC found by +/// the preflight comes back as [`Opened::Empty`] without probing. A size-checked container is refused when probing +/// selects a different reader or one that was never checked, so incidental payload magic cannot bypass those limits. +fn open(bytes: &[u8], detected: FileFormat) -> Result { + let start = skip_id3v2(bytes); + if start > 0 && start >= bytes.len() { + return Err(AudioError::Malformed("ID3v2 tag extends past the end of the file".into())); + } + let checked = precheck(bytes, start, detected)?; + if let Precheck::Empty(info) = checked { + return Ok(Opened::Empty(info)); + } + let mut mss = MediaSourceStream::new(Box::new(Cursor::new(bytes.to_vec())), MediaSourceStreamOptions::default()); + // Seek (rather than pre-positioning the cursor) so the stream's absolute position is `start` + // too: container offsets such as MP4's `stco` are absolute file offsets. + mss.seek(SeekFrom::Start(u64::try_from(start).unwrap_or(u64::MAX)))?; + let reader: Box = + symphonia::default::get_probe().probe(&Hint::new(), mss, FormatOptions::default(), MetadataOptions::default()).map_err(map_err)?; + let probed = reader.format_info().short_name; + match checked { + // Only containers whose readers allocate from sizes in the file need the check. A stream + // the probe reads with another reader (MP3, Ogg, AIFF, ...) has no such size fields, so a + // marker found in its audio data must not reject it. + Precheck::Checked(expected) if SIZE_CHECKED_PROBE_FORMATS.contains(&probed) && probed != expected => { + return Err(AudioError::Malformed(format!("the probed container ({probed}) is not the one whose size checks ran"))); + } + Precheck::Unchecked if SIZE_CHECKED_PROBE_FORMATS.contains(&probed) => { + return Err(AudioError::Malformed(format!("the probed container ({probed}) was not size-checked"))); + } + _ => {} + } + Ok(Opened::Reader(reader)) +} + +fn format_from_codec(codec: AudioCodecId, detected: FileFormat) -> FileFormat { + if detected != FileFormat::Other { + return detected; + } + match codec { + c if c == CODEC_ID_FLAC => FileFormat::Flac, + c if c == CODEC_ID_MP3 || c == CODEC_ID_MP2 || c == CODEC_ID_MP1 => FileFormat::Mp3, + c if c == CODEC_ID_AAC => FileFormat::Aac, + c if c == CODEC_ID_ALAC => FileFormat::Alac, + c if c == CODEC_ID_VORBIS => FileFormat::Ogg, + _ => FileFormat::Other, + } +} + +/// The stored sample format, using the codec and its declared bit depth. Lossy and unusual +/// codecs decode to `Compressed`. +fn sample_format_of(p: &AudioCodecParameters) -> SampleFormat { + let c = p.codec; + if c == CODEC_ID_PCM_F32LE || c == CODEC_ID_PCM_F32BE { + return SampleFormat::Float32; + } + if c == CODEC_ID_PCM_F64LE || c == CODEC_ID_PCM_F64BE { + return SampleFormat::Float64; + } + let lossy = [ + CODEC_ID_MP1, + CODEC_ID_MP2, + CODEC_ID_MP3, + CODEC_ID_AAC, + CODEC_ID_VORBIS, + CODEC_ID_OPUS, + CODEC_ID_ADPCM_MS, + CODEC_ID_ADPCM_IMA_WAV, + CODEC_ID_ADPCM_IMA_QT, + CODEC_ID_ADPCM_G722, + CODEC_ID_ADPCM_G726, + CODEC_ID_ADPCM_G726LE, + CODEC_ID_PCM_ALAW, + CODEC_ID_PCM_MULAW, + ]; + let lossless = !lossy.contains(&c); + match (lossless, p.bits_per_sample) { + (true, Some(1..=8)) => SampleFormat::Int8, + (true, Some(9..=16)) => SampleFormat::Int16, + (true, Some(17..=24)) => SampleFormat::Int24, + (true, Some(25..=32)) => SampleFormat::Int32, + _ => SampleFormat::Compressed, + } +} + +/// The audio parameters of `track`, if it carries any. +fn audio_params(track: &Track) -> Option<&AudioCodecParameters> { + track.codec_params.as_ref()?.audio() +} + +/// The first track with a decodable audio codec, with the stream information the UI shows. +fn info_of(reader: &dyn FormatReader, detected: FileFormat) -> Result<(u32, AudioInfo)> { + let track = reader + .tracks() + .iter() + .find(|t| audio_params(t).is_some_and(|p| p.codec != CODEC_ID_NULL_AUDIO)) + .ok_or_else(|| AudioError::Unsupported("no decodable audio track".into()))?; + let p = audio_params(track).ok_or_else(|| AudioError::Unsupported("no decodable audio track".into()))?; + let sample_rate = p.sample_rate.filter(|&r| r > 0).ok_or_else(|| AudioError::Malformed("unknown sample rate".into()))?; + let channels = p.channels.as_ref().map(|c| c.count()).unwrap_or(0); + let channels = u16::try_from(channels).map_err(|_| AudioError::Unsupported("too many channels".into()))?; + Ok(( + track.id, + AudioInfo { + format: format_from_codec(p.codec, detected), + sample_format: sample_format_of(p), + sample_rate, + channels, + frames: track.num_frames.unwrap_or(0), + bwf: None, + }, + )) +} + +/// A demux error ends the decode; it is only an error when nothing was decoded yet. +fn demux_failure(frames: u64, e: SymError) -> Result<()> { + if frames == 0 { + return Err(map_err(e)); + } + log::warn!("stopping decode after demux error: {e}"); + Ok(()) +} + +/// Note a corrupt packet (the `errors`-th); a stream that never decodes is malformed. +fn corrupt_packet(errors: usize, frames: u64, msg: &str) -> Result<()> { + log::debug!("skipping corrupt packet: {msg}"); + if errors > 1000 && frames == 0 { + return Err(AudioError::Malformed(format!("no decodable packets: {msg}"))); + } + Ok(()) +} + +/// The decoded stream is authoritative for the channel count: record it and allocate the planes. +fn init_planar(info: &mut AudioInfo, ch: usize) -> Result>> { + info.channels = u16::try_from(ch).map_err(|_| AudioError::Unsupported("too many channels".into()))?; + Ok((0..ch).map(|_| Vec::new()).collect()) +} + +/// Append one interleaved block to the planes, replacing non-finite samples with silence. +fn append_interleaved(planar: &mut [Vec], interleaved: &[f32], ch: usize) { + for frame in interleaved.chunks_exact(ch) { + for (dst, &s) in planar.iter_mut().zip(frame) { + dst.push(if s.is_finite() { s } else { 0.0 }); + } + } +} + +/// Decode the whole first audio track. When `count_only`, samples are counted but not stored. +fn run(bytes: &[u8], detected: FileFormat, count_only: bool) -> Result<(AudioInfo, AudioBuffer)> { + let mut reader = match open(bytes, detected)? { + Opened::Reader(reader) => reader, + Opened::Empty(info) => { + let buf = AudioBuffer::new(info.sample_rate, usize::from(info.channels), 0); + return Ok((info, buf)); + } + }; + let (track_id, mut info) = info_of(reader.as_ref(), detected)?; + let track = reader.tracks().iter().find(|t| t.id == track_id).ok_or_else(|| AudioError::Malformed("track vanished".into()))?; + if let Some(n) = track.num_frames { + let total = n.saturating_mul(u64::from(info.channels.max(1))); + if !count_only && total > MAX_DECODE_SAMPLES { + return Err(AudioError::TooLarge(format!("{total} samples exceeds the decode limit"))); + } + } + let params = audio_params(track).cloned().ok_or_else(|| AudioError::Malformed("track vanished".into()))?; + let alac = params.extra_data.as_deref().filter(|_| params.codec == CODEC_ID_ALAC).and_then(|d| be_u32(d, 0)); + if params.max_frames_per_packet.into_iter().chain(alac.map(u64::from)).any(|n| n > MAX_PACKET_FRAMES) { + return Err(AudioError::Malformed("the file declares packets far longer than any codec uses".into())); + } + let mut decoder = symphonia::default::get_codecs().make_audio_decoder(¶ms, &decoder_options()).map_err(map_err)?; + + let mut planar: Vec> = Vec::new(); + let mut interleaved: Vec = Vec::new(); + let mut frames: u64 = 0; + let mut errors = 0usize; + loop { + let packet = match reader.next_packet() { + Ok(Some(packet)) => packet, + Ok(None) => break, + Err(SymError::IoError(e)) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(SymError::ResetRequired) => break, + Err(e) => { + demux_failure(frames, e)?; + break; + } + }; + if packet.track_id != track_id { + continue; + } + let decoded = match decoder.decode(&packet) { + Ok(d) => d, + Err(SymError::DecodeError(m)) => { + errors += 1; + corrupt_packet(errors, frames, m)?; + continue; + } + Err(SymError::IoError(_)) => break, + Err(e) => return Err(map_err(e)), + }; + // The interleaved copy below is laid out by planes: use their count so a decoder whose + // channel spec disagrees with its planes cannot trip the copy's length assertion. + let ch = decoded.num_planes(); + if ch == 0 { + continue; + } + if planar.is_empty() { + planar = init_planar(&mut info, ch)?; + } + let n = decoded.frames() as u64; + frames = frames.saturating_add(n); + if frames.saturating_mul(ch as u64) > MAX_DECODE_SAMPLES { + return Err(AudioError::TooLarge("decoded audio exceeds the 4 GiB limit".into())); + } + if count_only { + continue; + } + let needed = usize::try_from(n) + .ok() + .and_then(|n| n.checked_mul(ch)) + .ok_or_else(|| AudioError::TooLarge("decoded audio exceeds the 4 GiB limit".into()))?; + if interleaved.len() < needed { + interleaved.try_reserve(needed - interleaved.len()).map_err(|_| AudioError::TooLarge("cannot allocate decoded audio".into()))?; + interleaved.resize(needed, 0.0); + } + decoded.copy_to_slice_interleaved(&mut interleaved[..needed]); + append_interleaved(&mut planar, &interleaved[..needed], ch); + } + if info.channels == 0 { + return Err(AudioError::Malformed("stream contains no audio".into())); + } + if planar.is_empty() { + planar = (0..usize::from(info.channels)).map(|_| Vec::new()).collect(); + } + info.frames = if count_only { frames } else { planar.iter().map(Vec::len).min().unwrap_or(0) as u64 }; + let sample_rate = info.sample_rate; + Ok((info, AudioBuffer { sample_rate, channels: planar })) +} + +/// Frames symphonia's AAC decoder yields for every packet; it supports no other frame length. +const AAC_FRAMES_PER_PACKET: u64 = 1024; + +/// Whether `track` of `reader` is AAC in MP4. The MP4 reader reports the stts total as the length, +/// but the decoder yields a whole frame per packet (sample durations are not applied), so a last +/// sample whose duration differs from a frame makes the two disagree. +fn is_mp4_aac(reader: &dyn FormatReader, track_id: u32) -> bool { + reader.format_info().short_name == "isomp4" + && reader.tracks().iter().find(|t| t.id == track_id).and_then(audio_params).is_some_and(|p| p.codec == CODEC_ID_AAC) +} + +/// The decoded length of AAC in MP4: count the track's packets without decoding them. +fn mp4_aac_frames(reader: &mut dyn FormatReader, track_id: u32) -> Result { + let mut frames: u64 = 0; + loop { + match reader.next_packet() { + Ok(Some(packet)) if packet.track_id == track_id => frames = frames.saturating_add(AAC_FRAMES_PER_PACKET), + Ok(Some(_)) => {} + Ok(None) => break, + Err(SymError::IoError(e)) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(SymError::ResetRequired) => break, + Err(e) => { + demux_failure(frames, e)?; + break; + } + } + } + Ok(frames) +} + +pub(crate) fn probe(bytes: &[u8], detected: FileFormat) -> Result { + guarded(|| { + let mut reader = match open(bytes, detected)? { + Opened::Reader(reader) => reader, + Opened::Empty(info) => return Ok(info), + }; + let (track_id, mut info) = info_of(reader.as_ref(), detected)?; + if info.frames > 0 && info.channels > 0 && is_mp4_aac(reader.as_ref(), track_id) { + info.frames = mp4_aac_frames(reader.as_mut(), track_id)?; + } + if info.frames > 0 && info.channels > 0 { + return Ok(info); + } + // Length not in the header (e.g. MP3 without a Xing frame): count by decoding. + run(bytes, detected, true).map(|(i, _)| i) + }) +} + +pub(crate) fn decode(bytes: &[u8], detected: FileFormat) -> Result<(AudioInfo, AudioBuffer)> { + guarded(|| run(bytes, detected, false)) +} + +#[cfg(test)] +mod tests { + use super::*; + use symphonia::core::codecs::audio::well_known::{ + CODEC_ID_AAC, CODEC_ID_ADPCM_MS, CODEC_ID_ALAC, CODEC_ID_FLAC, CODEC_ID_MP3, CODEC_ID_PCM_F32LE, CODEC_ID_PCM_F64LE, CODEC_ID_PCM_S16LE, + CODEC_ID_VORBIS, + }; + + /// Build codec parameters for a test, optionally declaring a bit depth. + fn params(codec: AudioCodecId, bits_per_sample: Option) -> AudioCodecParameters { + let mut params = AudioCodecParameters::new(); + params.for_codec(codec); + if let Some(bits) = bits_per_sample { + params.with_bits_per_sample(bits); + } + params + } + + /// Codec ids map to their reported formats, while a format detected from the container wins over the codec. + #[test] + fn codecs_map_to_the_reported_formats() { + assert_eq!(format_from_codec(CODEC_ID_FLAC, FileFormat::Other), FileFormat::Flac); + assert_eq!(format_from_codec(CODEC_ID_MP3, FileFormat::Other), FileFormat::Mp3); + assert_eq!(format_from_codec(CODEC_ID_AAC, FileFormat::Other), FileFormat::Aac); + assert_eq!(format_from_codec(CODEC_ID_ALAC, FileFormat::Other), FileFormat::Alac); + assert_eq!(format_from_codec(CODEC_ID_VORBIS, FileFormat::Other), FileFormat::Ogg); + // The container detected from the bytes wins over the codec. + assert_eq!(format_from_codec(CODEC_ID_ALAC, FileFormat::Aac), FileFormat::Aac); + } + + /// PCM and FLAC keep their declared bit depth; lossy codecs report `Compressed` regardless of their decoded + /// width. + #[test] + fn sample_formats_keep_their_depth_and_lossy_marker() { + assert_eq!(sample_format_of(¶ms(CODEC_ID_PCM_S16LE, Some(16))), SampleFormat::Int16); + assert_eq!(sample_format_of(¶ms(CODEC_ID_FLAC, Some(24))), SampleFormat::Int24); + assert_eq!(sample_format_of(¶ms(CODEC_ID_FLAC, Some(8))), SampleFormat::Int8); + assert_eq!(sample_format_of(¶ms(CODEC_ID_FLAC, Some(32))), SampleFormat::Int32); + assert_eq!(sample_format_of(¶ms(CODEC_ID_PCM_F32LE, Some(32))), SampleFormat::Float32); + assert_eq!(sample_format_of(¶ms(CODEC_ID_PCM_F64LE, Some(64))), SampleFormat::Float64); + // Lossy codecs report no fixed bit depth, whatever their decoded sample format is. + assert_eq!(sample_format_of(¶ms(CODEC_ID_AAC, Some(16))), SampleFormat::Compressed); + assert_eq!(sample_format_of(¶ms(CODEC_ID_ADPCM_MS, Some(4))), SampleFormat::Compressed); + assert_eq!(sample_format_of(¶ms(CODEC_ID_ALAC, None)), SampleFormat::Compressed); + } + + /// Decoder options keep gapless trimming enabled so the encoder delay and padding reported by the demuxer are + /// removed. + #[test] + fn gapless_decoding_stays_enabled() { + assert!(decoder_options().gapless); + } + + /// A complete MPEG frame header is a candidate; loose or unregistered sync pairs are rejected before scoring, + /// while a registered but truncated header is rejected by the reader's score after consuming a budget slot. + #[test] + fn only_the_readers_own_audio_markers_are_candidates() { + /// Whether the preflight would score `b` as an audio candidate at offset 0. + fn candidate(b: &[u8]) -> bool { + let mut stream = None; + let mut budget = ScoreBudget::new(); + matches!(audio_sync_at(b, 0, &mut stream, &mut budget), Ok(true)) + } + let mut mp3_frame = vec![0u8; 417]; + mp3_frame[..4].copy_from_slice(&[0xFF, 0xFB, 0x90, 0x00]); // MPEG-1 Layer III, 128 kbps, 44.1 kHz + assert!(candidate(&[&mp3_frame[..], &mp3_frame].concat())); + for false_sync in [vec![0xFF, 0xE0, 0x00], vec![0xFF, 0xF0, 0x00], vec![0xFF, 0xFB, 0xF0, 0x00]] { + assert!(!candidate(&false_sync), "{false_sync:?} is not a probe candidate"); + } + } + + /// A prefix made of false MPEG sync markers must not ask the readers to score every byte + /// position: the per-scan budget caps the expensive work. A cheap container marker reached + /// before the next audio candidate is still found, but once the cap is spent an audio + /// candidate stops the scan so later payload magic cannot be preflight-authoritative. + #[test] + fn a_prefix_crowded_with_sync_markers_scores_only_the_budget() { + let mut crowded = vec![0u8; 1 << 16]; + for pair in crowded.as_chunks_mut::<2>().0 { + pair.copy_from_slice(&[0xFF, 0xE2]); + } + let mut budget = ScoreBudget::new(); + assert!(matches!(first_marker_with_budget(&crowded, 0, &mut budget), Ok(None))); + assert_eq!(budget.used, MAX_AUDIO_SCORE_ATTEMPTS, "no candidate after the cap is scored"); + + // At the cap boundary, with no later audio candidate, the cheap marker is still found. + let mut at_cap: Vec = (0..MAX_AUDIO_SCORE_ATTEMPTS).flat_map(|_| [0xFFu8, 0xE2]).collect(); + at_cap.extend_from_slice(b"fLaC"); + let mut budget = ScoreBudget::new(); + assert!( + matches!(first_marker_with_budget(&at_cap, 0, &mut budget), Ok(Some(FirstMarker::Flac(_)))), + "a container marker reached before the next audio candidate is still found" + ); + assert_eq!(budget.used, MAX_AUDIO_SCORE_ATTEMPTS); + + // One more false candidate between the cap and the magic makes the scan stop unchecked: + // the same magic could instead sit in a valid audio payload. + let mut after_candidate = at_cap[..2 * MAX_AUDIO_SCORE_ATTEMPTS].to_vec(); + after_candidate.extend_from_slice(&[0xFF, 0xE2]); + after_candidate.extend_from_slice(b"fLaC"); + let mut budget = ScoreBudget::new(); + assert!(matches!(first_marker_with_budget(&after_candidate, 0, &mut budget), Ok(None)), "an audio candidate after the cap stops the scan"); + assert_eq!(budget.used, MAX_AUDIO_SCORE_ATTEMPTS); + } + + /// ID3v2 sizes are read in their 7-bit syncsafe form; consecutive tags are skipped by their declared sizes, and + /// the v2.3 reserved footer bit does not extend a tag. + #[test] + fn leading_id3_tags_are_skipped_by_their_declared_size() { + let tag = [&b"ID3\x04\0\0\0\0\0\x05"[..], b"hello"].concat(); + assert_eq!(skip_id3v2(&[&tag[..], &tag[..]].concat()), 30); + assert_eq!(skip_id3v2(b"fLaC"), 0); + // The 10-byte footer flag exists only in v2.4; the same bit in v2.3 is reserved. + assert_eq!(crate::id3v2_len(b"ID3\x03\0\x10\0\0\0\0"), Some(10)); + assert_eq!(crate::id3v2_len(b"ID3\x04\0\x10\0\0\0\0"), Some(20)); + } +} diff --git a/crates/audio-io/src/wav.rs b/crates/audio-io/src/wav.rs new file mode 100644 index 0000000000..9edbb3d84f --- /dev/null +++ b/crates/audio-io/src/wav.rs @@ -0,0 +1,574 @@ +//! Native WAV / BWF / RF64 reader and writer (RIFF chunk layout per the Microsoft multimedia +//! specification, `bext` per EBU Tech 3285, `ds64` per EBU Tech 3306). + +use crate::pcm::{PcmKind, Quantizer, deinterleave, le_u16, le_u32, le_u64, sample_format, tag, validate_buffer, validate_layout, write_interleaved}; +use crate::{AudioBuffer, AudioError, AudioInfo, BitDepth, BwfInfo, FileFormat, Result, SampleFormat}; +use std::io::{Cursor, Seek, SeekFrom, Write}; + +const WAVE_FORMAT_PCM: u16 = 1; +const WAVE_FORMAT_IEEE_FLOAT: u16 = 3; +const WAVE_FORMAT_EXTENSIBLE: u16 = 0xFFFE; +/// Tail of the KSDATAFORMAT_SUBTYPE_* GUIDs (bytes 2..16); bytes 0..2 hold the format tag. +const GUID_TAIL: [u8; 14] = [0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x80, 0x00, 0x00, 0xAA, 0x00, 0x38, 0x9B, 0x71]; +/// Tail of the Ambisonic subtype GUIDs {0000000x-0721-11D3-8644-C8C1CA000000}: PCM (x = 1) and float (x = 3) as well. +const AMBISONIC_TAIL: [u8; 14] = [0x00, 0x00, 0x21, 0x07, 0xD3, 0x11, 0x86, 0x44, 0xC8, 0xC1, 0xCA, 0x00, 0x00, 0x00]; +const BEXT_MIN_LEN: usize = 602; + +/// Parsed WAV layout (no sample data decoded yet). +pub(crate) struct WavLayout { + pub info: AudioInfo, + pub kind: PcmKind, + pub data_start: usize, + pub data_len: usize, +} + +/// Outcome of parsing a WAV header. +pub(crate) enum WavParse { + /// A PCM / float layout we decode natively. + Native(WavLayout), + /// A valid WAV whose codec we hand to symphonia (ADPCM, A-law, ...). + Foreign { bwf: Option }, +} + +struct Fmt { + tag: u16, + channels: u16, + sample_rate: u32, + block_align: u16, + bits: u16, + valid_bits: u16, +} + +fn parse_fmt(c: &[u8]) -> Result { + let short = || AudioError::Malformed("fmt chunk too short".into()); + let mut fmt = Fmt { + tag: le_u16(c, 0).ok_or_else(short)?, + channels: le_u16(c, 2).ok_or_else(short)?, + sample_rate: le_u32(c, 4).ok_or_else(short)?, + block_align: le_u16(c, 12).ok_or_else(short)?, + bits: le_u16(c, 14).unwrap_or(0), + valid_bits: 0, + }; + fmt.valid_bits = fmt.bits; + if fmt.tag == WAVE_FORMAT_EXTENSIBLE { + // cbSize(2) validBits(2) channelMask(4) subFormat GUID(16) + let (Some(cb_size), Some(valid), Some(sub)) = (le_u16(c, 16), le_u16(c, 18), le_u16(c, 24)) else { + return Err(AudioError::Malformed("WAVE_FORMAT_EXTENSIBLE fmt chunk too short".into())); + }; + let Some(tail) = c.get(26..40) else { + return Err(AudioError::Malformed("WAVE_FORMAT_EXTENSIBLE fmt chunk too short".into())); + }; + if cb_size < 22 { + return Err(AudioError::Malformed("WAVE_FORMAT_EXTENSIBLE cbSize is below 22".into())); + } + if valid > fmt.bits { + return Err(AudioError::Malformed("WAVE_FORMAT_EXTENSIBLE valid bits exceed the container size".into())); + } + if valid != 0 { + fmt.valid_bits = valid; + } + // A subtype outside the KSDATAFORMAT_SUBTYPE_* family is not PCM/float, whatever its first two bytes say: + // keep the extensible tag so the file goes to the foreign decoder. + if tail == GUID_TAIL || tail == AMBISONIC_TAIL { + fmt.tag = sub; + } + } + Ok(fmt) +} + +/// Read a NUL-padded ASCII field. +fn fixed_str(b: &[u8]) -> String { + let end = b.iter().position(|&c| c == 0).unwrap_or(b.len()); + String::from_utf8_lossy(b.get(..end).unwrap_or_default()).trim_end().to_string() +} + +fn parse_bext(c: &[u8]) -> Option { + if c.len() < 346 { + return None; + } + let lo = u64::from(le_u32(c, 338)?); + let hi = u64::from(le_u32(c, 342)?); + Some(BwfInfo { + description: fixed_str(c.get(0..256)?), + originator: fixed_str(c.get(256..288)?), + originator_reference: fixed_str(c.get(288..320)?), + origination_date: fixed_str(c.get(320..330)?), + origination_time: fixed_str(c.get(330..338)?), + time_reference: (hi << 32) | lo, + }) +} + +/// Refuse the WAVE file at `at`, which symphonia reads (A-law, ADPCM, ...), if an INFO entry +/// claims more bytes than the file holds: symphonia allocates each entry's length. Chunks are +/// walked where symphonia reads them: after the bytes a chunk's parser used, not its length. +pub(crate) fn check_info(b: &[u8], at: usize) -> Result<()> { + if tag(b, at + 8) != Some(*b"WAVE") { + return Ok(()); + } + let mut pos = at + 12; + while let Some(id) = tag(b, pos) { + let Some(len) = le_u32(b, pos + 4).map(|n| n as usize) else { break }; + let (body, pad) = (pos + 8, len & 1); + let used = match &id { + b"data" => break, + b"fmt " => fmt_used(b, body, len), + b"LIST" if len < 4 => break, + b"LIST" => { + let entries = if tag(b, body) == Some(*b"INFO") { info_entries(b, body + 4, len - 4)? } else { 0 }; + // The list pads itself, then the chunk is padded again. + 4 + entries.max(len - 4) + pad + } + _ => len, + }; + pos = body.saturating_add(used).saturating_add(pad); + } + Ok(()) +} + +/// The bytes symphonia reads of a `fmt ` chunk: some formats' parsers go by their own extra size. +fn fmt_used(b: &[u8], body: usize, len: usize) -> usize { + match le_u16(b, body) { + Some(2 | 6 | 7 | 0x11) => le_u16(b, body + 16).map_or(len, |extra| 18 + usize::from(extra)), + Some(0xfffe) => 40, + _ => len, + } +} + +/// Check the INFO entries in the `len` bytes from `start`; returns the bytes symphonia reads of +/// them, with the pad it reads after an odd entry. +fn info_entries(b: &[u8], start: usize, len: usize) -> Result { + let mut used = 0; + while used + 8 <= len { + let Some(n) = le_u32(b, start + used + 4).map(|n| n as usize) else { break }; + if n > b.len().saturating_sub(start + used + 8) { + return Err(AudioError::Malformed("a WAV INFO entry is larger than the file".into())); + } + used += 8 + n; + used += used & 1; + } + Ok(used) +} + +/// True when the bytes start like a RIFF/RF64/BW64 WAVE file. +pub(crate) fn is_wav(b: &[u8]) -> bool { + matches!(tag(b, 0), Some(t) if &t == b"RIFF" || &t == b"RF64" || &t == b"BW64") && tag(b, 8).is_some_and(|t| &t == b"WAVE") +} + +/// True when everything from `from` to the end of the file is whole RIFF chunks (four printable +/// ASCII id bytes, a size that fits, the pad byte after an odd size; the last chunk's pad may be +/// missing): the metadata of a finished file, not samples. +fn only_chunks_follow(b: &[u8], from: usize) -> bool { + let mut pos = from; + while pos < b.len() { + let (Some(id), Some(size)) = (tag(b, pos), le_u32(b, pos + 4)) else { return false }; + if !id.iter().all(|c| (0x20..0x7f).contains(c)) { + return false; + } + let size = size as usize; + let Some(end) = pos.checked_add(8).and_then(|body| body.checked_add(size)) else { return false }; + if end > b.len() { + return false; + } + let next = end + (size & 1); + if end == b.len() || next == b.len() { + return true; + } + pos = next; + } + false +} + +/// A `data` chunk header as found in the file, for [`data_chunk_len`]. +struct DataChunk { + body: usize, + size32: u32, + is_rf64: bool, + remaining: usize, + declared: usize, +} + +/// Bytes of audio a `data` chunk holds: leniently, whatever is present when the size is wrong or +/// the recording was never finalised. +fn data_chunk_len(b: &[u8], c: DataChunk) -> usize { + let finished = u64::from(le_u32(b, 4).unwrap_or(0)) + 8 == b.len() as u64; + let empty = c.size32 == 0 && !c.is_rf64 && finished && only_chunks_follow(b, c.body); + if c.size32 == 0 && !c.is_rf64 && !empty { c.remaining } else { c.declared.min(c.remaining) } +} + +/// The sample layout of a `fmt` chunk, `None` for a format we do not read natively. +fn pcm_layout(fmt: &Fmt) -> Result> { + Ok(Some(match (fmt.tag, fmt.bits) { + (WAVE_FORMAT_PCM, bits @ 1..=32) => { + let bytes = usize::from(bits.div_ceil(8)); + let sf = match fmt.valid_bits.min(bits) { + 0..=8 => SampleFormat::Int8, + 9..=16 => SampleFormat::Int16, + 17..=24 => SampleFormat::Int24, + _ => SampleFormat::Int32, + }; + (PcmKind::Int { bytes, big_endian: false, unsigned: bytes == 1 }, sf) + } + (WAVE_FORMAT_IEEE_FLOAT, 32) => (PcmKind::Float { bytes: 4, big_endian: false }, SampleFormat::Float32), + (WAVE_FORMAT_IEEE_FLOAT, 64) => (PcmKind::Float { bytes: 8, big_endian: false }, SampleFormat::Float64), + (WAVE_FORMAT_PCM | WAVE_FORMAT_IEEE_FLOAT, bits) => { + return Err(AudioError::Unsupported(format!("WAV format {} with {bits} bits per sample", fmt.tag))); + } + _ => return Ok(None), + })) +} + +pub(crate) fn parse(b: &[u8]) -> Result { + if !is_wav(b) { + return Err(AudioError::Malformed("not a RIFF/RF64 WAVE file".into())); + } + let is_rf64 = tag(b, 0).is_some_and(|t| &t != b"RIFF"); + let mut ds64_data: Option = None; + let mut fmt: Option = None; + let mut bwf: Option = None; + let mut data: Option<(usize, usize)> = None; + + let mut pos: usize = 12; + // Bound the number of chunks we look at so a file of tiny chunks can't spin forever. + let mut chunks = 0usize; + while let (Some(id), Some(size32)) = (tag(b, pos), le_u32(b, pos + 4)) { + chunks += 1; + if chunks > 100_000 { + break; + } + let body = pos.checked_add(8).ok_or_else(|| AudioError::Malformed("chunk offset overflow".into()))?; + let remaining = b.len().saturating_sub(body); + let mut size = u64::from(size32); + if &id == b"data" && is_rf64 && size32 == u32::MAX { + size = ds64_data.unwrap_or(remaining as u64); + } + let declared = usize::try_from(size).unwrap_or(usize::MAX); + if &id == b"data" { + // Lenient: a truncated recording (or a streaming writer's 0 / 0xFFFFFFFF size) still + // yields whatever audio is actually present. + // A size of 0 in a file whose RIFF size is right and that has only chunks after it is an + // empty data chunk followed by metadata (LIST, bext, …); otherwise it is a recording + // that was never finalised, and what follows is audio. + let len = data_chunk_len(b, DataChunk { body, size32, is_rf64, remaining, declared }); + if data.is_none() { + data = Some((body, len)); + } + if declared > remaining { + break; + } + } else { + if declared > remaining { + // A truncated non-audio chunk at the end of the file: stop scanning. + break; + } + let c = b.get(body..body + declared).unwrap_or_default(); + match &id { + b"fmt " => fmt = Some(parse_fmt(c)?), + b"bext" => bwf = parse_bext(c), + b"ds64" => ds64_data = le_u64(c, 8), + _ => {} + } + } + let padded = declared.saturating_add(declared & 1); + pos = match body.checked_add(padded) { + Some(p) => p, + None => break, + }; + } + + let fmt = fmt.ok_or_else(|| AudioError::Malformed("WAV has no fmt chunk".into()))?; + if fmt.channels == 0 { + return Err(AudioError::Malformed("WAV declares zero channels".into())); + } + if fmt.sample_rate == 0 { + return Err(AudioError::Malformed("WAV declares a zero sample rate".into())); + } + let channels = usize::from(fmt.channels); + let Some((kind, sample_format)) = pcm_layout(&fmt)? else { return Ok(WavParse::Foreign { bwf }) }; + let block = kind.bytes().checked_mul(channels).ok_or_else(|| AudioError::Malformed("block size overflow".into()))?; + if usize::from(fmt.block_align) != block { + log::debug!("WAV block_align {} disagrees with computed {block}; using computed", fmt.block_align); + } + let (data_start, data_len) = data.ok_or_else(|| AudioError::Malformed("WAV has no data chunk".into()))?; + let frames = (data_len / block) as u64; + Ok(WavParse::Native(WavLayout { + info: AudioInfo { format: FileFormat::Wav, sample_format, sample_rate: fmt.sample_rate, channels: fmt.channels, frames, bwf }, + kind, + data_start, + data_len, + })) +} + +pub(crate) fn decode(b: &[u8], layout: &WavLayout) -> Result { + let end = layout.data_start.saturating_add(layout.data_len).min(b.len()); + let data = b.get(layout.data_start..end).unwrap_or_default(); + let channels = deinterleave(data, layout.kind, usize::from(layout.info.channels), layout.info.frames)?; + Ok(AudioBuffer { sample_rate: layout.info.sample_rate, channels }) +} + +/// Default speaker mask for a channel count (callers that know the layout pass their own): +/// mono, stereo, LCR, quad, 5.0, 5.1, 6.1, 7.1 (sides + rears), 7.1.4. +fn channel_mask(channels: usize) -> u32 { + match channels { + 1 => 0x4, + 2 => 0x3, + 3 => 0x7, + 4 => 0x33, + 5 => 0x37, + 6 => 0x3F, + 7 => 0x13F, + 8 => 0x63F, + // L R C LFE Lrs Rrs Lss Rss Ltf Rtf Ltr Rtr (7.1.4). + 12 => 0x2_D63F, + n if n <= 18 => (1u32 << n) - 1, + _ => 0, + } +} + +/// The `dwChannelMask` of a WAVE_FORMAT_EXTENSIBLE file (None for plain PCM/float headers). +pub(crate) fn read_channel_mask(b: &[u8]) -> Option { + if !is_wav(b) { + return None; + } + let mut pos = 12usize; + for _ in 0..10_000 { + let (id, size) = (tag(b, pos)?, usize::try_from(le_u32(b, pos + 4)?).ok()?); + if &id == b"fmt " { + return (le_u16(b, pos + 8)? == WAVE_FORMAT_EXTENSIBLE).then(|| le_u32(b, pos + 8 + 20)).flatten(); + } + pos = pos.checked_add(8)?.checked_add(size)?.checked_add(size & 1)?; + } + None +} + +fn put_fixed_str(out: &mut Vec, s: &str, len: usize) { + let bytes = s.as_bytes(); + let mut n = bytes.len().min(len); + while !s.is_char_boundary(n) { + n -= 1; + } + out.extend_from_slice(bytes.get(..n).unwrap_or_default()); + out.resize(out.len() + (len - n), 0); +} + +fn bext_chunk(bwf: &BwfInfo) -> Vec { + let mut c = Vec::with_capacity(BEXT_MIN_LEN); + put_fixed_str(&mut c, &bwf.description, 256); + put_fixed_str(&mut c, &bwf.originator, 32); + put_fixed_str(&mut c, &bwf.originator_reference, 32); + put_fixed_str(&mut c, &bwf.origination_date, 10); + put_fixed_str(&mut c, &bwf.origination_time, 8); + c.extend_from_slice(&((bwf.time_reference & 0xFFFF_FFFF) as u32).to_le_bytes()); + c.extend_from_slice(&((bwf.time_reference >> 32) as u32).to_le_bytes()); + c.extend_from_slice(&1u16.to_le_bytes()); // version 1 + c.resize(BEXT_MIN_LEN, 0); // UMID, loudness, reserved; no coding history + c +} + +fn push_chunk(out: &mut Vec, id: &[u8; 4], body: &[u8]) { + out.extend_from_slice(id); + out.extend_from_slice(&u32::try_from(body.len()).unwrap_or(u32::MAX).to_le_bytes()); + out.extend_from_slice(body); + if body.len() % 2 == 1 { + out.push(0); + } +} + +/// A WAV written block by block: a placeholder header first, then the real one (the same length, +/// RF64 above 4 GiB, or always when `force_rf64`) written over it by [`Writer::finish`]. +pub(crate) struct Writer { + out: W, + start: u64, + channels: usize, + sample_rate: u32, + format: (u16, bool), + bwf: Option, + mask: Option, + force_rf64: bool, + quantizers: Vec, + frames: u64, + bytes: Vec, + /// Set when a block write has failed: the sink may hold a prefix of that block while + /// `frames` does not count it, so the writer refuses every later call. + failed: bool, +} + +/// The optional header parts of a WAV file: BWF metadata, speaker mask, and whether to force RF64. +#[derive(Clone, Copy)] +pub(crate) struct HeaderExtras<'a> { + pub(crate) bwf: Option<&'a BwfInfo>, + pub(crate) mask: Option, + pub(crate) force_rf64: bool, +} + +impl Writer { + pub(crate) fn new(mut out: W, channels: usize, sample_rate: u32, depth: BitDepth, dither: bool, extras: HeaderExtras<'_>) -> Result { + let HeaderExtras { bwf, mask, force_rf64 } = extras; + validate_layout(channels, sample_rate)?; + let format = sample_format(depth); + let start = out.stream_position()?; + out.write_all(&header(channels, sample_rate, format, bwf, mask, 0, force_rf64)?)?; + let quantizers = Quantizer::per_channel(u32::from(format.0), dither, channels); + Ok(Writer { + out, + start, + channels, + sample_rate, + format, + bwf: bwf.cloned(), + mask, + force_rf64, + quantizers, + frames: 0, + bytes: Vec::new(), + failed: false, + }) + } + + fn header(&self, frames: u64) -> Result> { + header(self.channels, self.sample_rate, self.format, self.bwf.as_ref(), self.mask, frames, self.force_rf64) + } + + /// The frame count after `frames` more (RF64 has room for any bounce). + pub(crate) fn check_room(&self, frames: u64) -> Result { + let total = self.frames.saturating_add(frames); + self.header(total)?; + Ok(total) + } + + /// Appends `frames` frames of planar audio (missing samples are silence). + pub(crate) fn write(&mut self, block: &[Vec], frames: usize) -> Result<()> { + if self.failed { + return Err(crate::failed_write_error()); + } + let total = self.check_room(frames as u64)?; + if let Err(e) = write_interleaved(&mut self.out, &mut self.bytes, block, frames, &mut self.quantizers, self.format, false) { + self.failed = true; + return Err(AudioError::from(e)); + } + self.frames = total; + Ok(()) + } + + pub(crate) fn finish(mut self) -> Result { + if self.failed { + return Err(crate::failed_write_error()); + } + let data_len = u64::from(self.format.0 / 8).saturating_mul(self.channels as u64).saturating_mul(self.frames); + if data_len % 2 == 1 { + self.out.write_all(&[0])?; + } + let head = self.header(self.frames)?; + self.out.seek(SeekFrom::Start(self.start))?; + self.out.write_all(&head)?; + self.out.seek(SeekFrom::End(0))?; + self.out.flush()?; + Ok(self.out) + } +} + +/// Encode a WAV file. `force_rf64` writes the RF64 layout even for small files (for tests); +/// it is used automatically when the file would exceed 4 GiB. +/// `mask` overrides the speaker mask derived from the channel count. +pub(crate) fn encode( + buf: &AudioBuffer, + depth: BitDepth, + dither: bool, + bwf: Option<&BwfInfo>, + force_rf64: bool, + mask: Option, +) -> Result> { + validate_buffer(buf)?; + let channels = buf.channels.len(); + let frames = buf.frames(); + let format = sample_format(depth); + let head = header(channels, buf.sample_rate, format, bwf, mask, frames as u64, force_rf64)?; + let data_len = (usize::from(format.0 / 8) * channels).checked_mul(frames).ok_or_else(|| AudioError::TooLarge("data size overflow".into()))?; + let mut out = Vec::new(); + out.try_reserve_exact(head.len().saturating_add(data_len).saturating_add(1)) + .map_err(|_| AudioError::TooLarge(format!("cannot allocate {data_len} bytes for WAV output")))?; + let mut w = Writer::new(Cursor::new(out), channels, buf.sample_rate, depth, dither, HeaderExtras { bwf, mask, force_rf64 })?; + w.write(&buf.channels, frames)?; + Ok(w.finish()?.into_inner()) +} + +/// Everything before the sample data of a WAV holding `frames` frames. +pub(crate) fn header( + channels: usize, + sample_rate: u32, + (bits, is_float): (u16, bool), + bwf: Option<&BwfInfo>, + mask: Option, + frames: u64, + force_rf64: bool, +) -> Result> { + let block = usize::from(bits / 8).checked_mul(channels).ok_or_else(|| AudioError::TooLarge("block size overflow".into()))?; + let block_align = u16::try_from(block).map_err(|_| AudioError::Encode("too many channels for WAV".into()))?; + let byte_rate = sample_rate.checked_mul(u32::from(block_align)).ok_or_else(|| AudioError::Encode("WAV byte rate overflow".into()))?; + let data_len = u64::from(block_align).checked_mul(frames).ok_or_else(|| AudioError::TooLarge("data size overflow".into()))?; + let extensible = channels > 2 || bits > 16 || mask.is_some_and(|m| m != channel_mask(channels)); + let fmt_tag = if is_float { WAVE_FORMAT_IEEE_FLOAT } else { WAVE_FORMAT_PCM }; + + let mut fmt = Vec::with_capacity(40); + fmt.extend_from_slice(&(if extensible { WAVE_FORMAT_EXTENSIBLE } else { fmt_tag }).to_le_bytes()); + fmt.extend_from_slice(&(channels as u16).to_le_bytes()); + fmt.extend_from_slice(&sample_rate.to_le_bytes()); + fmt.extend_from_slice(&byte_rate.to_le_bytes()); + fmt.extend_from_slice(&block_align.to_le_bytes()); + fmt.extend_from_slice(&bits.to_le_bytes()); + if extensible { + fmt.extend_from_slice(&22u16.to_le_bytes()); + fmt.extend_from_slice(&bits.to_le_bytes()); + fmt.extend_from_slice(&mask.unwrap_or_else(|| channel_mask(channels)).to_le_bytes()); + fmt.extend_from_slice(&fmt_tag.to_le_bytes()); + fmt.extend_from_slice(&GUID_TAIL); + } + + let mut head = Vec::with_capacity(1024); + let rf64_body_len = 28usize; + // Reserve room for ds64 (as JUNK when not needed, so a later writer could upgrade in place). + let mut meta = Vec::new(); + if let Some(bwf) = bwf { + push_chunk(&mut meta, b"bext", &bext_chunk(bwf)); + } + push_chunk(&mut meta, b"fmt ", &fmt); + if is_float { + push_chunk(&mut meta, b"fact", &u32::try_from(frames).unwrap_or(u32::MAX).to_le_bytes()); + } + let total = ((4 + (8 + rf64_body_len) + meta.len() + 8) as u64) + .checked_add(data_len) + .and_then(|t| t.checked_add(data_len & 1)) + .ok_or_else(|| AudioError::TooLarge("WAV size overflow".into()))?; + let rf64 = force_rf64 || u32::try_from(total).is_err(); + + if rf64 { + head.extend_from_slice(b"RF64"); + head.extend_from_slice(&u32::MAX.to_le_bytes()); + head.extend_from_slice(b"WAVE"); + let mut ds64 = Vec::with_capacity(rf64_body_len); + ds64.extend_from_slice(&total.to_le_bytes()); + ds64.extend_from_slice(&data_len.to_le_bytes()); + ds64.extend_from_slice(&frames.to_le_bytes()); + ds64.extend_from_slice(&0u32.to_le_bytes()); + push_chunk(&mut head, b"ds64", &ds64); + } else { + head.extend_from_slice(b"RIFF"); + head.extend_from_slice(&(total as u32).to_le_bytes()); + head.extend_from_slice(b"WAVE"); + push_chunk(&mut head, b"JUNK", &[0u8; 28]); + } + head.extend_from_slice(&meta); + head.extend_from_slice(b"data"); + head.extend_from_slice(&(if rf64 { u32::MAX } else { data_len as u32 }).to_le_bytes()); + Ok(head) +} + +#[cfg(test)] +mod tests { + use super::header; + + #[test] + fn rejects_byte_rate_overflow() { + let result = header(2, 1_100_000_000, (16, false), None, None, 0, false); + assert!(result.is_err(), "overflowing byte rate was accepted"); + } +} diff --git a/crates/av1enc/Cargo.toml b/crates/av1enc/Cargo.toml new file mode 100644 index 0000000000..d7228d1add --- /dev/null +++ b/crates/av1enc/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "craft-av1enc" +description = "Clean-room pure-Rust AV1 encoder for the Craft ecosystem" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] diff --git a/crates/av1enc/README.md b/crates/av1enc/README.md new file mode 100644 index 0000000000..dcc3436935 --- /dev/null +++ b/crates/av1enc/README.md @@ -0,0 +1,76 @@ +# effectcraft-av1enc + +Clean-room, pure-Rust AV1 encoder for EffectCraft's MP4 / WebM export (layer L0, `std` only, +no `unsafe`, builds for `wasm32-unknown-unknown`). + +Output is decoded **bit-exactly** by libdav1d (`ffmpeg -c:v libdav1d`, checked in +`tests/it/ffmpeg_oracle.rs` when ffmpeg is installed) and by FilmCraft's spec-derived +`filmcraft-av1` decoder (`tests/it/roundtrip.rs`): every decoded frame equals the encoder's own +reconstruction, so prediction never drifts. + +## What is implemented + +- **Stream**: Main profile (`seq_profile` 0), 4:2:0, 8 or 10 bit, BT.709 colour description, + limited or full range. Sequence header with one operating point; `seq_level_idx` given or the + lowest Annex A level whose MaxPicSize / MaxH/VSize / MaxDisplayRate (and, in bitrate mode, Main + tier MaxBitrate) fit. Order hints, film grain, superres, CDEF, loop restoration, intra edge + filter, filter intra, inter-intra, masked / distance compound, warped motion and dual filter + are disabled in the sequence header. +- **Packets**: one temporal unit per frame as stored in MP4 / Matroska samples: OBUs with size + fields, no temporal delimiter; key frames start with the sequence header OBU, followed by one + `OBU_FRAME` (frame header + tile group). Every frame is shown, in input order. + `Encoder::av1c()` returns the `AV1CodecConfigurationRecord` (also the WebM `CodecPrivate`). +- **Frames**: a `KEY_FRAME` every `keyint` frames, `INTER_FRAME`s otherwise. Inter frames use + slot 0 (the previous reconstruction) for all seven references and refresh slot 0 only, so + `LAST_FRAME` is always the previous frame. CDFs adapt per symbol and are saved at the end of + every frame; inter frames load them from `LAST_FRAME` (`primary_ref_frame` = 0). One tile, + `TX_MODE_LARGEST`, `reduced_tx_set`, no segmentation, no delta q / lf. +- **Deblocking**: the normative loop filter (7.14) runs on the reconstruction; the frame level + (the same for both luma directions and both chroma planes, no deltas, sharpness 0) is searched + per frame for the lowest luma error. +- **Tiles**: the multi-symbol arithmetic encoder is the exact inverse of the spec's symbol + decoder (8.2), including the exit process padding; CDF adaptation as in 8.2.6. +- **Partitions**: 64x64 superblocks, `PARTITION_NONE` / `PARTITION_SPLIT` down to 8x8 chosen by + a rate-distortion cost (SSE + λ·estimated bits); blocks crossing the frame edge are split as + the syntax requires. 64x64 blocks are only used as inter skip blocks. +- **Intra**: DC, V, H, Paeth, smooth, smooth-V, smooth-H for luma and chroma, + ranked by SATD with a rate-distortion choice among the best three; the spec's edge preparation + and above-right / below-left availability. +- **Inter**: single reference (`LAST_FRAME`), `NEWMV` / `NEARESTMV` / `NEARMV` / `GLOBALMV` + chosen from the spec's motion vector prediction stack (7.10.2, mirrored exactly); motion search + from neighbour / co-located / parent candidates, a whole-sample diamond search and half- then + quarter-sample refinement (`allow_high_precision_mv` = 0); the normative 8-tap `EIGHTTAP` + prediction for luma and chroma; skip blocks; per-block intra / inter choice. +- **Residual**: one transform per plane and block (4x4 ... 32x32): for intra luma `DCT_DCT` or + the mode's preferred `ADST` combination (rate-distortion choice over the two best modes), the + mode-dependent type for intra chroma, `DCT_DCT` for inter blocks; dead-zone quantisation with the + spec's `Dc_Qlookup` / `Ac_Qlookup`; the normative dequantisation and inverse transforms in the + reconstruction loop; full coefficient coding (all_zero, tx type, eob_pt, eob_extra, + coeff_base_eob, coeff_base, coeff_br, dc_sign, Golomb) with the exact contexts. +- **Rate control**: `ConstantQ(base_q_idx)`; `Bitrate { kbps }` adapts `base_q_idx` per frame + towards the average target (key frames get a lower quantizer and a larger share of the budget of + their group of `keyint` frames, the inter frames the rest; the first key frame is searched in a + few passes, a later key frame that misses its budget by more than 19 % is coded again; the + deviation from the budget is paid back over 16 frames). + +## Limits + +- One tile: width and height up to 4096 and at most 2304 superblocks (4096x2304 samples). +- No CDEF or loop restoration, no compound prediction, no rectangular or 4xN blocks, no + directional intra modes other than V / H, no CfL or palette, no transform-size splits, so + compression is below mature encoders. +- 4:2:0 only (Main profile), no 12 bit. + +## Specification and provenance + +Implemented from the **AV1 Bitstream & Decoding Process Specification, version 1.0.0 with +Errata 1** (AOMediaCodec, 2019; the edition FilmCraft's decoder follows) and the **AV1 Codec +ISO Media File Format Binding** (v1.2.0) for `av1C`. + +The constant tables (default CDFs, scan orders, quantizer lookup tables, interpolation filter +taps, smooth weights, block / transform size tables, ...) and the decoder-side processes the +encoder must reproduce exactly (inverse transforms, intra and inter prediction rounding, context +derivations, motion vector prediction) are mirrored from FilmCraft's first-party, spec-derived +`filmcraft-av1` decoder (MIT OR Apache-2.0), which is also the round-trip oracle. No source of +libaom, dav1d, rav1e, SVT-AV1, ffmpeg or any other third-party encoder or decoder was consulted; +ffmpeg / libdav1d is used only as an external test oracle. diff --git a/crates/av1enc/src/bitw.rs b/crates/av1enc/src/bitw.rs new file mode 100644 index 0000000000..e4c8a1e691 --- /dev/null +++ b/crates/av1enc/src/bitw.rs @@ -0,0 +1,87 @@ +//! MSB-first bit writer for the OBU headers (spec 4.10 `f(n)` descriptors) and `leb128()`. + +#[derive(Default)] +pub(crate) struct BitWriter { + pub buf: Vec, + nbits: usize, +} + +impl BitWriter { + pub fn new() -> Self { + BitWriter::default() + } + + /// `f(n)`: writes the low `n` bits of `v`, most significant first. + pub fn f(&mut self, n: u32, v: u32) { + for i in (0..n).rev() { + self.bit((v >> i) & 1 != 0); + } + } + + pub fn bit(&mut self, b: bool) { + if self.nbits.is_multiple_of(8) { + self.buf.push(0); + } + if b { + let last = self.buf.len() - 1; + self.buf[last] |= 0x80 >> (self.nbits % 8); + } + self.nbits += 1; + } + + /// `byte_alignment()`: zero bits up to the next byte boundary. + pub fn byte_align(&mut self) { + while !self.nbits.is_multiple_of(8) { + self.bit(false); + } + } + + /// `trailing_bits()`: a one bit followed by zero bits up to the byte boundary. + pub fn trailing_bits(&mut self) { + self.bit(true); + self.byte_align(); + } + + pub fn into_bytes(self) -> Vec { + self.buf + } +} + +/// `leb128()` encoding of an OBU size. +pub(crate) fn leb128(mut v: usize, out: &mut Vec) { + loop { + let mut b = (v & 0x7f) as u8; + v >>= 7; + if v != 0 { + b |= 0x80; + } + out.push(b); + if v == 0 { + break; + } + } +} + +/// An OBU with `obu_has_size_field` = 1 and no extension header. +pub(crate) fn obu(obu_type: u8, payload: &[u8], out: &mut Vec) { + out.push((obu_type << 3) | 0x02); + leb128(payload.len(), out); + out.extend_from_slice(payload); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bits_and_leb128() { + let mut w = BitWriter::new(); + w.f(3, 0b101); + w.f(6, 0b110011); + w.trailing_bits(); + assert_eq!(w.into_bytes(), vec![0b1011_1001, 0b1100_0000]); + let mut v = Vec::new(); + leb128(300, &mut v); + assert_eq!(v, vec![0xAC, 0x02]); + } +} diff --git a/crates/av1enc/src/cdf.rs b/crates/av1enc/src/cdf.rs new file mode 100644 index 0000000000..8bbdee6758 --- /dev/null +++ b/crates/av1enc/src/cdf.rs @@ -0,0 +1,172 @@ +//! The adaptive CDFs this encoder codes with (the subset of spec 6.8.2 init_non_coeff_cdfs / +//! init_coeff_cdfs that its syntax uses). Layout as in filmcraft-av1: each innermost array +//! holds N cumulative values (the last is 32768) plus the adaptation counter. + +use crate::tables::{ + DEFAULT_ANGLE_DELTA_CDF, DEFAULT_COEFF_BASE_CDF, DEFAULT_COEFF_BASE_EOB_CDF, DEFAULT_COEFF_BR_CDF, DEFAULT_DC_SIGN_CDF, DEFAULT_DRL_MODE_CDF, + DEFAULT_EOB_EXTRA_CDF, DEFAULT_EOB_PT_16_CDF, DEFAULT_EOB_PT_32_CDF, DEFAULT_EOB_PT_64_CDF, DEFAULT_EOB_PT_128_CDF, DEFAULT_EOB_PT_256_CDF, + DEFAULT_EOB_PT_512_CDF, DEFAULT_EOB_PT_1024_CDF, DEFAULT_INTER_TX_TYPE_SET3_CDF, DEFAULT_INTRA_TX_TYPE_SET2_CDF, DEFAULT_IS_INTER_CDF, + DEFAULT_MV_BIT_CDF, DEFAULT_MV_CLASS_CDF, DEFAULT_MV_CLASS0_BIT_CDF, DEFAULT_MV_CLASS0_FR_CDF, DEFAULT_MV_FR_CDF, DEFAULT_MV_JOINT_CDF, + DEFAULT_MV_SIGN_CDF, DEFAULT_NEW_MV_CDF, DEFAULT_PARTITION_W8_CDF, DEFAULT_PARTITION_W16_CDF, DEFAULT_PARTITION_W32_CDF, + DEFAULT_PARTITION_W64_CDF, DEFAULT_REF_MV_CDF, DEFAULT_SINGLE_REF_CDF, DEFAULT_SKIP_CDF, DEFAULT_TXB_SKIP_CDF, DEFAULT_UV_MODE_CFL_ALLOWED_CDF, + DEFAULT_UV_MODE_CFL_NOT_ALLOWED_CDF, DEFAULT_Y_MODE_CDF, DEFAULT_ZERO_MV_CDF, +}; + +#[derive(Clone)] +pub(crate) struct Cdfs { + pub y_mode: [[u16; 14]; 4], + pub uv_mode_cfl_not_allowed: [[u16; 14]; 13], + pub uv_mode_cfl_allowed: [[u16; 15]; 13], + pub angle_delta: [[u16; 8]; 8], + pub partition_w8: [[u16; 5]; 4], + pub partition_w16: [[u16; 11]; 4], + pub partition_w32: [[u16; 11]; 4], + pub partition_w64: [[u16; 11]; 4], + pub skip: [[u16; 3]; 3], + pub is_inter: [[u16; 3]; 4], + pub single_ref: [[[u16; 3]; 6]; 3], + pub new_mv: [[u16; 3]; 6], + pub zero_mv: [[u16; 3]; 2], + pub ref_mv: [[u16; 3]; 6], + pub drl_mode: [[u16; 3]; 3], + pub mv_joint: [u16; 5], + pub mv_class: [[u16; 12]; 2], + pub mv_class0_bit: [[u16; 3]; 2], + pub mv_fr: [[u16; 5]; 2], + pub mv_class0_fr: [[[u16; 5]; 2]; 2], + pub mv_sign: [[u16; 3]; 2], + pub mv_bit: [[[u16; 3]; 10]; 2], + pub intra_tx_type_set2: [[[u16; 6]; 13]; 3], + pub inter_tx_type_set3: [[u16; 3]; 4], + pub txb_skip: [[[u16; 3]; 13]; 5], + pub eob_pt_16: [[[u16; 6]; 2]; 2], + pub eob_pt_32: [[[u16; 7]; 2]; 2], + pub eob_pt_64: [[[u16; 8]; 2]; 2], + pub eob_pt_128: [[[u16; 9]; 2]; 2], + pub eob_pt_256: [[[u16; 10]; 2]; 2], + pub eob_pt_512: [[u16; 11]; 2], + pub eob_pt_1024: [[u16; 12]; 2], + pub eob_extra: [[[[u16; 3]; 9]; 2]; 5], + pub dc_sign: [[[u16; 3]; 3]; 2], + pub coeff_base_eob: [[[[u16; 4]; 4]; 2]; 5], + pub coeff_base: [[[[u16; 5]; 42]; 2]; 5], + pub coeff_br: [[[[u16; 5]; 21]; 2]; 5], +} + +impl Cdfs { + /// init_non_coeff_cdfs( ) followed by init_coeff_cdfs( ) for `base_q_idx`. + pub fn new(base_q_idx: u32) -> Box { + let idx = match base_q_idx { + 0..=20 => 0, + 21..=60 => 1, + 61..=120 => 2, + _ => 3, + }; + Box::new(Cdfs { + y_mode: DEFAULT_Y_MODE_CDF, + uv_mode_cfl_not_allowed: DEFAULT_UV_MODE_CFL_NOT_ALLOWED_CDF, + uv_mode_cfl_allowed: DEFAULT_UV_MODE_CFL_ALLOWED_CDF, + angle_delta: DEFAULT_ANGLE_DELTA_CDF, + partition_w8: DEFAULT_PARTITION_W8_CDF, + partition_w16: DEFAULT_PARTITION_W16_CDF, + partition_w32: DEFAULT_PARTITION_W32_CDF, + partition_w64: DEFAULT_PARTITION_W64_CDF, + skip: DEFAULT_SKIP_CDF, + is_inter: DEFAULT_IS_INTER_CDF, + single_ref: DEFAULT_SINGLE_REF_CDF, + new_mv: DEFAULT_NEW_MV_CDF, + zero_mv: DEFAULT_ZERO_MV_CDF, + ref_mv: DEFAULT_REF_MV_CDF, + drl_mode: DEFAULT_DRL_MODE_CDF, + mv_joint: DEFAULT_MV_JOINT_CDF, + mv_class: DEFAULT_MV_CLASS_CDF, + mv_class0_bit: [DEFAULT_MV_CLASS0_BIT_CDF; 2], + mv_fr: DEFAULT_MV_FR_CDF, + mv_class0_fr: DEFAULT_MV_CLASS0_FR_CDF, + mv_sign: [DEFAULT_MV_SIGN_CDF; 2], + mv_bit: [DEFAULT_MV_BIT_CDF; 2], + intra_tx_type_set2: DEFAULT_INTRA_TX_TYPE_SET2_CDF, + inter_tx_type_set3: DEFAULT_INTER_TX_TYPE_SET3_CDF, + txb_skip: DEFAULT_TXB_SKIP_CDF[idx], + eob_pt_16: DEFAULT_EOB_PT_16_CDF[idx], + eob_pt_32: DEFAULT_EOB_PT_32_CDF[idx], + eob_pt_64: DEFAULT_EOB_PT_64_CDF[idx], + eob_pt_128: DEFAULT_EOB_PT_128_CDF[idx], + eob_pt_256: DEFAULT_EOB_PT_256_CDF[idx], + eob_pt_512: DEFAULT_EOB_PT_512_CDF[idx], + eob_pt_1024: DEFAULT_EOB_PT_1024_CDF[idx], + eob_extra: DEFAULT_EOB_EXTRA_CDF[idx], + dc_sign: DEFAULT_DC_SIGN_CDF[idx], + coeff_base_eob: DEFAULT_COEFF_BASE_EOB_CDF[idx], + coeff_base: DEFAULT_COEFF_BASE_CDF[idx], + coeff_br: DEFAULT_COEFF_BR_CDF[idx], + }) + } +} + +impl Cdfs { + /// The counter reset of load_cdfs( ): the last entry of every CDF array is set to 0. + pub fn clear_counts(&mut self) { + macro_rules! clr { + ($($f:ident),*) => { $( self.$f.clear_count(); )* }; + } + clr!( + y_mode, + uv_mode_cfl_not_allowed, + uv_mode_cfl_allowed, + angle_delta, + partition_w8, + partition_w16, + partition_w32, + partition_w64, + skip, + is_inter, + single_ref, + new_mv, + zero_mv, + ref_mv, + drl_mode, + mv_joint, + mv_class, + mv_class0_bit, + mv_fr, + mv_class0_fr, + mv_sign, + mv_bit, + intra_tx_type_set2, + inter_tx_type_set3, + txb_skip, + eob_pt_16, + eob_pt_32, + eob_pt_64, + eob_pt_128, + eob_pt_256, + eob_pt_512, + eob_pt_1024, + eob_extra, + dc_sign, + coeff_base_eob, + coeff_base, + coeff_br + ); + } +} + +/// Zeroes the adaptation counter (last element) of every innermost CDF array. +trait ClearCount { + fn clear_count(&mut self); +} + +impl ClearCount for [u16; N] { + fn clear_count(&mut self) { + self[N - 1] = 0; + } +} + +impl ClearCount for [T; M] { + fn clear_count(&mut self) { + for e in self.iter_mut() { + e.clear_count(); + } + } +} diff --git a/crates/av1enc/src/decide.rs b/crates/av1enc/src/decide.rs new file mode 100644 index 0000000000..c2ce97fef5 --- /dev/null +++ b/crates/av1enc/src/decide.rs @@ -0,0 +1,1076 @@ +//! Mode decision and reconstruction (not normative): partitions, intra modes, motion vectors, +//! transform coefficients. Every prediction and reconstruction step uses the normative +//! processes, so the reconstruction built here is exactly what a decoder produces; the syntax +//! is written afterwards by `writer.rs`. + +use crate::pred::{INTRA_MODES_USED, InterBlock, IntraEdge, Plane, edges, half_planes, predict_inter, predict_intra}; +use crate::tables::{ + AC_QLOOKUP, BLOCK_8X8, BLOCK_64X64, DC_PRED, DC_QLOOKUP, DCT_DCT, DEFAULT_SCAN_4X4, DEFAULT_SCAN_8X8, DEFAULT_SCAN_16X16, DEFAULT_SCAN_32X32, + MODE_TO_TXFM, NEWMV, NUM_4X4_BLOCKS_WIDE, PARTITION_SPLIT, PARTITION_SUBSIZE, TX_4X4, TX_8X8, TX_16X16, TX_32X32, TX_64X64, TX_SET_DCTONLY, + TX_SET_INTER_3, TX_SET_INTRA_2, TX_SIZE_SQR_UP, TX_TYPE_IN_SET_INTER, TX_TYPE_IN_SET_INTRA, +}; +use crate::transform::{forward_2d, inverse_2d}; + +/// Decisions for one coded block. +#[derive(Clone)] +pub(crate) struct Leaf { + pub bsize: usize, + pub is_inter: bool, + pub y_mode: usize, + pub uv_mode: usize, + pub tx_type_y: usize, + /// Motion vector [row, col] in 1/8 luma samples (inter blocks). + pub mv: [i32; 2], + pub skip: bool, + /// Quantised coefficients of the single transform block of each plane (raster order). + pub coefs: [Vec; 3], +} + +pub(crate) enum Node { + Out, + Leaf(Box), + Split(Box<[Node; 4]>), +} + +/// A reconstructed frame kept as a reference. +pub(crate) struct RefFrame { + pub planes: [Plane; 3], + /// Motion vectors chosen for the frame (per 4x4), used as search candidates. + pub mvs: Vec<[i32; 2]>, + /// The frame's loop filter level. + pub lf_level: u32, + /// The CDFs saved at the end of the frame (loaded by the next frame). + pub cdfs: Box, +} + +pub(crate) struct FrameGeom { + pub width: usize, + pub height: usize, + pub mi_cols: usize, + pub mi_rows: usize, + pub sb_cols: usize, + pub sb_rows: usize, +} + +impl FrameGeom { + pub fn new(width: usize, height: usize) -> FrameGeom { + let mi_cols = 2 * width.div_ceil(8); + let mi_rows = 2 * height.div_ceil(8); + FrameGeom { width, height, mi_cols, mi_rows, sb_cols: mi_cols.div_ceil(16), sb_rows: mi_rows.div_ceil(16) } + } + /// Padded plane size (whole superblocks). + pub fn padded(&self, plane: usize) -> (usize, usize) { + let s = (plane > 0) as usize; + ((self.sb_cols * 64) >> s, (self.sb_rows * 64) >> s) + } + /// Visible plane size. + pub fn visible(&self, plane: usize) -> (usize, usize) { + if plane == 0 { (self.width, self.height) } else { (self.width.div_ceil(2), self.height.div_ceil(2)) } + } +} + +pub(crate) fn tx_size_for(log2: u32) -> usize { + match log2 { + 2 => TX_4X4, + 3 => TX_8X8, + 4 => TX_16X16, + 5 => TX_32X32, + _ => TX_64X64, + } +} + +/// get_tx_set( ) for the square sizes used here with reduced_tx_set = 1. +pub(crate) fn tx_set(tx_sz: usize, is_inter: bool) -> usize { + let sqr_up = TX_SIZE_SQR_UP[tx_sz] as usize; + if sqr_up > TX_32X32 { + return TX_SET_DCTONLY; + } + if is_inter { + TX_SET_INTER_3 + } else if sqr_up == TX_32X32 { + TX_SET_DCTONLY + } else { + TX_SET_INTRA_2 + } +} + +pub(crate) fn tx_in_set(set: usize, is_inter: bool, tx_type: usize) -> bool { + if is_inter { TX_TYPE_IN_SET_INTER[set][tx_type] == 1 } else { TX_TYPE_IN_SET_INTRA[set][tx_type] == 1 } +} + +/// The chroma transform type of an intra block (compute_tx_type for plane > 0). +pub(crate) fn intra_uv_tx_type(uv_mode: usize, uv_tx: usize) -> usize { + let t = MODE_TO_TXFM[uv_mode] as usize; + if tx_in_set(tx_set(uv_tx, false), false, t) { t } else { DCT_DCT } +} + +/// Geometry and coding mode of one square transform block. +#[derive(Clone, Copy)] +struct TransformBlock { + plane: usize, + x: usize, + y: usize, + log2: u32, + tx_type: usize, + intra: bool, +} + +struct TxResult { + levels: Vec, + nonzero: bool, + dist: f64, + bits: f64, +} + +/// The output and scratch buffers of one reconstruction, each `n * n` entries long. +struct ReconBuffers<'a> { + deq: &'a mut [i32], + resid: &'a mut [i32], + recon: &'a mut [u16], +} + +/// Outcome of trying one transform block; the levels and the reconstruction stay in `TxScratch`. +#[derive(Clone, Copy)] +struct TxEval { + nonzero: bool, + dist: f64, + bits: f64, +} + +/// Buffers of the transform path, sized for the largest coded transform block (32x32). Only the +/// first `n * n` entries are used; they are reused across candidates instead of being allocated. +struct TxScratch { + res: [i32; 1024], + coef: [f32; 1024], + levels: [i32; 1024], + deq: [i32; 1024], + resid: [i32; 1024], + recon: [u16; 1024], +} + +impl Default for TxScratch { + fn default() -> Self { + TxScratch { res: [0; 1024], coef: [0.0; 1024], levels: [0; 1024], deq: [0; 1024], resid: [0; 1024], recon: [0; 1024] } + } +} + +/// Saved encoder state of a block area (reconstruction, BlockDecoded flags). +struct Snapshot { + planes: [Vec; 3], + decoded: Box<[[[bool; 35]; 35]; 3]>, +} + +pub(crate) struct Decider<'a> { + pub g: &'a FrameGeom, + pub bd: u32, + pub intra_only: bool, + pub src: &'a [Plane; 3], + pub rec: [Plane; 3], + pub refr: Option<&'a RefFrame>, + decoded: Box<[[[bool; 35]; 35]; 3]>, + /// Decision motion vectors per 4x4 (search candidates). + pub mvs: Vec<[i32; 2]>, + dc_q: f32, + ac_q: f32, + dc_qi: i64, + ac_qi: i64, + /// Smallest magnitude that quantises to a non-zero level, `[dc, ac][intra]`. + dead_zone: [[f32; 2]; 2], + lambda: f64, + lambda_sad: f64, + /// Half-sample interpolations of the reference luma (visible area): (8, 0), (0, 8), (8, 8). + half: [Vec; 3], + /// Transform buffers, taken out while a block is being tried (see `code_tx`). + scratch: Option>, +} + +const MODE_BITS_INTRA: f64 = 3.0; +/// Dead-zone quantiser rounding offsets (below one half) for inter and intra blocks. +const INTER_RND: f32 = 0.30; +const INTRA_RND: f32 = 0.38; + +impl<'a> Decider<'a> { + pub fn new(g: &'a FrameGeom, bd: u32, qidx: u8, intra_only: bool, src: &'a [Plane; 3], refr: Option<&'a RefFrame>) -> Self { + let bdi = ((bd - 8) >> 1) as usize; + let dc_qi = DC_QLOOKUP[bdi][qidx as usize] as i64; + let ac_qi = AC_QLOOKUP[bdi][qidx as usize] as i64; + // Lagrangian multiplier in squared-sample units of the coded bit depth. + let step = ac_qi as f64 / 8.0; + let lambda = 0.25 * step * step; + let rec = std::array::from_fn(|p| { + let (w, h) = g.padded(p); + Plane::new(w, h) + }); + Decider { + g, + bd, + intra_only, + src, + rec, + refr, + decoded: Box::new([[[false; 35]; 35]; 3]), + mvs: vec![[0, 0]; g.mi_cols * g.mi_rows], + dc_q: dc_qi as f32, + ac_q: ac_qi as f32, + dc_qi, + ac_qi, + dead_zone: [dc_qi, ac_qi].map(|q| [INTER_RND, INTRA_RND].map(|rnd| zero_threshold(q as f32, rnd))), + lambda, + lambda_sad: lambda.sqrt(), + half: match refr { + Some(rf) => half_planes(&rf.planes[0], g.width, g.height, bd), + None => Default::default(), + }, + scratch: None, + } + } + + /// Decides one 64x64 superblock (reconstruction is written to `rec`). + pub fn superblock(&mut self, r: usize, c: usize) -> Node { + // clear_block_decoded_flags( r, c, sbSize4 ) + for plane in 0..3 { + let s = (plane > 0) as usize; + let sb_w4 = (self.g.mi_cols as isize - c as isize) >> s; + let sb_h4 = (self.g.mi_rows as isize - r as isize) >> s; + let bd = &mut self.decoded[plane]; + for y in -1..=((16 >> s) as isize) { + for x in -1..=((16 >> s) as isize) { + let v = if y < 0 && x < sb_w4 { true } else { x < 0 && y < sb_h4 }; + bd[(y + 1) as usize][(x + 1) as usize] = v; + } + } + bd[(16 >> s) + 1][0] = false; + } + self.partition(r, c, BLOCK_64X64, None).0 + } + + fn decoded_at(&self, plane: usize, y: isize, x: isize) -> bool { + let (yy, xx) = (y + 1, x + 1); + if yy < 0 || xx < 0 || yy > 34 || xx > 34 { + return false; + } + self.decoded[plane][yy as usize][xx as usize] + } + + /// Marks the transform block of `plane` covering the block at (r, c) as decoded. + fn mark_decoded(&mut self, plane: usize, r: usize, c: usize, n4: usize) { + let s = (plane > 0) as usize; + let by = (r & 15) >> s; + let bx = (c & 15) >> s; + let step = (n4 >> s).max(1); + for i in 0..step { + for j in 0..step { + if by + i + 1 < 35 && bx + j + 1 < 35 { + self.decoded[plane][by + i + 1][bx + j + 1] = true; + } + } + } + } + + fn snapshot(&self, r: usize, c: usize, n4: usize) -> Snapshot { + let planes = std::array::from_fn(|p| { + let s = (p > 0) as usize; + let (x0, y0, n) = ((c * 4) >> s, (r * 4) >> s, (n4 * 4) >> s); + let pl = &self.rec[p]; + let mut v = Vec::with_capacity(n * n); + for y in y0..y0 + n { + v.extend_from_slice(&pl.data[y * pl.stride + x0..y * pl.stride + x0 + n]); + } + v + }); + Snapshot { planes, decoded: self.decoded.clone() } + } + + fn restore(&mut self, snap: &Snapshot, r: usize, c: usize, n4: usize) { + for p in 0..3 { + let s = (p > 0) as usize; + let (x0, y0, n) = ((c * 4) >> s, (r * 4) >> s, (n4 * 4) >> s); + let pl = &mut self.rec[p]; + for (i, y) in (y0..y0 + n).enumerate() { + pl.data[y * pl.stride + x0..y * pl.stride + x0 + n].copy_from_slice(&snap.planes[p][i * n..i * n + n]); + } + } + self.decoded.clone_from(&snap.decoded); + } + + fn set_mvs(&mut self, r: usize, c: usize, n4: usize, mv: [i32; 2]) { + for y in r..(r + n4).min(self.g.mi_rows) { + for x in c..(c + n4).min(self.g.mi_cols) { + self.mvs[y * self.g.mi_cols + x] = mv; + } + } + } + + fn partition(&mut self, r: usize, c: usize, bsize: usize, parent_mv: Option<[i32; 2]>) -> (Node, f64) { + let g = self.g; + if r >= g.mi_rows || c >= g.mi_cols { + return (Node::Out, 0.0); + } + let n4 = NUM_4X4_BLOCKS_WIDE[bsize] as usize; + let half = n4 >> 1; + let has_rows = r + half < g.mi_rows; + let has_cols = c + half < g.mi_cols; + let can_none = has_rows && has_cols && (bsize != BLOCK_64X64 || !self.intra_only); + let can_split = bsize > BLOCK_8X8; + let sub = PARTITION_SUBSIZE[PARTITION_SPLIT][bsize] as usize; + if !can_none { + return self.split(r, c, sub, half, parent_mv); + } + let before = self.snapshot(r, c, n4); + let (leaf, cost_none) = self.leaf(r, c, bsize, parent_mv); + let cost_none = cost_none + self.lambda * 1.0; + if !can_split { + return (Node::Leaf(Box::new(leaf)), cost_none); + } + // a cheap skip block rarely gains from splitting + if leaf.skip && cost_none < self.lambda * 8.0 { + return (Node::Leaf(Box::new(leaf)), cost_none); + } + let after_none = self.snapshot(r, c, n4); + self.restore(&before, r, c, n4); + let pm = if leaf.is_inter { Some(leaf.mv) } else { parent_mv }; + let (node, cost_split) = self.split(r, c, sub, half, pm); + let cost_split = cost_split + self.lambda * 2.0; + if cost_split < cost_none { + (node, cost_split) + } else { + self.restore(&after_none, r, c, n4); + self.set_mvs(r, c, n4, if leaf.is_inter { leaf.mv } else { [0, 0] }); + (Node::Leaf(Box::new(leaf)), cost_none) + } + } + + fn split(&mut self, r: usize, c: usize, sub: usize, half: usize, pm: Option<[i32; 2]>) -> (Node, f64) { + let (n0, c0) = self.partition(r, c, sub, pm); + let (n1, c1) = self.partition(r, c + half, sub, pm); + let (n2, c2) = self.partition(r + half, c, sub, pm); + let (n3, c3) = self.partition(r + half, c + half, sub, pm); + (Node::Split(Box::new([n0, n1, n2, n3])), c0 + c1 + c2 + c3) + } + + /// Best coding of the block as a single (NONE partition) block. + fn leaf(&mut self, r: usize, c: usize, bsize: usize, parent_mv: Option<[i32; 2]>) -> (Leaf, f64) { + let n4 = NUM_4X4_BLOCKS_WIDE[bsize] as usize; + if self.intra_only { + let res = self.intra_leaf(r, c, bsize); + self.set_mvs(r, c, n4, [0, 0]); + return res; + } + let before = self.snapshot(r, c, n4); + let (inter, cost_inter) = self.inter_leaf(r, c, bsize, parent_mv); + if bsize == BLOCK_64X64 || inter.skip { + self.set_mvs(r, c, n4, inter.mv); + return (inter, cost_inter); + } + let after_inter = self.snapshot(r, c, n4); + self.restore(&before, r, c, n4); + let (intra, cost_intra) = self.intra_leaf(r, c, bsize); + if cost_intra < cost_inter { + self.set_mvs(r, c, n4, [0, 0]); + (intra, cost_intra) + } else { + self.restore(&after_inter, r, c, n4); + self.set_mvs(r, c, n4, inter.mv); + (inter, cost_inter) + } + } + + fn intra_leaf(&mut self, r: usize, c: usize, bsize: usize) -> (Leaf, f64) { + let n4 = NUM_4X4_BLOCKS_WIDE[bsize] as usize; + let log2 = (n4 * 4).trailing_zeros(); + let (sbr, sbc) = ((r & 15) as isize, (c & 15) as isize); + let bd = self.bd; + let mut pred = [0u16; 32 * 32]; + let mut best_pred = [0u16; 32 * 32]; + let mut best_levels = [0i32; 32 * 32]; + let mut best_recon = [0u16; 32 * 32]; + let n = 1usize << log2; + // luma + let e = IntraEdge { + x: c * 4, + y: r * 4, + have_left: c > 0, + have_above: r > 0, + have_above_right: self.decoded_at(0, sbr - 1, sbc + n4 as isize), + have_below_left: self.decoded_at(0, sbr + n4 as isize, sbc - 1), + log2, + max_x: (self.g.mi_cols * 4) as i32 - 1, + max_y: (self.g.mi_rows * 4) as i32 - 1, + bit_depth: bd, + }; + let ed = edges(&self.rec[0], &e); + // modes ranked by SATD, then a rate-distortion choice of mode and transform type among + // the best three + let mut ranked: [(f64, usize); INTRA_MODES_USED.len()] = INTRA_MODES_USED.map(|mode| { + predict_intra(&ed, &e, mode, &mut pred); + let sad = self.satd(0, c * 4, r * 4, n, &pred[..n * n]) as f64; + (sad + self.lambda_sad * if mode == DC_PRED { 1.0 } else { MODE_BITS_INTRA }, mode) + }); + ranked.sort_by(|a, b| a.0.total_cmp(&b.0)); + let tx_y = tx_size_for(log2); + let mut best: Option<(f64, usize, usize, TxEval, bool)> = None; + let mut sc = self.scratch.take().unwrap_or_default(); + for &(_, mode) in ranked.iter().take(3) { + predict_intra(&ed, &e, mode, &mut pred); + let mut types = [DCT_DCT; 2]; + let mut n_types = 1; + let alt = MODE_TO_TXFM[mode] as usize; + if alt != DCT_DCT && tx_in_set(tx_set(tx_y, false), false, alt) { + types[1] = alt; + n_types = 2; + } + for &t in &types[..n_types] { + let (res, has_recon) = + self.try_tx(&mut sc, TransformBlock { plane: 0, x: c * 4, y: r * 4, log2, tx_type: t, intra: true }, &pred[..n * n]); + let type_bits = if !res.nonzero || tx_set(tx_y, false) == TX_SET_DCTONLY { + 0.0 + } else if t == DCT_DCT { + 1.0 + } else { + 2.0 + }; + let mode_bits = if mode == DC_PRED { 1.0 } else { MODE_BITS_INTRA }; + let cost = res.dist + self.lambda * (res.bits + type_bits + mode_bits); + if best.as_ref().is_none_or(|b| cost < b.0) { + best_pred[..n * n].copy_from_slice(&pred[..n * n]); + best_levels[..n * n].copy_from_slice(&sc.levels[..n * n]); + if has_recon { + best_recon[..n * n].copy_from_slice(&sc.recon[..n * n]); + } + best = Some((cost, mode, t, res, has_recon)); + } + } + } + self.scratch = Some(sc); + // Every ranked mode tries DCT_DCT, so `best` is always set; without a candidate the block + // can't be coded intra (an infinite cost keeps it from being chosen). + let Some((_, y_mode, tx_type_y, ty, has_recon)) = best else { + let leaf = Leaf { + bsize, + is_inter: false, + y_mode: DC_PRED, + uv_mode: DC_PRED, + tx_type_y: DCT_DCT, + mv: [0, 0], + skip: true, + coefs: Default::default(), + }; + return (leaf, f64::INFINITY); + }; + self.put(0, c * 4, r * 4, n, if has_recon { &best_recon[..n * n] } else { &best_pred[..n * n] }); + let ty = TxResult { levels: best_levels[..n * n].to_vec(), nonzero: ty.nonzero, dist: ty.dist, bits: ty.bits }; + let tx_type_y = if ty.nonzero { tx_type_y } else { DCT_DCT }; + self.mark_decoded(0, r, c, n4); + // chroma (one mode for both planes) + let cn = n >> 1; + let clog2 = log2 - 1; + let (csbr, csbc) = (sbr >> 1, sbc >> 1); + let ce: [IntraEdge; 2] = std::array::from_fn(|i| IntraEdge { + x: c * 2, + y: r * 2, + have_left: c > 0, + have_above: r > 0, + have_above_right: self.decoded_at(1 + i, csbr - 1, csbc + (n4 >> 1) as isize), + have_below_left: self.decoded_at(1 + i, csbr + (n4 >> 1) as isize, csbc - 1), + log2: clog2, + max_x: (self.g.mi_cols * 2) as i32 - 1, + max_y: (self.g.mi_rows * 2) as i32 - 1, + bit_depth: bd, + }); + let ced = [edges(&self.rec[1], &ce[0]), edges(&self.rec[2], &ce[1])]; + let mut best_uv = (f64::MAX, DC_PRED); + for &mode in INTRA_MODES_USED.iter() { + let mut sad = 0u64; + for i in 0..2 { + predict_intra(&ced[i], &ce[i], mode, &mut pred); + sad += self.satd(1 + i, c * 2, r * 2, cn, &pred[..cn * cn]); + } + let cost = sad as f64 + self.lambda_sad * if mode == DC_PRED { 1.0 } else { 2.5 }; + if cost < best_uv.0 { + best_uv = (cost, mode); + } + } + let uv_mode = best_uv.1; + let uv_tx = tx_size_for(clog2); + let uv_type = intra_uv_tx_type(uv_mode, uv_tx); + let mut code_uv = |i: usize| { + predict_intra(&ced[i], &ce[i], uv_mode, &mut pred); + let t = self.code_tx(TransformBlock { plane: 1 + i, x: c * 2, y: r * 2, log2: clog2, tx_type: uv_type, intra: true }, &pred[..cn * cn]); + self.mark_decoded(1 + i, r, c, n4); + t + }; + let u = code_uv(0); + let v = code_uv(1); + let skip = !ty.nonzero && !u.nonzero && !v.nonzero; + let dist = ty.dist + u.dist + v.dist; + let bits = if skip { 1.0 } else { ty.bits + u.bits + v.bits + 1.0 } + MODE_BITS_INTRA + 2.0; + let leaf = Leaf { bsize, is_inter: false, y_mode, uv_mode, tx_type_y, mv: [0, 0], skip, coefs: [ty.levels, u.levels, v.levels] }; + (leaf, dist + self.lambda * bits) + } + + fn inter_leaf(&mut self, r: usize, c: usize, bsize: usize, parent_mv: Option<[i32; 2]>) -> (Leaf, f64) { + let n4 = NUM_4X4_BLOCKS_WIDE[bsize] as usize; + let n = n4 * 4; + let log2 = n.trailing_zeros(); + // Inter blocks are only decided with a reference frame; without one, code intra. + let Some(refr) = self.refr else { return self.intra_leaf(r, c, bsize) }; + let (mv, mv_bits) = self.motion_search(r, c, n, parent_mv); + let mut pred = vec![0u16; n * n]; + let mut tx = Vec::with_capacity(3); + let mut dist = 0.0; + for p in 0..3 { + let s = (p > 0) as usize; + let (vw, vh) = self.g.visible(p); + let pn = n >> s; + let (x, y) = ((c * 4) >> s, (r * 4) >> s); + predict_inter( + &refr.planes[p], + &InterBlock { last_x: vw as i32 - 1, last_y: vh as i32 - 1, x, y, width: pn, height: pn, mv, subsampling: s, bit_depth: self.bd }, + &mut pred[..pn * pn], + ); + if bsize == BLOCK_64X64 { + // coded as a skip block: the prediction is the reconstruction + dist += self.sse(p, x, y, pn, &pred[..pn * pn]); + self.put(p, x, y, pn, &pred[..pn * pn]); + } else { + let t = self.code_tx(TransformBlock { plane: p, x, y, log2: log2 - s as u32, tx_type: DCT_DCT, intra: false }, &pred[..pn * pn]); + dist += t.dist; + tx.push(t); + } + self.mark_decoded(p, r, c, n4); + } + let mode_bits = if mv == [0, 0] { 1.5 } else { 2.0 + mv_bits }; + if bsize == BLOCK_64X64 { + let leaf = Leaf { bsize, is_inter: true, y_mode: NEWMV, uv_mode: DC_PRED, tx_type_y: DCT_DCT, mv, skip: true, coefs: Default::default() }; + return (leaf, dist + self.lambda * (mode_bits + 1.0)); + } + let skip = tx.iter().all(|t| !t.nonzero); + let bits = mode_bits + if skip { 1.0 } else { 1.0 + tx.iter().map(|t| t.bits).sum::() }; + let [y, u, v]: [TxResult; 3] = match tx.try_into() { + Ok(planes) => planes, + // One transform block per plane was coded above. + Err(_) => { + return ( + Leaf { bsize, is_inter: true, y_mode: NEWMV, uv_mode: DC_PRED, tx_type_y: DCT_DCT, mv, skip: true, coefs: Default::default() }, + f64::INFINITY, + ); + } + }; + let leaf = + Leaf { bsize, is_inter: true, y_mode: NEWMV, uv_mode: DC_PRED, tx_type_y: DCT_DCT, mv, skip, coefs: [y.levels, u.levels, v.levels] }; + (leaf, dist + self.lambda * bits) + } + + /// Full-sample motion search: candidate vectors then a shrinking diamond search. + /// Returns the vector and an estimate of its coding cost in bits. + fn motion_search(&self, r: usize, c: usize, n: usize, parent_mv: Option<[i32; 2]>) -> ([i32; 2], f64) { + let g = self.g; + let Some(refr) = self.refr else { return ([0, 0], 0.0) }; + let mi = |rr: usize, cc: usize| self.mvs[rr * g.mi_cols + cc]; + let mut pred_mv = [0, 0]; + let mut cands: Vec<[i32; 2]> = vec![[0, 0]]; + if c > 0 { + pred_mv = mi(r, c - 1); + cands.push(pred_mv); + } + if r > 0 { + cands.push(mi(r - 1, c)); + let cr = c + n / 4; + if cr < g.mi_cols { + cands.push(mi(r - 1, cr)); + } + } + if let Some(p) = parent_mv { + cands.push(p); + } + cands.push(refr.mvs[r * g.mi_cols + c]); + let (x0, y0) = ((c * 4) as i32, (r * 4) as i32); + let (w, h) = (g.width as i32, g.height as i32); + // keep the referenced block within 64 samples of the picture + let lim_x = (-x0 - n as i32 - 64, w - x0 + 64); + let lim_y = (-y0 - n as i32 - 64, h - y0 + 64); + let valid = |m: [i32; 2]| { + let (dy, dx) = (m[0] >> 3, m[1] >> 3); + dx >= lim_x.0 && dx <= lim_x.1 && dy >= lim_y.0 && dy <= lim_y.1 + }; + let lg = log2_table(); + let mv_bits = |m: [i32; 2]| -> f64 { + let comp = |d: i32| if d == 0 { 0.5 } else { 2.0 + 2.0 * log2_of(lg, (d.abs() / 8) as usize + 1) }; + comp(m[0] - pred_mv[0]) + comp(m[1] - pred_mv[1]) + }; + let cost = |m: [i32; 2]| -> f64 { + self.sad_ref(&refr.planes[0], x0 + (m[1] >> 3), y0 + (m[0] >> 3), c * 4, r * 4, n) as f64 + self.lambda_sad * mv_bits(m) + }; + let mut best = ([0, 0], f64::MAX); + for &m in &cands { + let m = [m[0] & !7, m[1] & !7]; + if !valid(m) { + continue; + } + let cst = cost(m); + if cst < best.1 { + best = (m, cst); + } + } + let mut step = 8 * if n >= 32 { + 16 + } else if n == 16 { + 8 + } else { + 4 + }; + while step >= 8 { + let mut moved = true; + let mut iters = 0; + while moved && iters < 16 { + moved = false; + iters += 1; + let center = best.0; + for d in [[-step, 0], [step, 0], [0, -step], [0, step]] { + let m = [center[0] + d[0], center[1] + d[1]]; + if !valid(m) { + continue; + } + let cst = cost(m); + if cst < best.1 { + best = (m, cst); + moved = true; + } + } + } + step >>= 1; + } + // half- then quarter-sample refinement (plus the exact neighbour candidates) + let mut buf = vec![0u16; n * n]; + let sub_cost = |m: [i32; 2], buf: &mut Vec| -> f64 { + self.search_block(m, c * 4, r * 4, n, buf); + self.sad(0, c * 4, r * 4, n, buf) as f64 + self.lambda_sad * mv_bits(m) + }; + for m in cands { + if m[0] & 7 != 0 || m[1] & 7 != 0 { + let m = [m[0] & !1, m[1] & !1]; + if valid(m) { + let cst = sub_cost(m, &mut buf); + if cst < best.1 { + best = (m, cst); + } + } + } + } + for step in [4, 2] { + let center = best.0; + for d in [[-1, -1], [-1, 0], [-1, 1], [0, -1], [0, 1], [1, -1], [1, 0], [1, 1]] { + let m = [center[0] + d[0] * step, center[1] + d[1] * step]; + if !valid(m) { + continue; + } + let cst = sub_cost(m, &mut buf); + if cst < best.1 { + best = (m, cst); + } + } + } + (best.0, mv_bits(best.0)) + } + + /// Approximate luma prediction for motion search: exact at whole and half-sample + /// positions (precomputed planes), averaged neighbours at quarter-sample positions. + fn search_block(&self, mv: [i32; 2], x: usize, y: usize, n: usize, out: &mut [u16]) { + let Some(refr) = self.refr else { return }; + let (w, h) = (self.g.width as i32, self.g.height as i32); + // per axis: up to two (integer offset, half) taps + let axis = |v: i32| -> ([(i32, bool); 2], usize) { + let p = 2 * v; // 1/16 units + let (i, f) = (p >> 4, p & 15); + match f { + 0 => ([(i, false), (i, false)], 1), + 8 => ([(i, true), (i, true)], 1), + 4 => ([(i, false), (i, true)], 2), + _ => ([(i, true), (i + 1, false)], 2), + } + }; + let (ax, nx) = axis(mv[1]); + let (ay, ny) = axis(mv[0]); + let taps = nx.max(ny); + let fetch = |k: usize, xx: i32, yy: i32| -> u16 { + let (ox, hx) = ax[k.min(nx - 1)]; + let (oy, hy) = ay[k.min(ny - 1)]; + let sx = (xx + ox).clamp(0, w - 1) as usize; + let sy = (yy + oy).clamp(0, h - 1) as usize; + match (hx, hy) { + (false, false) => refr.planes[0].at(sx, sy), + (true, false) => self.half[0][sy * w as usize + sx], + (false, true) => self.half[1][sy * w as usize + sx], + (true, true) => self.half[2][sy * w as usize + sx], + } + }; + let vw = n.min(self.g.width.saturating_sub(x)); + let vh = n.min(self.g.height.saturating_sub(y)); + // fast path: every tap inside the picture + let (x0, y0) = (x as i32, y as i32); + let inside = (0..taps).all(|k| { + let (ox, _) = ax[k.min(nx - 1)]; + let (oy, _) = ay[k.min(ny - 1)]; + x0 + ox >= 0 && y0 + oy >= 0 && x0 + ox + vw as i32 <= w && y0 + oy + vh as i32 <= h + }); + if inside { + let src = |k: usize| -> (&[u16], usize, usize) { + let (ox, hx) = ax[k.min(nx - 1)]; + let (oy, hy) = ay[k.min(ny - 1)]; + let (sx, sy) = ((x0 + ox) as usize, (y0 + oy) as usize); + match (hx, hy) { + (false, false) => (&refr.planes[0].data, refr.planes[0].stride, sy * refr.planes[0].stride + sx), + (true, false) => (&self.half[0], w as usize, sy * w as usize + sx), + (false, true) => (&self.half[1], w as usize, sy * w as usize + sx), + (true, true) => (&self.half[2], w as usize, sy * w as usize + sx), + } + }; + let (a, sa, oa) = src(0); + if taps == 1 { + for i in 0..vh { + out[i * n..i * n + vw].copy_from_slice(&a[oa + i * sa..oa + i * sa + vw]); + } + } else { + let (b, sb, ob) = src(1); + for i in 0..vh { + let (ra, rb) = (&a[oa + i * sa..oa + i * sa + vw], &b[ob + i * sb..ob + i * sb + vw]); + for (o, (&p, &q)) in out[i * n..i * n + vw].iter_mut().zip(ra.iter().zip(rb)) { + *o = ((p as u32 + q as u32 + 1) >> 1) as u16; + } + } + } + return; + } + for i in 0..vh { + for j in 0..vw { + let (xx, yy) = (x0 + j as i32, y0 + i as i32); + out[i * n + j] = if taps == 1 { fetch(0, xx, yy) } else { ((fetch(0, xx, yy) as u32 + fetch(1, xx, yy) as u32 + 1) >> 1) as u16 }; + } + } + } + + /// SAD of the source block at (x, y) (luma) against the reference block at (rx, ry). + fn sad_ref(&self, refp: &Plane, rx: i32, ry: i32, x: usize, y: usize, n: usize) -> u64 { + let (vw, vh) = (self.g.width, self.g.height); + let w = n.min(vw.saturating_sub(x)); + let h = n.min(vh.saturating_sub(y)); + let src = &self.src[0]; + let (lx, ly) = (vw as i32 - 1, vh as i32 - 1); + let mut sad = 0u64; + let inside = rx >= 0 && ry >= 0 && rx + w as i32 - 1 <= lx && ry + h as i32 - 1 <= ly; + for i in 0..h { + let s = &src.row(y + i)[x..x + w]; + let rr = refp.row((ry + i as i32).clamp(0, ly) as usize); + if inside { + let rrow = &rr[rx as usize..rx as usize + w]; + sad += s.iter().zip(rrow).map(|(&a, &b)| (a as i32 - b as i32).unsigned_abs()).sum::() as u64; + } else { + for j in 0..w { + let b = rr[(rx + j as i32).clamp(0, lx) as usize]; + sad += (s[j] as i32 - b as i32).unsigned_abs() as u64; + } + } + } + sad + } + + /// Sum of absolute Hadamard-transformed differences (4x4 or 8x8 sub-blocks) of a prediction + /// against the source, scaled to about the size of the SAD. + fn satd(&self, p: usize, x: usize, y: usize, n: usize, pred: &[u16]) -> u64 { + let src = &self.src[p]; + let k = n.min(8); + let mut total = 0u64; + let mut d = [0i32; 64]; + for by in (0..n).step_by(k) { + for bx in (0..n).step_by(k) { + for i in 0..k { + let row = &src.row(y + by + i)[x + bx..x + bx + k]; + for j in 0..k { + d[i * k + j] = row[j] as i32 - pred[(by + i) * n + bx + j] as i32; + } + } + for i in 0..k { + hadamard(&mut d[i * k..i * k + k]); + } + let mut col = [0i32; 8]; + let mut s = 0u64; + for j in 0..k { + for i in 0..k { + col[i] = d[i * k + j]; + } + hadamard(&mut col[..k]); + s += col[..k].iter().map(|v| v.unsigned_abs() as u64).sum::(); + } + total += s / (k as u64 / 2); + } + } + total + } + + /// SAD of a prediction against the source over the visible part of the block. + fn sad(&self, p: usize, x: usize, y: usize, n: usize, pred: &[u16]) -> u64 { + let (vw, vh) = self.g.visible(p); + let w = n.min(vw.saturating_sub(x)); + let h = n.min(vh.saturating_sub(y)); + let src = &self.src[p]; + let mut s = 0u64; + for i in 0..h { + let row = &src.row(y + i)[x..x + w]; + s += row.iter().zip(&pred[i * n..i * n + w]).map(|(&a, &b)| (a as i32 - b as i32).unsigned_abs()).sum::() as u64; + } + s + } + + fn sse(&self, p: usize, x: usize, y: usize, n: usize, blk: &[u16]) -> f64 { + let (vw, vh) = self.g.visible(p); + let w = n.min(vw.saturating_sub(x)); + let h = n.min(vh.saturating_sub(y)); + let src = &self.src[p]; + let mut s = 0u64; + for i in 0..h { + let row = &src.row(y + i)[x..x + w]; + s += row + .iter() + .zip(&blk[i * n..i * n + w]) + .map(|(&a, &b)| { + let d = a as i64 - b as i64; + (d * d) as u64 + }) + .sum::(); + } + s as f64 + } + + fn put(&mut self, p: usize, x: usize, y: usize, n: usize, blk: &[u16]) { + let pl = &mut self.rec[p]; + for i in 0..n { + let o = (y + i) * pl.stride + x; + pl.data[o..o + n].copy_from_slice(&blk[i * n..i * n + n]); + } + } + + /// Transform, quantise and reconstruct one square transform block of `plane` at (x, y) + /// over the prediction `pred`; the reconstruction is written to `rec`. + fn code_tx(&mut self, block: TransformBlock, pred: &[u16]) -> TxResult { + let TransformBlock { plane: p, x, y, log2, .. } = block; + let mut sc = self.scratch.take().unwrap_or_default(); + let (res, has_recon) = self.try_tx(&mut sc, block, pred); + let n = 1usize << log2; + self.put(p, x, y, n, if has_recon { &sc.recon[..n * n] } else { pred }); + let levels = sc.levels[..n * n].to_vec(); + self.scratch = Some(sc); + TxResult { levels, nonzero: res.nonzero, dist: res.dist, bits: res.bits } + } + + /// Transform and quantise one block without writing it. The levels are left in `sc.levels`; + /// the flag tells whether `sc.recon` holds the reconstruction (false: it is the prediction). + fn try_tx(&self, sc: &mut TxScratch, block: TransformBlock, pred: &[u16]) -> (TxEval, bool) { + let TransformBlock { plane: p, x, y, log2, tx_type, intra } = block; + let n = 1usize << log2; + let nn = n * n; + let tx_sz = tx_size_for(log2); + let bd = self.bd; + let src = &self.src[p]; + let TxScratch { res, coef, levels, deq, resid, recon } = sc; + let (res, coef, levels) = (&mut res[..nn], &mut coef[..nn], &mut levels[..nn]); + for i in 0..n { + let row = &src.row(y + i)[x..x + n]; + for j in 0..n { + res[i * n + j] = row[j] as i32 - pred[i * n + j] as i32; + } + } + forward_2d(res, coef, n, tx_type); + // dead-zone quantiser: rounding offset below one half + let rnd = if intra { INTRA_RND } else { INTER_RND }; + let [dc_thr, ac_thr] = [self.dead_zone[0][intra as usize], self.dead_zone[1][intra as usize]]; + levels.fill(0); + let mut nonzero = false; + for (k, (&cf, l)) in coef.iter().zip(levels.iter_mut()).enumerate() { + let (q, thr) = if k == 0 { (self.dc_q, dc_thr) } else { (self.ac_q, ac_thr) }; + // dead zone: below the exact threshold the level is zero, no division needed + if cf.abs() < thr { + continue; + } + // the operand is >= 0, so truncation equals floor + let a = (cf.abs() / q + rnd) as i32; + if a != 0 { + *l = if cf < 0.0 { -a.min(1 << 16) } else { a.min(1 << 16) }; + nonzero = true; + } + } + let zero_dist = self.sse(p, x, y, n, pred); + if !nonzero { + return (TxEval { nonzero, dist: zero_dist, bits: 1.0 }, false); + } + let recon = &mut recon[..nn]; + self.reconstruct(levels, ReconBuffers { deq: &mut deq[..nn], resid: &mut resid[..nn], recon }, tx_sz, tx_type, pred, bd); + let dist = self.sse(p, x, y, n, recon); + let bits = estimate_bits(levels, tx_sz); + if dist + self.lambda * bits >= zero_dist + self.lambda { + levels.fill(0); + return (TxEval { nonzero: false, dist: zero_dist, bits: 1.0 }, false); + } + (TxEval { nonzero: true, dist, bits }, true) + } + + /// Dequantisation (7.12.3) and the inverse transform, added to the prediction; the result is + /// written to `recon`. `deq` and `resid` are scratch buffers of `n * n` entries. + fn reconstruct(&self, levels: &[i32], bufs: ReconBuffers, tx_sz: usize, tx_type: usize, pred: &[u16], bd: u32) { + let ReconBuffers { deq, resid, recon } = bufs; + let denom: i64 = if tx_sz == TX_32X32 { 2 } else { 1 }; + let lim = 1i64 << (7 + bd); + deq.fill(0); + for (k, (&l, d)) in levels.iter().zip(deq.iter_mut()).enumerate() { + if l == 0 { + continue; + } + let q = if k == 0 { self.dc_qi } else { self.ac_qi }; + let dq = l as i64 * q; + let mag = (dq.abs() & 0xFF_FFFF) / denom; + *d = (if dq < 0 { -mag } else { mag }).clamp(-lim, lim - 1) as i32; + } + inverse_2d(deq, resid, tx_sz, tx_type, bd); + let max = (1i32 << bd) - 1; + for ((o, &p), &r) in recon.iter_mut().zip(pred).zip(resid.iter()) { + *o = (p as i32 + r).clamp(0, max) as u16; + } + } +} + +/// Smallest `x >= 0` with `(x / q + rnd) as i32 >= 1` in f32 arithmetic. The expression is +/// monotone in `x`, so a binary search over the bit patterns finds the exact boundary. +fn zero_threshold(q: f32, rnd: f32) -> f32 { + let (mut lo, mut hi) = (0u32, f32::MAX.to_bits()); + while lo < hi { + let mid = lo + (hi - lo) / 2; + if (f32::from_bits(mid) / q + rnd) as i32 >= 1 { + hi = mid; + } else { + lo = mid + 1; + } + } + f32::from_bits(lo) +} + +const LOG2_TABLE_LEN: usize = 1025; + +/// `log2(i)` for `i < LOG2_TABLE_LEN`, computed with the same `f64::log2` call as the direct form. +fn log2_table() -> &'static [f64; LOG2_TABLE_LEN] { + static TABLE: std::sync::OnceLock<[f64; LOG2_TABLE_LEN]> = std::sync::OnceLock::new(); + TABLE.get_or_init(|| std::array::from_fn(|i| (i as f64).log2())) +} + +fn log2_of(table: &[f64; LOG2_TABLE_LEN], v: usize) -> f64 { + if v < LOG2_TABLE_LEN { table[v] } else { (v as f64).log2() } +} + +/// Unnormalised Walsh-Hadamard transform of 4 or 8 values, in place. +fn hadamard(v: &mut [i32]) { + let n = v.len(); + let mut h = 1; + while h < n { + for i in (0..n).step_by(2 * h) { + for j in i..i + h { + let (a, b) = (v[j], v[j + h]); + v[j] = a + b; + v[j + h] = a - b; + } + } + h *= 2; + } +} + +/// Rough coefficient coding cost in bits. +fn estimate_bits(levels: &[i32], tx_sz: usize) -> f64 { + let scan = scan_for(tx_sz); + let mut eob = 0; + for (i, &pos) in scan.iter().enumerate() { + if levels[pos as usize] != 0 { + eob = i + 1; + } + } + if eob == 0 { + return 1.0; + } + let lg = log2_table(); + let mut bits = 3.0 + log2_of(lg, eob); + for &pos in &scan[..eob] { + let a = levels[pos as usize].unsigned_abs(); + bits += match a { + 0 => 0.7, + 1 => 2.5, + _ => 3.5 + 1.6 * log2_of(lg, a as usize), + }; + } + bits +} + +pub(crate) fn scan_for(tx_sz: usize) -> &'static [u16] { + match tx_sz { + TX_4X4 => &DEFAULT_SCAN_4X4, + TX_8X8 => &DEFAULT_SCAN_8X8, + TX_16X16 => &DEFAULT_SCAN_16X16, + _ => &DEFAULT_SCAN_32X32, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tables::{AC_QLOOKUP, DC_QLOOKUP}; + + /// The quantiser as it was before the dead-zone threshold and the truncation. + fn quant_ref(cf: f32, q: f32, rnd: f32) -> i32 { + (cf.abs() / q + rnd).floor() as i32 + } + + fn quant_new(cf: f32, q: f32, rnd: f32, thr: f32) -> i32 { + if cf.abs() < thr { + return 0; + } + (cf.abs() / q + rnd) as i32 + } + + #[test] + fn dead_zone_threshold_is_exact() { + let mut state = 0x1234_5678_9abc_def0u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + for table in AC_QLOOKUP.iter().chain(DC_QLOOKUP.iter()) { + for &qi in table.iter() { + let q = qi as f32; + for rnd in [INTER_RND, INTRA_RND] { + let thr = zero_threshold(q, rnd); + // the boundary itself and its neighbours + let b = thr.to_bits(); + for bits in b.saturating_sub(3)..=b + 3 { + let cf = f32::from_bits(bits); + assert_eq!(quant_ref(cf, q, rnd), quant_new(cf, q, rnd, thr), "q {q} rnd {rnd} cf {cf}"); + assert_eq!(quant_ref(-cf, q, rnd), quant_new(-cf, q, rnd, thr)); + } + for _ in 0..200 { + let cf = f32::from_bits((next() >> 32) as u32 & 0x7fff_ffff); + if cf.is_nan() { + continue; + } + assert_eq!(quant_ref(cf, q, rnd), quant_new(cf, q, rnd, thr)); + let cf = (next() % 100_000) as f32 / 100.0; + assert_eq!(quant_ref(cf, q, rnd), quant_new(cf, q, rnd, thr)); + } + } + } + } + } + + #[test] + fn log2_table_matches_direct_log2() { + let t = log2_table(); + for v in 1..3000usize { + assert_eq!(log2_of(t, v).to_bits(), (v as f64).log2().to_bits(), "{v}"); + } + } +} diff --git a/crates/av1enc/src/lib.rs b/crates/av1enc/src/lib.rs new file mode 100644 index 0000000000..0e384200a3 --- /dev/null +++ b/crates/av1enc/src/lib.rs @@ -0,0 +1,663 @@ +//! Clean-room pure-Rust AV1 encoder (Main profile, 4:2:0, 8 and 10 bit), implemented from the +//! AV1 Bitstream & Decoding Process Specification v1.0.0 with Errata 1. +//! +//! Key frames and single-reference inter frames (LAST_FRAME = the previous frame), 64x64 +//! superblocks split down to 8x8 by rate-distortion decisions, intra modes DC / V / H / Paeth / +//! smooth, full-sample motion search with the spec's motion vector prediction, DCT / ADST +//! transforms up to 32x32, one tile per frame, in-loop filters off. See the crate README. +//! +//! The public API below is the contract `crates/export` codes against; keep it stable. + +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod bitw; +mod cdf; +mod decide; +mod loopfilter; +mod pred; +mod symbol; +mod tables; +mod transform; +mod writer; + +use bitw::{BitWriter, obu}; +use decide::{Decider, FrameGeom, RefFrame}; +use loopfilter::{LfInfo, loop_filter}; +use pred::Plane; +use tables::AC_QLOOKUP; +use writer::TileWriter; + +/// Rate control. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RateControl { + /// Constant quantizer index (`base_q_idx`, 1–255; lower is better). + ConstantQ(u8), + /// Average bitrate target in kbit/s (frame-level quantizer adaptation). + Bitrate { kbps: u32 }, +} + +/// Encoder settings. Main profile (seq_profile 0), 4:2:0. +#[derive(Clone, Debug, PartialEq)] +pub struct EncoderConfig { + pub width: u32, + pub height: u32, + pub fps_num: u32, + pub fps_den: u32, + /// 8 or 10. + pub bit_depth: u8, + /// `seq_level_idx` (e.g. 8 = level 4.0); `None` picks the lowest level that fits. + pub level_idx: Option, + pub rate: RateControl, + /// Frames between key frames (1 = all key frames). + pub keyint: u32, + /// BT.709 limited range is signalled unless this is set. + pub full_range: bool, +} + +impl EncoderConfig { + pub fn new(width: u32, height: u32, fps_num: u32, fps_den: u32) -> Self { + EncoderConfig { + width, + height, + fps_num, + fps_den, + bit_depth: 8, + level_idx: None, + rate: RateControl::ConstantQ(100), + keyint: 60, + full_range: false, + } + } +} + +/// One input picture, 4:2:0, samples at the configured bit depth. +#[derive(Clone, Copy, Debug)] +pub struct Frame<'a> { + pub y: &'a [u16], + pub u: &'a [u16], + pub v: &'a [u16], + pub y_stride: usize, + pub uv_stride: usize, +} + +/// One temporal unit as stored in MP4 / WebM samples: OBUs with `obu_has_size_field` = 1 and +/// no temporal delimiter (key frames start with the sequence header OBU). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Packet { + pub data: Vec, + pub keyframe: bool, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Error { + InvalidConfig(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::InvalidConfig(s) => write!(f, "invalid AV1 encoder config: {s}"), + } + } +} + +impl std::error::Error for Error {} + +const OBU_SEQUENCE_HEADER: u8 = 1; +const OBU_FRAME: u8 = 6; + +/// Annex A level limits: (seq_level_idx, MaxPicSize, MaxHSize, MaxVSize, MaxDisplayRate, +/// MaxBitrate (Main tier, Mbit/s)). +const LEVELS: [(u8, u64, u32, u32, u64, u64); 14] = [ + (0, 147_456, 2048, 1152, 4_423_680, 1_500_000), + (1, 278_784, 2816, 1584, 8_363_520, 3_000_000), + (4, 665_856, 4352, 2448, 19_975_680, 6_000_000), + (5, 1_065_024, 5504, 3096, 31_950_720, 10_000_000), + (8, 2_359_296, 6144, 3456, 70_778_880, 12_000_000), + (9, 2_359_296, 6144, 3456, 141_557_760, 20_000_000), + (12, 8_912_896, 8192, 4352, 267_386_880, 30_000_000), + (13, 8_912_896, 8192, 4352, 534_773_760, 40_000_000), + (14, 8_912_896, 8192, 4352, 1_069_547_520, 60_000_000), + (15, 8_912_896, 8192, 4352, 1_069_547_520, 60_000_000), + (16, 35_651_584, 16384, 8704, 1_069_547_520, 60_000_000), + (17, 35_651_584, 16384, 8704, 2_139_095_040, 100_000_000), + (18, 35_651_584, 16384, 8704, 4_278_190_080, 160_000_000), + (19, 35_651_584, 16384, 8704, 4_278_190_080, 160_000_000), +]; + +/// The lowest level whose limits fit the stream (31 = no level limits when none does). +fn pick_level(cfg: &EncoderConfig) -> u8 { + let pic = cfg.width as u64 * cfg.height as u64; + let rate = pic * cfg.fps_num as u64 / cfg.fps_den.max(1) as u64; + let bitrate = match cfg.rate { + RateControl::Bitrate { kbps } => kbps as u64 * 1000, + RateControl::ConstantQ(_) => 0, + }; + for &(idx, max_pic, max_h, max_v, max_rate, max_br) in LEVELS.iter() { + if pic <= max_pic && cfg.width <= max_h && cfg.height <= max_v && rate <= max_rate && bitrate <= max_br { + return idx; + } + } + 31 +} + +/// Frame-level rate control state for `RateControl::Bitrate`. +struct RateState { + /// Bits per frame at the target rate. + frame_bits: f64, + /// Quantizer index for inter frames (continuous). + q: f64, + /// Key frame quantizer index minus the inter frame one. + key_off: f64, + /// Quantizer index steps per halving of the frame size, measured from consecutive inter + /// frames (the size/quantizer curve is much steeper than the default on noisy content). + slope: f64, + /// Quantizer index and size (in bits) of the previous inter frame. + prev_inter: Option<(f64, f64)>, + /// Accumulated (spent - budget) bits. + debt: f64, +} + +/// Quantizer index steps per halving of the frame size before any measurement. +const RC_SLOPE: f64 = 28.0; +/// Fraction of the quantizer step the size error asks for that is applied per frame. +const RC_GAIN: f64 = 0.8; +/// Frames over which an accumulated deviation from the budget is paid back. +const RC_PAYBACK: f64 = 16.0; + +impl RateState { + /// The nominal bit budget and the starting quantizer index of a key or inter frame. A key + /// frame gets a larger share, and the inter frames of the group get the rest, so that a whole + /// group of `keyint` frames adds up to `keyint * frame_bits`. + fn frame_budget(&self, keyint: u32, key: bool) -> (f64, f64) { + let keyint = keyint.max(1) as f64; + let boost = (1.0 + (keyint.clamp(1.0, 60.0) - 1.0).sqrt() * 0.75).min(0.5 * keyint + 0.5); + if key { + (self.frame_bits * boost, (self.q + self.key_off).clamp(1.0, 255.0)) + } else if keyint > 1.0 { + (self.frame_bits * (keyint - boost) / (keyint - 1.0), self.q) + } else { + (self.frame_bits, self.q) + } + } + + /// Adapts the quantizers to the size of the frame just coded at `q_used` against `target`. + fn update(&mut self, key: bool, first: bool, q_used: f64, bits: f64, target: f64) { + let err = (bits / target).log2().clamp(-1.5, 1.5); + if first && key { + // four passes ran: the key frame quantizer is known, the inter one follows from the offset + self.q = (q_used - self.key_off).clamp(1.0, 255.0); + } else if key { + self.key_off = q_used - self.q + RC_GAIN * self.slope * err; + } else { + if let Some((pq, pbits)) = self.prev_inter { + let (dq, dr) = (q_used - pq, (pbits / bits).log2()); + if dq.abs() >= 3.0 && dr.abs() >= 0.15 && dq * dr > 0.0 { + self.slope = (0.6 * self.slope + 0.4 * (dq / dr)).clamp(3.0, 45.0); + } + } + self.prev_inter = Some((q_used, bits)); + self.q = (q_used + RC_GAIN * self.slope * err).clamp(1.0, 255.0); + } + } +} + +/// Most passes on the first frame, and how far from the target (octaves, 3 %) its size may be +/// before it is accepted. +const RC_FIRST_PASSES: usize = 4; +const RC_FIRST_TOLERANCE: f64 = 0.04; +/// Quantizer index steps per halving of the size, for the first correction of the first frame +/// (the starting quantizer comes from the bits per sample; the first pass shows how far off it +/// is, and from there on the passes are placed by `next_probe_q`). +const RC_FIRST_SLOPE: f64 = 30.0; +/// The same for a key frame after the first: coded again when its size is more than 0.25 octaves +/// (19 %) off the target. +const RC_KEY_PASSES: usize = 3; +const RC_KEY_TOLERANCE: f64 = 0.25; +/// Octaves of frame size per octave of quantizer step assumed for the first correction of a frame. +const RC_PROBE_SLOPE: f64 = 1.0; +/// Largest change of the quantizer step (octaves) between two passes on one frame. +const RC_PROBE_MAX_STEP: f64 = 2.0; + +/// The quantizer index whose step size is closest to `2^x`. +fn q_for_step(bit_depth: u8, x: f64) -> u8 { + let table = &AC_QLOOKUP[((bit_depth - 8) / 2) as usize]; + let mut best = 1; + for q in 1..=255usize { + if ((table[q] as f64).log2() - x).abs() < ((table[best] as f64).log2() - x).abs() { + best = q; + } + } + best as u8 +} + +/// The next quantizer to try for a frame, from the `(quantizer, log2(size / target))` +/// pairs coded so far. The size follows the quantizer step (not the index, which is linear in the +/// step only below about 60) in a near log-log line: false position between the closest pair of +/// passes that enclose the target, otherwise a step from the latest one with the slope of the +/// last two. `None` when no untried quantizer lies between the enclosing passes. +fn next_probe_q(bit_depth: u8, tried: &[(u8, f64)]) -> Option { + let table = &AC_QLOOKUP[((bit_depth - 8) / 2) as usize]; + let x = |q: u8| (table[q as usize] as f64).log2(); + let over = tried.iter().filter(|t| t.1 > 0.0).max_by_key(|t| t.0); + let under = tried.iter().filter(|t| t.1 <= 0.0).min_by_key(|t| t.0); + if let (Some(&(q_lo, e_lo)), Some(&(q_hi, e_hi))) = (over, under) { + if q_hi - q_lo < 2 { + return None; + } + let next = x(q_lo) + (x(q_hi) - x(q_lo)) * e_lo / (e_lo - e_hi); + return Some(q_for_step(bit_depth, next).clamp(q_lo + 1, q_hi - 1)); + } + let &(q, err) = tried.last()?; + let mut slope = RC_PROBE_SLOPE; + if let [.., (q0, e0), (q1, e1)] = tried + && (x(*q1) - x(*q0)).abs() > 0.05 + && (e0 - e1) / (x(*q1) - x(*q0)) > 0.0 + { + slope = ((e0 - e1) / (x(*q1) - x(*q0))).clamp(0.25, 3.0); + } + Some(q_for_step(bit_depth, x(q) + (err / slope).clamp(-RC_PROBE_MAX_STEP, RC_PROBE_MAX_STEP))) +} + +/// AV1 encoder. Frames come out in input order (no hidden/reordered frames). +pub struct Encoder { + cfg: EncoderConfig, + geom: FrameGeom, + level: u8, + seq_obu: Vec, + frame_num: u64, + refr: Option, + rc: Option, +} + +impl Encoder { + pub fn new(cfg: EncoderConfig) -> Result { + if cfg.width == 0 || cfg.height == 0 { + return Err(Error::InvalidConfig("empty frame".into())); + } + if cfg.width > 4096 || cfg.height > 4096 { + return Err(Error::InvalidConfig(format!("{}x{} exceeds the single-tile limit (4096 samples per side)", cfg.width, cfg.height))); + } + if cfg.bit_depth != 8 && cfg.bit_depth != 10 { + return Err(Error::InvalidConfig(format!("bit depth {} (8 or 10 supported)", cfg.bit_depth))); + } + if cfg.fps_num == 0 || cfg.fps_den == 0 { + return Err(Error::InvalidConfig("frame rate".into())); + } + let geom = FrameGeom::new(cfg.width as usize, cfg.height as usize); + if geom.sb_cols * geom.sb_rows > 2304 { + return Err(Error::InvalidConfig(format!("{}x{} exceeds the single-tile area limit (MAX_TILE_AREA)", cfg.width, cfg.height))); + } + if let RateControl::Bitrate { kbps } = cfg.rate + && kbps == 0 + { + return Err(Error::InvalidConfig("zero bitrate".into())); + } + let level = cfg.level_idx.unwrap_or_else(|| pick_level(&cfg)); + if level > 31 { + return Err(Error::InvalidConfig(format!("seq_level_idx {level}"))); + } + let rc = match cfg.rate { + RateControl::ConstantQ(_) => None, + RateControl::Bitrate { kbps } => { + let frame_bits = kbps as f64 * 1000.0 * cfg.fps_den as f64 / cfg.fps_num as f64; + // initial guess from the bits per sample + let bpp = frame_bits / (cfg.width as f64 * cfg.height as f64); + let q = (110.0 - 40.0 * (bpp / 0.1).log2()).clamp(20.0, 230.0); + Some(RateState { frame_bits, q, key_off: -24.0, slope: RC_SLOPE, prev_inter: None, debt: 0.0 }) + } + }; + let mut enc = Encoder { cfg, geom, level, seq_obu: Vec::new(), frame_num: 0, refr: None, rc }; + let payload = enc.sequence_header_payload(); + obu(OBU_SEQUENCE_HEADER, &payload, &mut enc.seq_obu); + Ok(enc) + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// The sequence header OBU (with size field). + pub fn sequence_header_obu(&self) -> Vec { + self.seq_obu.clone() + } + + /// AV1CodecConfigurationRecord (`av1C` payload, also the WebM CodecPrivate). + pub fn av1c(&self) -> Vec { + let high = (self.cfg.bit_depth > 8) as u8; + let mut v = vec![ + 0x81, // marker (1), version (1) + self.level & 0x1f, // seq_profile 0, seq_level_idx_0 + (high << 6) | (1 << 3) | (1 << 2), // tier 0, high_bitdepth, twelve_bit 0, mono 0, ss_x 1, ss_y 1, csp 0 + 0, // no initial_presentation_delay + ]; + v.extend_from_slice(&self.seq_obu); + v + } + + /// sequence_header_obu( ) payload (5.5) including trailing bits. + fn sequence_header_payload(&self) -> Vec { + let c = &self.cfg; + let mut w = BitWriter::new(); + w.f(3, 0); // seq_profile + w.f(1, 0); // still_picture + w.f(1, 0); // reduced_still_picture_header + w.f(1, 0); // timing_info_present_flag + w.f(1, 0); // initial_display_delay_present_flag + w.f(5, 0); // operating_points_cnt_minus_1 + w.f(12, 0); // operating_point_idc[ 0 ] + w.f(5, self.level as u32); // seq_level_idx[ 0 ] + if self.level > 7 { + w.f(1, 0); // seq_tier[ 0 ] + } + let wbits = 32 - (c.width - 1).max(1).leading_zeros(); + let hbits = 32 - (c.height - 1).max(1).leading_zeros(); + w.f(4, wbits - 1); + w.f(4, hbits - 1); + w.f(wbits, c.width - 1); + w.f(hbits, c.height - 1); + w.f(1, 0); // frame_id_numbers_present_flag + w.f(1, 0); // use_128x128_superblock + w.f(1, 0); // enable_filter_intra + w.f(1, 0); // enable_intra_edge_filter + w.f(1, 0); // enable_interintra_compound + w.f(1, 0); // enable_masked_compound + w.f(1, 0); // enable_warped_motion + w.f(1, 0); // enable_dual_filter + w.f(1, 0); // enable_order_hint + w.f(1, 0); // seq_choose_screen_content_tools + w.f(1, 0); // seq_force_screen_content_tools + w.f(1, 0); // enable_superres + w.f(1, 0); // enable_cdef + w.f(1, 0); // enable_restoration + // color_config( ) + w.f(1, (c.bit_depth > 8) as u32); // high_bitdepth + w.f(1, 0); // mono_chrome + w.f(1, 1); // color_description_present_flag + w.f(8, 1); // color_primaries: BT.709 + w.f(8, 1); // transfer_characteristics: BT.709 + w.f(8, 1); // matrix_coefficients: BT.709 + w.f(1, c.full_range as u32); // color_range + w.f(2, 0); // chroma_sample_position: unknown + w.f(1, 0); // separate_uv_delta_q + w.f(1, 0); // film_grain_params_present + w.trailing_bits(); + w.into_bytes() + } + + /// uncompressed_header( ) (5.9) for a shown KEY_FRAME or an INTER_FRAME referencing slot 0. + fn frame_header(&self, w: &mut BitWriter, key: bool, qidx: u8, lf_level: u32) { + w.f(1, 0); // show_existing_frame + w.f(2, if key { 0 } else { 1 }); // frame_type + w.f(1, 1); // show_frame + if !key { + w.f(1, 0); // error_resilient_mode + } + w.f(1, 0); // disable_cdf_update + w.f(1, 0); // frame_size_override_flag + if !key { + w.f(3, 0); // primary_ref_frame: LAST_FRAME's saved CDFs + w.f(8, 1); // refresh_frame_flags: slot 0 becomes LAST for the next frame + for _ in 0..7 { + w.f(3, 0); // ref_frame_idx[ i ] + } + w.f(1, 0); // render_and_frame_size_different + w.f(1, 0); // allow_high_precision_mv + w.f(1, 0); // is_filter_switchable + w.f(2, 0); // interpolation_filter = EIGHTTAP + w.f(1, 0); // is_motion_mode_switchable + } else { + w.f(1, 0); // render_and_frame_size_different + } + w.f(1, 0); // disable_frame_end_update_cdf + // tile_info( ): uniform spacing, one tile + let g = &self.geom; + let tile_log2 = |blk: usize, target: usize| { + let mut k = 0; + while (blk << k) < target { + k += 1; + } + k + }; + w.f(1, 1); // uniform_tile_spacing_flag + if tile_log2(1, g.sb_cols.min(64)) > 0 { + w.f(1, 0); // increment_tile_cols_log2 + } + if tile_log2(1, g.sb_rows.min(64)) > 0 { + w.f(1, 0); // increment_tile_rows_log2 + } + // quantization_params( ) + w.f(8, qidx as u32); + w.f(1, 0); // DeltaQYDc + w.f(1, 0); // DeltaQUDc + w.f(1, 0); // DeltaQUAc + w.f(1, 0); // using_qmatrix + w.f(1, 0); // segmentation_enabled + w.f(1, 0); // delta_q_present (base_q_idx > 0) + // loop_filter_params( ): the same level for every plane and direction + w.f(6, lf_level); + w.f(6, lf_level); + if lf_level != 0 { + w.f(6, lf_level); + w.f(6, lf_level); + } + w.f(3, 0); // loop_filter_sharpness + w.f(1, 0); // loop_filter_delta_enabled + w.f(1, 0); // tx_mode_select (TX_MODE_LARGEST) + if !key { + w.f(1, 0); // reference_select + } + w.f(1, 1); // reduced_tx_set + if !key { + for _ in 0..7 { + w.f(1, 0); // is_global + } + } + } + + /// Copies a picture into superblock-padded planes (edge replication). + fn load_source(&self, frame: &Frame) -> [Plane; 3] { + let maxv = (1u16 << self.cfg.bit_depth) - 1; + std::array::from_fn(|p| { + let (vw, vh) = self.geom.visible(p); + let (pw, ph) = self.geom.padded(p); + let (data, stride) = match p { + 0 => (frame.y, frame.y_stride), + 1 => (frame.u, frame.uv_stride), + _ => (frame.v, frame.uv_stride), + }; + assert!(stride >= vw && data.len() >= stride * (vh - 1) + vw, "plane {p} is too small for {vw}x{vh}"); + let mut pl = Plane::new(pw, ph); + for y in 0..ph { + let sy = y.min(vh - 1); + let src = &data[sy * stride..sy * stride + vw]; + let row = &mut pl.data[y * pw..y * pw + pw]; + for (x, d) in row.iter_mut().enumerate() { + *d = src[x.min(vw - 1)].min(maxv); + } + } + pl + }) + } + + /// Codes one frame at `qidx`; returns the frame OBU and the reconstruction. + fn encode_frame(&self, src: &[Plane; 3], key: bool, qidx: u8) -> (Vec, RefFrame) { + let g = &self.geom; + let refr = if key { None } else { self.refr.as_ref() }; + let mut dec = Decider::new(g, self.cfg.bit_depth as u32, qidx, key, src, refr); + let mut tw = TileWriter::new(g, key, qidx, refr.map(|r| &*r.cdfs)); + for sbr in 0..g.sb_rows { + tw.start_sb_row(); + for sbc in 0..g.sb_cols { + let node = dec.superblock(sbr * 16, sbc * 16); + tw.superblock(sbr * 16, sbc * 16, &node); + } + } + let lf_level = self.pick_filter_level(&dec.rec, &tw.lf_info(), src, qidx); + loop_filter(&mut dec.rec, &tw.lf_info(), [lf_level; 4], self.cfg.bit_depth as u32, false, 1); + let (tile, cdfs) = tw.finish(); + let mut w = BitWriter::new(); + self.frame_header(&mut w, key, qidx, lf_level); + w.byte_align(); + // tile_group_obu( ): a single tile, no tile_start_and_end_present_flag + let mut payload = w.into_bytes(); + payload.extend_from_slice(&tile); + let mut out = Vec::with_capacity(payload.len() + 8); + obu(OBU_FRAME, &payload, &mut out); + let rf = RefFrame { planes: dec.rec, mvs: dec.mvs, lf_level, cdfs }; + (out, rf) + } + + /// The loop filter level minimising the luma error (searched around a guess from the + /// quantizer, on the luma of a subset of the superblock rows). + fn pick_filter_level(&self, rec: &[Plane; 3], info: &LfInfo, src: &[Plane; 3], qidx: u8) -> u32 { + let (w, h) = (self.geom.width, self.geom.height); + let sb_row_step = if self.geom.sb_rows >= 9 { 3 } else { 1 }; + let sse = |pl: &Plane| -> u64 { + let mut s = 0u64; + for y in 0..h { + if !(y >> 6).is_multiple_of(sb_row_step) { + continue; + } + for (a, b) in pl.row(y)[..w].iter().zip(&src[0].row(y)[..w]) { + let d = *a as i64 - *b as i64; + s += (d * d) as u64; + } + } + s + }; + let bd = self.cfg.bit_depth as u32; + let mut cache: Vec<(u32, u64)> = vec![(0, sse(&rec[0]))]; + // one working copy of the luma plane for all candidate levels + let mut planes = [rec[0].clone(), Plane::new(0, 0), Plane::new(0, 0)]; + let mut eval = |lvl: u32| -> u64 { + if let Some(&(_, e)) = cache.iter().find(|(l, _)| *l == lvl) { + return e; + } + planes[0].data.copy_from_slice(&rec[0].data); + loop_filter(&mut planes, info, [lvl; 4], bd, true, sb_row_step); + let e = sse(&planes[0]); + cache.push((lvl, e)); + e + }; + let guess = ((qidx as u32 * 3) / 16).clamp(2, 50); + let mut best = (0u32, eval(0)); + let mut step = (guess / 2).max(1); + let mut center = guess; + let e = eval(center); + if e < best.1 { + best = (center, e); + } + while step >= 2 { + for lvl in [center.saturating_sub(step), (center + step).min(63)] { + let e = eval(lvl); + if e < best.1 { + best = (lvl, e); + } + } + center = best.0.max(1); + step /= 2; + } + best.0 + } + + /// Encodes one frame. + pub fn encode(&mut self, frame: &Frame) -> Packet { + let key = self.frame_num.is_multiple_of(self.cfg.keyint.max(1) as u64); + let src = self.load_source(frame); + let (frame_obu, rf) = match self.cfg.rate { + RateControl::ConstantQ(q) => self.encode_frame(&src, key, q.max(1)), + RateControl::Bitrate { .. } => match self.rc.take() { + Some(mut rc) => { + let out = self.encode_rate_controlled(&src, key, &mut rc); + self.rc = Some(rc); + out + } + // The state is created with the encoder for bitrate mode; without it, encode at a + // middle quantizer rather than fail. + None => self.encode_frame(&src, key, 128), + }, + }; + self.refr = Some(rf); + self.frame_num += 1; + let mut data = Vec::with_capacity(frame_obu.len() + self.seq_obu.len()); + if key { + data.extend_from_slice(&self.seq_obu); + } + data.extend_from_slice(&frame_obu); + Packet { data, keyframe: key } + } + + fn encode_rate_controlled(&mut self, src: &[Plane; 3], key: bool, rc: &mut RateState) -> (Vec, RefFrame) { + let first = self.frame_num == 0; + let (budget, q_start) = rc.frame_budget(self.cfg.keyint, key); + // pay back the accumulated deviation over the next RC_PAYBACK frames + let target = (budget - rc.debt / RC_PAYBACK).clamp(budget * 0.35, budget * 2.0); + let mut q = q_start; + let mut best: Option<(Vec, RefFrame, f64, f64)> = None; + // The first frame is coded up to four times to find its quantizer (it fixes the + // quantizers of everything after it), a later key frame up to three times (it is a large + // part of the budget of its group). A frame is coded again only while its size is far from + // the target and an untried quantizer is left to try; other frames are coded once. + let (max_passes, tolerance) = if first { + (RC_FIRST_PASSES, RC_FIRST_TOLERANCE) + } else if key { + (RC_KEY_PASSES, RC_KEY_TOLERANCE) + } else { + (1, 0.0) + }; + let mut tried: Vec<(u8, f64)> = Vec::new(); + for _ in 0..max_passes { + let qi = q.round().clamp(1.0, 255.0) as u8; + if tried.iter().any(|t| t.0 == qi) { + break; + } + let (obu, rf) = self.encode_frame(src, key, qi); + let bits = obu.len() as f64 * 8.0; + let err = (bits / target).log2(); + tried.push((qi, err)); + if best.as_ref().is_none_or(|b| err.abs() < (b.2 / target).log2().abs()) { + best = Some((obu, rf, bits, qi as f64)); + } + if err.abs() <= tolerance { + break; + } + let next = if first && tried.len() == 1 { + Some((q + RC_FIRST_SLOPE * err).clamp(1.0, 255.0)) + } else { + next_probe_q(self.cfg.bit_depth, &tried).map(f64::from) + }; + match next { + Some(next) => q = next, + None => break, + } + } + // At least one pass ran, so `best` is set. + let Some((obu, rf, bits, q_used)) = best else { return self.encode_frame(src, key, q.round().clamp(1.0, 255.0) as u8) }; + rc.debt += bits - budget; + rc.update(key, first, q_used, bits, target); + (obu, rf) + } + + /// The loop filter level of the last encoded frame. For tests. + #[doc(hidden)] + pub fn last_loop_filter_level(&self) -> Option { + self.refr.as_ref().map(|r| r.lf_level) + } + + /// The reconstruction of the last encoded frame (visible area, planes Y, U, V), equal to + /// what a decoder outputs. For tests. + #[doc(hidden)] + pub fn last_reconstruction(&self) -> Option<[Vec; 3]> { + let rf = self.refr.as_ref()?; + Some(std::array::from_fn(|p| { + let (vw, vh) = self.geom.visible(p); + let pl = &rf.planes[p]; + let mut v = Vec::with_capacity(vw * vh); + for y in 0..vh { + v.extend_from_slice(&pl.row(y)[..vw]); + } + v + })) + } +} diff --git a/crates/av1enc/src/loopfilter.rs b/crates/av1enc/src/loopfilter.rs new file mode 100644 index 0000000000..37365fa0ae --- /dev/null +++ b/crates/av1enc/src/loopfilter.rs @@ -0,0 +1,217 @@ +//! Deblocking loop filter (spec 7.14), mirrored from the decoder process so that the encoder's +//! references equal a decoder's. Segmentation and loop filter deltas are off and the sharpness +//! is 0 in the frames this encoder writes. + +use crate::pred::Plane; +use crate::tables::{MAX_LOOP_FILTER, NUM_4X4_BLOCKS_HIGH, NUM_4X4_BLOCKS_WIDE, SUBSAMPLED_SIZE, TX_HEIGHT, TX_WIDTH}; + +/// Mode info the loop filter reads (per 4x4 luma unit, row stride `mi_cols`), and the +/// transform size per 4x4 unit of each plane (`LoopfilterTxSizes`, row stride `mi_cols + 32`). +pub(crate) struct LfInfo<'a> { + pub mi_cols: usize, + pub mi_rows: usize, + pub width: usize, + pub height: usize, + pub mi_size: &'a [u8], + pub skip: &'a [bool], + pub is_inter: &'a [bool], + pub tx: [&'a [u8]; 3], +} + +/// Applies the loop filter with `loop_filter_level[ 0..4 ]` to `planes`. For level search +/// (`luma_only`), only plane 0 and only every `sb_row_step`-th superblock row are filtered. +pub(crate) fn loop_filter(planes: &mut [Plane; 3], info: &LfInfo, level: [u32; 4], bit_depth: u32, luma_only: bool, sb_row_step: usize) { + if level[0] == 0 && level[1] == 0 { + return; + } + let nplanes = if luma_only { 1 } else { 3 }; + for (plane, pl) in planes.iter_mut().enumerate().take(nplanes) { + if plane > 0 && level[1 + plane] == 0 { + continue; + } + for pass in 0..2 { + let step = if plane == 0 { 1 } else { 2 }; + let mut row = 0; + while row < info.mi_rows { + if !(row >> 4).is_multiple_of(sb_row_step) { + row += step; + continue; + } + let mut col = 0; + while col < info.mi_cols { + edge(pl, info, level, bit_depth, plane, pass, (row, col)); + col += step; + } + row += step; + } + } + } +} + +fn edge(pl: &mut Plane, info: &LfInfo, level: [u32; 4], bd: u32, plane: usize, pass: usize, position: (usize, usize)) { + let (row, col) = position; + let s = (plane > 0) as usize; + let (dx, dy) = if pass == 0 { (1usize, 0usize) } else { (0, 1) }; + let x = col * 4; + let y = row * 4; + let row = row | s; + let col = col | s; + let on_screen = !(x >= info.width || y >= info.height || (pass == 0 && x == 0) || (pass == 1 && y == 0)); + if !on_screen { + return; + } + let xp = x >> s; + let yp = y >> s; + let prev_row = row - (dy << s); + let prev_col = col - (dx << s); + let i = row * info.mi_cols + col; + let mi_size = info.mi_size[i] as usize; + let tx_stride = info.mi_cols + 32; + let tx_sz = info.tx[plane][(row >> s) * tx_stride + (col >> s)] as usize; + let plane_size = SUBSAMPLED_SIZE[mi_size][s][s] as usize; + let skip = info.skip[i]; + let is_intra = !info.is_inter[i]; + let prev_tx_sz = info.tx[plane][(prev_row >> s) * tx_stride + (prev_col >> s)] as usize; + let is_block_edge = if pass == 0 { + xp.is_multiple_of(4 * NUM_4X4_BLOCKS_WIDE[plane_size] as usize) + } else { + yp.is_multiple_of(4 * NUM_4X4_BLOCKS_HIGH[plane_size] as usize) + }; + let is_tx_edge = if pass == 0 { xp.is_multiple_of(TX_WIDTH[tx_sz] as usize) } else { yp.is_multiple_of(TX_HEIGHT[tx_sz] as usize) }; + let apply_filter = is_tx_edge && (is_block_edge || !skip || is_intra); + let base_size = if pass == 0 { TX_WIDTH[prev_tx_sz].min(TX_WIDTH[tx_sz]) } else { TX_HEIGHT[prev_tx_sz].min(TX_HEIGHT[tx_sz]) } as usize; + let filter_size = if plane == 0 { 16.min(base_size) } else { 8.min(base_size) }; + // 7.14.4 with no deltas, segmentation or sharpness: the same strength for every block + let lvl = level[if plane == 0 { pass } else { plane + 1 }].min(MAX_LOOP_FILTER as u32) as i32; + if !apply_filter || lvl == 0 { + return; + } + let limit = lvl.max(1); + let blimit = 2 * (lvl + 2) + limit; + let thresh = lvl >> 4; + let strength = FilterStrength { limit, blimit, thresh }; + for k in 0..4 { + sample_filter(pl, (xp + dy * k, yp + dx * k), plane, &strength, (dx as isize, dy as isize), filter_size, bd); + } +} + +/// Thresholds derived from the loop-filter level (spec 7.14.4). +struct FilterStrength { + limit: i32, + blimit: i32, + thresh: i32, +} + +#[inline] +fn sample_filter( + pl: &mut Plane, + position: (usize, usize), + plane: usize, + strength: &FilterStrength, + direction: (isize, isize), + filter_size: usize, + bd: u32, +) { + let (x, y) = position; + let (dx, dy) = direction; + let FilterStrength { limit, blimit, thresh } = *strength; + let stride = pl.stride as isize; + let step = dy * stride + dx; + let base = y as isize * stride + x as isize; + let reach: isize = if filter_size >= 16 { 7 } else { 4 }; + let mut v = [0i32; 16]; + for k in -reach..reach { + v[(k + 8) as usize] = pl.data[(base + k * step) as usize] as i32; + } + let at = |k: isize| -> i32 { v[(k + 8) as usize] }; + let (q0, q1, q2, q3) = (at(0), at(1), at(2), at(3)); + let (p0, p1, p2, p3) = (at(-1), at(-2), at(-3), at(-4)); + let d = &mut pl.data; + let sh = bd - 8; + let thresh_bd = thresh << sh; + let hev = (p1 - p0).abs() > thresh_bd || (q1 - q0).abs() > thresh_bd; + let filter_len = if filter_size == 4 { + 4 + } else if plane != 0 { + 6 + } else if filter_size == 8 { + 8 + } else { + 16 + }; + let limit_bd = limit << sh; + let blimit_bd = blimit << sh; + let mut mask = (p1 - p0).abs() > limit_bd || (q1 - q0).abs() > limit_bd || (p0 - q0).abs() * 2 + (p1 - q1).abs() / 2 > blimit_bd; + if filter_len >= 6 { + mask |= (p2 - p1).abs() > limit_bd || (q2 - q1).abs() > limit_bd; + } + if filter_len >= 8 { + mask |= (p3 - p2).abs() > limit_bd || (q3 - q2).abs() > limit_bd; + } + if mask { + return; + } + let t_bd = 1 << sh; + let flat = if filter_size >= 8 { + let mut m = (p1 - p0).abs() > t_bd || (q1 - q0).abs() > t_bd || (p2 - p0).abs() > t_bd || (q2 - q0).abs() > t_bd; + if filter_len >= 8 { + m |= (p3 - p0).abs() > t_bd || (q3 - q0).abs() > t_bd; + } + !m + } else { + false + }; + let flat2 = if filter_size >= 16 { + let (q4, q5, q6, p4, p5, p6) = (at(4), at(5), at(6), at(-5), at(-6), at(-7)); + !((p6 - p0).abs() > t_bd + || (q6 - q0).abs() > t_bd + || (p5 - p0).abs() > t_bd + || (q5 - q0).abs() > t_bd + || (p4 - p0).abs() > t_bd + || (q4 - q0).abs() > t_bd) + } else { + false + }; + if filter_size == 4 || !flat { + // narrow filter (7.14.6.3) + let half = 1i32 << (bd - 1); + let c = |v: i32| v.clamp(-half, half - 1); + let off = 0x80 << sh; + let (ps1, ps0, qs0, qs1) = (p1 - off, p0 - off, q0 - off, q1 - off); + let mut filter = if hev { c(ps1 - qs1) } else { 0 }; + filter = c(filter + 3 * (qs0 - ps0)); + let filter1 = c(filter + 4) >> 3; + let filter2 = c(filter + 3) >> 3; + d[base as usize] = (c(qs0 - filter1) + off) as u16; + d[(base - step) as usize] = (c(ps0 + filter2) + off) as u16; + if !hev { + let f = (filter1 + 1) >> 1; + d[(base + step) as usize] = (c(qs1 - f) + off) as u16; + d[(base - 2 * step) as usize] = (c(ps1 + f) + off) as u16; + } + } else { + // wide filter (7.14.6.4) + let log2 = if filter_size == 8 || !flat2 { 3 } else { 4 }; + let n: isize = if log2 == 4 { + 6 + } else if plane == 0 { + 3 + } else { + 2 + }; + let n2: isize = if log2 == 3 && plane == 0 { 0 } else { 1 }; + let mut f = [0i32; 12]; + for i in -n..n { + let mut t = 0; + for j in -n..=n { + let p = (i + j).clamp(-(n + 1), n); + let tap = if j.abs() <= n2 { 2 } else { 1 }; + t += at(p) * tap; + } + f[(i + n) as usize] = (t + (1 << (log2 - 1))) >> log2; + } + for i in -n..n { + d[(base + i * step) as usize] = f[(i + n) as usize] as u16; + } + } +} diff --git a/crates/av1enc/src/pred.rs b/crates/av1enc/src/pred.rs new file mode 100644 index 0000000000..7384a1d39e --- /dev/null +++ b/crates/av1enc/src/pred.rs @@ -0,0 +1,454 @@ +//! Prediction processes, mirrored from the spec (7.11.2 intra prediction for the non-directional +//! modes plus V_PRED / H_PRED without edge filtering, 7.11.3 block inter prediction without +//! reference scaling) so that the encoder predicts exactly like a decoder. + +use crate::tables::{ + DC_PRED, EIGHTTAP, H_PRED, PAETH_PRED, SM_WEIGHTS_TX_4X4, SM_WEIGHTS_TX_8X8, SM_WEIGHTS_TX_16X16, SM_WEIGHTS_TX_32X32, SMOOTH_H_PRED, + SMOOTH_PRED, SMOOTH_V_PRED, SUBPEL_FILTERS, V_PRED, +}; +use crate::transform::round2; + +/// A plane buffer (padded to whole superblocks). +#[derive(Clone)] +pub(crate) struct Plane { + pub data: Vec, + pub stride: usize, +} + +impl Plane { + pub fn new(w: usize, h: usize) -> Plane { + Plane { data: vec![0; w * h], stride: w } + } + #[inline(always)] + pub fn at(&self, x: usize, y: usize) -> u16 { + self.data[y * self.stride + x] + } + #[inline(always)] + pub fn row(&self, y: usize) -> &[u16] { + &self.data[y * self.stride..(y + 1) * self.stride] + } +} + +/// The intra modes this encoder uses (values are the spec's y / uv mode numbers). +pub(crate) const INTRA_MODES_USED: [usize; 7] = [DC_PRED, V_PRED, H_PRED, PAETH_PRED, SMOOTH_PRED, SMOOTH_V_PRED, SMOOTH_H_PRED]; + +/// Inputs of the intra prediction process. +pub(crate) struct IntraEdge { + pub x: usize, + pub y: usize, + pub have_left: bool, + pub have_above: bool, + pub have_above_right: bool, + pub have_below_left: bool, + pub log2: u32, + /// ((MiCols * MI_SIZE) >> subX) - 1 and the same for rows. + pub max_x: i32, + pub max_y: i32, + pub bit_depth: u32, +} + +/// Edge samples: `above[0..2n]`, `left[0..2n]` and the top-left corner. +pub(crate) struct Edges { + above: [i32; 64], + left: [i32; 64], + corner: i32, +} + +pub(crate) fn edges(pl: &Plane, e: &IntraEdge) -> Edges { + let w = 1usize << e.log2; + let h = w; + let bd = e.bit_depth; + let (x, y) = (e.x, e.y); + let n = w + h; + let mut above = [0i32; 64]; + let mut left = [0i32; 64]; + if !e.have_above && e.have_left { + let v = pl.at(x - 1, y) as i32; + above[..n].iter_mut().for_each(|a| *a = v); + } else if !e.have_above && !e.have_left { + let v = (1 << (bd - 1)) - 1; + above[..n].iter_mut().for_each(|a| *a = v); + } else { + let above_limit = e.max_x.min(x as i32 + if e.have_above_right { 2 * w as i32 } else { w as i32 } - 1); + let row = pl.row(y - 1); + for (i, a) in above[..n].iter_mut().enumerate() { + *a = row[above_limit.min(x as i32 + i as i32) as usize] as i32; + } + } + if !e.have_left && e.have_above { + let v = pl.at(x, y - 1) as i32; + left[..n].iter_mut().for_each(|a| *a = v); + } else if !e.have_left && !e.have_above { + let v = (1 << (bd - 1)) + 1; + left[..n].iter_mut().for_each(|a| *a = v); + } else { + let left_limit = e.max_y.min(y as i32 + if e.have_below_left { 2 * h as i32 } else { h as i32 } - 1); + for (i, l) in left[..n].iter_mut().enumerate() { + *l = pl.at(x - 1, left_limit.min(y as i32 + i as i32) as usize) as i32; + } + } + let corner = if e.have_above && e.have_left { + pl.at(x - 1, y - 1) as i32 + } else if e.have_above { + pl.at(x, y - 1) as i32 + } else if e.have_left { + pl.at(x - 1, y) as i32 + } else { + 1 << (bd - 1) + }; + Edges { above, left, corner } +} + +fn sm_weights(log2: u32) -> &'static [u8] { + match log2 { + 2 => &SM_WEIGHTS_TX_4X4, + 3 => &SM_WEIGHTS_TX_8X8, + 4 => &SM_WEIGHTS_TX_16X16, + _ => &SM_WEIGHTS_TX_32X32, + } +} + +/// Predicts a square `2^log2` block with `mode` into `pred` (row stride = block width). +pub(crate) fn predict_intra(ed: &Edges, e: &IntraEdge, mode: usize, pred: &mut [u16]) { + let log2 = e.log2; + let w = 1usize << log2; + let h = w; + let bd = e.bit_depth; + let (above, left) = (&ed.above, &ed.left); + match mode { + DC_PRED => { + let v = if e.have_left && e.have_above { + let sum: i32 = left[..h].iter().sum::() + above[..w].iter().sum::() + ((w + h) >> 1) as i32; + sum / (w + h) as i32 + } else if e.have_left { + (left[..h].iter().sum::() + (h >> 1) as i32) >> log2 + } else if e.have_above { + (above[..w].iter().sum::() + (w >> 1) as i32) >> log2 + } else { + 1 << (bd - 1) + }; + let v = v.clamp(0, (1 << bd) - 1) as u16; + pred[..w * h].iter_mut().for_each(|p| *p = v); + } + V_PRED => { + for i in 0..h { + for j in 0..w { + pred[i * w + j] = above[j] as u16; + } + } + } + H_PRED => { + for i in 0..h { + pred[i * w..i * w + w].iter_mut().for_each(|p| *p = left[i] as u16); + } + } + PAETH_PRED => { + let tl = ed.corner; + for i in 0..h { + for j in 0..w { + let a = above[j]; + let l = left[i]; + let base = a + l - tl; + let p_left = (base - l).abs(); + let p_top = (base - a).abs(); + let p_tl = (base - tl).abs(); + pred[i * w + j] = if p_left <= p_top && p_left <= p_tl { + l as u16 + } else if p_top <= p_tl { + a as u16 + } else { + tl as u16 + }; + } + } + } + _ => { + let wx = sm_weights(log2); + let wy = wx; + for i in 0..h { + for j in 0..w { + let v = if mode == SMOOTH_PRED { + let s = wy[i] as i32 * above[j] + + (256 - wy[i] as i32) * left[h - 1] + + wx[j] as i32 * left[i] + + (256 - wx[j] as i32) * above[w - 1]; + round2(s, 9) + } else if mode == SMOOTH_V_PRED { + round2(wy[i] as i32 * above[j] + (256 - wy[i] as i32) * left[h - 1], 8) + } else { + round2(wx[j] as i32 * left[i] + (256 - wx[j] as i32) * above[w - 1], 8) + }; + pred[i * w + j] = v as u16; + } + } + } + } +} + +/// Block inter prediction (7.11.3.3 motion vector scaling without scaling, 7.11.3.4 block +/// inter prediction with the EIGHTTAP filter, single reference so InterRound1 = 11 and the +/// final rounding is the identity) of the `w x h` block at plane position (x, y) displaced by +/// `mv` (1/8 luma sample units, [row, col]). `last_x` / `last_y` are the largest sample +/// coordinates of the reference plane; the result goes to `pred` (row stride `w`). +pub(crate) struct InterBlock { + pub last_x: i32, + pub last_y: i32, + pub x: usize, + pub y: usize, + pub width: usize, + pub height: usize, + pub mv: [i32; 2], + pub subsampling: usize, + pub bit_depth: u32, +} + +pub(crate) fn predict_inter(refp: &Plane, block: &InterBlock, pred: &mut [u16]) { + let InterBlock { last_x, last_y, x, y, width: w, height: h, mv, subsampling: ss, bit_depth } = *block; + // positions in 1/16 sample units ((startX >> 6) & 15 is the filter phase) + let px = ((x as i32) << 4) + ((2 * mv[1]) >> ss); + let py = ((y as i32) << 4) + ((2 * mv[0]) >> ss); + let (fx, fy) = ((px & 15) as usize, (py & 15) as usize); + let (ix, iy) = (px >> 4, py >> 4); + let max = (1i32 << bit_depth) - 1; + if fx == 0 && fy == 0 { + // the filters' phase-0 kernel is a unit impulse: a clamped copy + for r in 0..h { + let ry = (iy + r as i32).clamp(0, last_y) as usize; + let row = refp.row(ry); + let out = &mut pred[r * w..r * w + w]; + let x0 = ix; + if x0 >= 0 && x0 + w as i32 - 1 <= last_x { + out.copy_from_slice(&row[x0 as usize..x0 as usize + w]); + } else { + for (c, o) in out.iter_mut().enumerate() { + *o = row[(x0 + c as i32).clamp(0, last_x) as usize]; + } + } + } + return; + } + let filt_h = if w <= 4 { 4 } else { EIGHTTAP }; + let filt_v = if h <= 4 { 4 } else { EIGHTTAP }; + let kh = &SUBPEL_FILTERS[filt_h][fx]; + let kv = &SUBPEL_FILTERS[filt_v][fy]; + let bx = ix - 3; + let x_inside = bx >= 0 && bx + w as i32 + 7 <= last_x + 1; + // horizontal pass of one reference row into `out` (Round2( sum, InterRound0 = 3 )) + let hfilter = |ry: usize, out: &mut [i32]| { + let row = refp.row(ry); + if x_inside { + let seg = &row[bx as usize..bx as usize + w + 7]; + for (c, o) in out.iter_mut().enumerate() { + let mut s = 0i32; + for t in 0..8 { + s += kh[t] as i32 * seg[c + t] as i32; + } + *o = round2(s, 3); + } + } else { + for (c, o) in out.iter_mut().enumerate() { + let mut s = 0i32; + for t in 0..8 { + s += kh[t] as i32 * row[(bx + c as i32 + t as i32).clamp(0, last_x) as usize] as i32; + } + *o = round2(s, 3); + } + } + }; + let mut inter = [0i32; (64 + 7) * 64]; + if fy == 0 { + // the vertical kernel is 128 at the centre: Round2( 128 * v, 11 ) = Round2( v, 4 ) + for r in 0..h { + let ry = (iy + r as i32).clamp(0, last_y) as usize; + hfilter(ry, &mut inter[..w]); + for (p, &v) in pred[r * w..r * w + w].iter_mut().zip(&inter[..w]) { + *p = round2(v, 4).clamp(0, max) as u16; + } + } + return; + } + if fx == 0 { + // the horizontal pass is exactly 16 * sample: Round2( 16 * sum, 11 ) = Round2( sum, 7 ) + for r in 0..h { + let rows: [&[u16]; 8] = std::array::from_fn(|t| refp.row((iy + r as i32 + t as i32 - 3).clamp(0, last_y) as usize)); + for c in 0..w { + let xx = if x_inside { (ix + c as i32) as usize } else { (ix + c as i32).clamp(0, last_x) as usize }; + let mut s = 0i32; + for t in 0..8 { + s += kv[t] as i32 * rows[t][xx] as i32; + } + pred[r * w + c] = round2(s, 7).clamp(0, max) as u16; + } + } + return; + } + let inter_h = h + 7; + for r in 0..inter_h { + let ry = (iy + r as i32 - 3).clamp(0, last_y) as usize; + hfilter(ry, &mut inter[r * w..r * w + w]); + } + for r in 0..h { + for c in 0..w { + let mut s = 0i32; + for t in 0..8 { + s += kv[t] as i32 * inter[(r + t) * w + c]; + } + pred[r * w + c] = round2(s, 11).clamp(0, max) as u16; + } + } +} + +/// The visible `w x h` area of a reference plane interpolated at the sub-sample phase +/// (`fx`, `fy`) (1/16 units) with the EIGHTTAP filter, as `predict_inter` computes it for +/// blocks wider and taller than 4. The reference for `half_planes`, which the encoder uses. +#[cfg(test)] +pub(crate) fn subpel_plane(refp: &Plane, w: usize, h: usize, fx: usize, fy: usize, bit_depth: u32) -> Vec { + let kh = &SUBPEL_FILTERS[EIGHTTAP][fx]; + let kv = &SUBPEL_FILTERS[EIGHTTAP][fy]; + let max = (1i32 << bit_depth) - 1; + let (lx, ly) = (w as i32 - 1, h as i32 - 1); + let mut inter = vec![0i32; w * h]; + for y in 0..h { + let row = refp.row(y); + for x in 0..w { + let mut s = 0i32; + for t in 0..8 { + s += kh[t] as i32 * row[(x as i32 + t as i32 - 3).clamp(0, lx) as usize] as i32; + } + inter[y * w + x] = round2(s, 3); + } + } + let mut out = vec![0u16; w * h]; + for y in 0..h { + for x in 0..w { + let mut s = 0i32; + for t in 0..8 { + s += kv[t] as i32 * inter[(y as i32 + t as i32 - 3).clamp(0, ly) as usize * w + x]; + } + out[y * w + x] = round2(s, 11).clamp(0, max) as u16; + } + } + out +} + +/// The three half-sample planes (8, 0), (0, 8), (8, 8) of `subpel_plane` in one pass: the +/// horizontal half-sample pass is shared by (8, 0) and (8, 8), and the phase-0 kernels reduce to +/// exact shifts (Round2( 128 * v, 11 ) = Round2( v, 4 ), Round2( 16 * s, 11 ) = Round2( s, 7 )), +/// so the result equals three `subpel_plane` calls bit for bit. +pub(crate) fn half_planes(refp: &Plane, w: usize, h: usize, bit_depth: u32) -> [Vec; 3] { + let k = &SUBPEL_FILTERS[EIGHTTAP][8]; + let max = (1i32 << bit_depth) - 1; + let (lx, ly) = (w as i32 - 1, h as i32 - 1); + // horizontal pass at phase 8: Round2( sum, 3 ) + let mut hp = vec![0i32; w * h]; + for y in 0..h { + let row = &refp.row(y)[..w]; + let out = &mut hp[y * w..y * w + w]; + let tap = |x: i32| -> i32 { + let mut s = 0i32; + for t in 0..8 { + s += k[t] as i32 * row[(x + t as i32 - 3).clamp(0, lx) as usize] as i32; + } + round2(s, 3) + }; + if w >= 8 { + for x in 0..3 { + out[x] = tap(x as i32); + } + for (x, o) in out.iter_mut().enumerate().take(w - 4).skip(3) { + let seg = &row[x - 3..x + 5]; + let mut s = 0i32; + for t in 0..8 { + s += k[t] as i32 * seg[t] as i32; + } + *o = round2(s, 3); + } + for x in w - 4..w { + out[x] = tap(x as i32); + } + } else { + for (x, o) in out.iter_mut().enumerate() { + *o = tap(x as i32); + } + } + } + let p80: Vec = hp.iter().map(|&v| round2(v, 4).clamp(0, max) as u16).collect(); + let mut p08 = vec![0u16; w * h]; + let mut p88 = vec![0u16; w * h]; + for y in 0..h { + let rows: [usize; 8] = std::array::from_fn(|t| (y as i32 + t as i32 - 3).clamp(0, ly) as usize); + let (o08, o88) = (&mut p08[y * w..y * w + w], &mut p88[y * w..y * w + w]); + for x in 0..w { + let (mut a, mut b) = (0i32, 0i32); + for t in 0..8 { + a += k[t] as i32 * refp.data[rows[t] * refp.stride + x] as i32; + b += k[t] as i32 * hp[rows[t] * w + x]; + } + o08[x] = round2(a, 7).clamp(0, max) as u16; + o88[x] = round2(b, 11).clamp(0, max) as u16; + } + } + [p80, p08, p88] +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Whole-plane interpolation equals block inter prediction at the same phase. + #[test] + fn subpel_plane_matches_block_prediction() { + let (w, h) = (37, 29); + let mut p = Plane::new(64, 64); + for (i, v) in p.data.iter_mut().enumerate() { + *v = ((i * 7919) % 256) as u16; + } + for (fx, fy) in [(8, 0), (0, 8), (8, 8), (4, 12)] { + let plane = subpel_plane(&p, w, h, fx, fy, 8); + for (bx, by) in [(0i32, 0i32), (8, 8), (24, 16), (-5, 20), (30, -3)] { + let mut pred = vec![0u16; 64]; + let mv = [by * 8 + fy as i32 / 2, bx * 8 + fx as i32 / 2]; + predict_inter( + &p, + &InterBlock { last_x: w as i32 - 1, last_y: h as i32 - 1, x: 0, y: 0, width: 8, height: 8, mv, subsampling: 0, bit_depth: 8 }, + &mut pred, + ); + for r in 0..8 { + for c in 0..8 { + let (x, y) = ((bx + c).clamp(0, w as i32 - 1), (by + r).clamp(0, h as i32 - 1)); + if bx + c == x && by + r == y { + assert_eq!( + pred[(r * 8 + c) as usize], + plane[y as usize * w + x as usize], + "phase ({fx}, {fy}) at ({}, {})", + bx + c, + by + r + ); + } + } + } + } + } + } + + #[test] + fn half_planes_match_subpel_plane() { + for (w, h) in [(37usize, 29usize), (5, 3), (1, 1), (64, 64)] { + let mut p = Plane::new(64, 64); + for (i, v) in p.data.iter_mut().enumerate() { + *v = ((i * 7919) % 1024) as u16; + } + let hp = half_planes(&p, w, h, 10); + for (i, (fx, fy)) in [(8, 0), (0, 8), (8, 8)].into_iter().enumerate() { + assert_eq!(hp[i], subpel_plane(&p, w, h, fx, fy, 10), "{w}x{h} phase ({fx}, {fy})"); + } + } + } + + #[test] + fn phase_zero_filters_are_impulses() { + for f in SUBPEL_FILTERS.iter() { + assert_eq!(f[0], [0, 0, 0, 128, 0, 0, 0, 0]); + } + } +} diff --git a/crates/av1enc/src/symbol.rs b/crates/av1enc/src/symbol.rs new file mode 100644 index 0000000000..579a51e208 --- /dev/null +++ b/crates/av1enc/src/symbol.rs @@ -0,0 +1,267 @@ +//! Multi-symbol arithmetic encoder: the exact inverse of the symbol decoder of spec 8.2. +//! +//! The decoder keeps `SymbolValue`, the bitwise complement of a 15-bit window into the tile data +//! minus the lower bounds of the symbols decoded so far, and selects symbol `k` when +//! `cur(k) <= SymbolValue < cur(k - 1)` with +//! `cur(k) = ((SymbolRange >> 8) * ((32768 - cdf[k]) >> 6) >> 1) + 4 * (N - k - 1)` and +//! `cur(-1) = SymbolRange`. The encoder therefore runs an ordinary carry-propagating range +//! coder over the complemented bit stream: coding symbol `k` adds `cur(k)` to `low` and sets the +//! range to `cur(k - 1) - cur(k)`; both are renormalised exactly like the decoder's range. +//! +//! At the end (spec 8.2.4 exit process) the decoder has consumed `S` renormalisation bits plus a +//! 15-bit window, and the bits from position `S` on must be a one followed by zeros. In the +//! complemented domain that window is `0b011_1111_1111_1111`, so the encoder picks the `S`-bit +//! prefix `D` with `D * 2^15 + 2^14 - 1` inside `[low, low + range)`, i.e. +//! `D = (low + 2^14) >> 15`, and writes `!D`, a one bit and zero padding. + +pub(crate) struct SymbolWriter { + /// Low end of the interval, holding `15 + cnt` bits not yet moved to `out` (plus carries). + low: u64, + rng: u32, + cnt: u32, + out: Vec, + pub disable_update: bool, +} + +impl SymbolWriter { + pub fn new(disable_update: bool) -> Self { + SymbolWriter { low: 0, rng: 1 << 15, cnt: 0, out: Vec::with_capacity(1 << 16), disable_update } + } + + /// Adds `k` to the bytes already output (carry propagation). + fn carry(&mut self, mut k: u64) { + let mut i = self.out.len(); + while k != 0 { + i -= 1; + let t = self.out[i] as u64 + k; + self.out[i] = t as u8; + k = t >> 8; + } + } + + #[inline(always)] + fn encode(&mut self, cdf: &[u16], n: usize, s: usize) { + let r8 = self.rng >> 8; + let cur = |k: usize| -> u32 { ((r8 * ((32768 - cdf[k] as u32) >> 6)) >> 1) + 4 * (n - k - 1) as u32 }; + let hi = if s == 0 { self.rng } else { cur(s - 1) }; + let lo = cur(s); + debug_assert!(hi > lo, "symbol {s} has an empty interval"); + self.low += lo as u64; + self.rng = hi - lo; + let d = 15 - (31 - self.rng.leading_zeros()); + if d == 0 { + return; + } + self.rng <<= d; + self.low <<= d; + self.cnt += d; + while self.cnt >= 8 { + let sh = 15 + self.cnt - 8; + let c = self.low >> sh; + self.low &= (1u64 << sh) - 1; + self.cnt -= 8; + self.carry(c >> 8); + self.out.push((c & 0xff) as u8); + } + } + + /// Codes `s` with the adaptive CDF `cdf` (N values plus the adaptation counter). + #[inline] + pub fn symbol(&mut self, s: usize, cdf: &mut [u16]) { + let n = cdf.len() - 1; + debug_assert!(s < n); + self.encode(cdf, n, s); + if !self.disable_update { + update_cdf(cdf, n, s); + } + } + + /// Codes `s` with a CDF that is not adapted. + pub fn symbol_fixed(&mut self, s: usize, cdf: &[u16]) { + let n = cdf.len() - 1; + self.encode(cdf, n, s); + } + + /// `read_bool()` counterpart. + pub fn bool(&mut self, b: bool) { + self.encode(&[1 << 14, 1 << 15, 0], 2, b as usize); + } + + /// `L(n)` counterpart (most significant bit first). + #[cfg(test)] + pub fn literal(&mut self, n: u32, v: u32) { + for i in (0..n).rev() { + self.bool((v >> i) & 1 != 0); + } + } + + /// Finishes the tile: returns its bytes (with the trailing one bit and zero padding). + pub fn finish(mut self) -> Vec { + let low = self.low + (1 << 14); + let mut tail = low >> 15; + self.carry(tail >> self.cnt); + tail &= (1u64 << self.cnt) - 1; + let mut bytes: Vec = self.out.iter().map(|b| !b).collect(); + let n = self.cnt + 1; // 1..=8 bits: !tail then the trailing one bit + let v = (((!tail) & ((1u64 << self.cnt) - 1)) << 1) | 1; + bytes.push((v << (8 - n)) as u8); + bytes + } +} + +/// CDF adaptation (spec 8.2.6, as in the decoder). +#[inline(always)] +pub(crate) fn update_cdf(cdf: &mut [u16], n: usize, symbol: usize) { + let count = cdf[n]; + let rate = 3 + (count > 15) as u32 + (count > 31) as u32 + (31 - (n as u32).leading_zeros()).min(2); + let mut tmp = 0u32; + for i in 0..n - 1 { + if i == symbol { + tmp = 1 << 15; + } + let c = cdf[i] as u32; + if tmp < c { + cdf[i] = (c - ((c - tmp) >> rate)) as u16; + } else { + cdf[i] = (c + ((tmp - c) >> rate)) as u16; + } + } + cdf[n] += (count < 32) as u16; +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The spec's symbol decoder (8.2.2 - 8.2.4), for checking the encoder. + struct Dec<'a> { + data: &'a [u8], + pos: usize, + value: u32, + range: u32, + max_bits: i64, + } + + impl<'a> Dec<'a> { + fn new(data: &'a [u8]) -> Self { + let mut d = Dec { data, pos: 0, value: 0, range: 1 << 15, max_bits: 8 * data.len() as i64 - 15 }; + let nb = (data.len() * 8).min(15) as u32; + let buf = d.bits(nb); + d.value = ((1 << 15) - 1) ^ (buf << (15 - nb)); + d + } + fn bits(&mut self, n: u32) -> u32 { + let mut x = 0; + for _ in 0..n { + let b = if self.pos < self.data.len() * 8 { (self.data[self.pos >> 3] >> (7 - (self.pos & 7))) & 1 } else { 0 }; + self.pos += 1; + x = (x << 1) | b as u32; + } + x + } + fn symbol(&mut self, cdf: &mut [u16], adapt: bool) -> usize { + let n = cdf.len() - 1; + let mut cur = self.range; + let mut s = 0; + let mut prev; + loop { + prev = cur; + let f = (1u32 << 15) - cdf[s] as u32; + cur = (((self.range >> 8) * (f >> 6)) >> 1) + 4 * (n - s - 1) as u32; + if self.value >= cur { + break; + } + s += 1; + } + self.range = prev - cur; + self.value -= cur; + let b = 15 - (31 - self.range.leading_zeros()); + if b > 0 { + self.range <<= b; + let nb = (b as i64).min(self.max_bits.max(0)) as u32; + let nd = self.bits(nb) << (b - nb); + self.value = nd ^ (((self.value + 1) << b) - 1); + self.max_bits -= b as i64; + } + if adapt { + update_cdf(cdf, n, s); + } + s + } + /// Position of the trailing one bit (8.2.4). + fn trailing_position(&self) -> i64 { + let consumed = (self.pos as i64).min(self.data.len() as i64 * 8); + consumed - (15i64).min(self.max_bits + 15) + } + } + + #[test] + fn bools_only() { + for len in 1..200 { + let bits: Vec = (0..len).map(|i| (i * 7 + len) % 3 == 0).collect(); + let mut w = SymbolWriter::new(false); + for &b in &bits { + w.bool(b); + } + let data = w.finish(); + let mut d = Dec::new(&data); + for (i, &b) in bits.iter().enumerate() { + assert_eq!(d.symbol(&mut [1 << 14, 1 << 15, 0], false) == 1, b, "len {len} bit {i} data {data:?}"); + } + } + } + + #[test] + fn round_trip_random_symbols() { + let mut seed = 0x1234_5678u32; + let mut rnd = || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed + }; + for trial in 0..300 { + let count = 1 + (rnd() % 3000) as usize; + let mut cdfs_e = vec![[4096u16, 11264, 19328, 32768, 0]; 4]; + let mut cdfs_d = cdfs_e.clone(); + let mut syms = Vec::new(); + let mut w = SymbolWriter::new(false); + for _ in 0..count { + let kind = rnd() % 3; + let ctx = (rnd() % 4) as usize; + // skewed symbols so that the range also gets small + let s = if rnd() % 4 == 0 { (rnd() % 4) as usize } else { 0 }; + match kind { + 0 => w.symbol(s, &mut cdfs_e[ctx]), + 1 => w.bool(s & 1 == 1), + _ => w.literal(5, s as u32 * 7), + } + syms.push((kind, ctx, s)); + } + let data = w.finish(); + let mut d = Dec::new(&data); + for (idx, &(kind, ctx, s)) in syms.iter().enumerate() { + let got = match kind { + 0 => d.symbol(&mut cdfs_d[ctx], true), + 1 => d.symbol(&mut [1 << 14, 1 << 15, 0], false), + _ => { + let mut v = 0; + for _ in 0..5 { + v = (v << 1) | d.symbol(&mut [1 << 14, 1 << 15, 0], false); + } + v / 7 + } + }; + let want = if kind == 1 { s & 1 } else { s }; + assert_eq!(got, want, "trial {trial} idx {idx} kind {kind} of {}", syms.len()); + } + let tp = d.trailing_position(); + assert!(tp >= 0 && (tp as usize) < data.len() * 8, "trial {trial}: trailing bit position {tp}"); + let tp = tp as usize; + assert_eq!((data[tp >> 3] >> (7 - (tp & 7))) & 1, 1, "trial {trial}: trailing one bit"); + for p in tp + 1..data.len() * 8 { + assert_eq!((data[p >> 3] >> (7 - (p & 7))) & 1, 0, "trial {trial}: padding"); + } + } + } +} diff --git a/crates/av1enc/src/tables.rs b/crates/av1enc/src/tables.rs new file mode 100644 index 0000000000..90616bcebd --- /dev/null +++ b/crates/av1enc/src/tables.rs @@ -0,0 +1,4836 @@ +// Tables mirrored from filmcraft-av1's spec_tables.rs, which extracts them from the AV1 Bitstream & +// Decoding Process Specification v1.0.0 with Errata 1. Only the tables this encoder uses are kept. +#![allow(dead_code, clippy::unreadable_literal)] + +pub const ADST_ADST: usize = 3; +pub const ADST_DCT: usize = 1; +pub const ADST_FLIPADST: usize = 7; +pub const AFFINE: usize = 3; +pub const ALTREF2_FRAME: usize = 6; +pub const ALTREF_FRAME: usize = 7; +pub const ANGLE_STEP: usize = 3; +pub const BIDIR_COMP_REFERENCE: usize = 1; +pub const BILINEAR: usize = 3; +pub const BLOCK_128X128: usize = 15; +pub const BLOCK_128X64: usize = 14; +pub const BLOCK_16X16: usize = 6; +pub const BLOCK_16X32: usize = 7; +pub const BLOCK_16X4: usize = 17; +pub const BLOCK_16X64: usize = 20; +pub const BLOCK_16X8: usize = 5; +pub const BLOCK_32X16: usize = 8; +pub const BLOCK_32X32: usize = 9; +pub const BLOCK_32X64: usize = 10; +pub const BLOCK_32X8: usize = 19; +pub const BLOCK_4X16: usize = 16; +pub const BLOCK_4X4: usize = 0; +pub const BLOCK_4X8: usize = 1; +pub const BLOCK_64X128: usize = 13; +pub const BLOCK_64X16: usize = 21; +pub const BLOCK_64X32: usize = 11; +pub const BLOCK_64X64: usize = 12; +pub const BLOCK_8X16: usize = 4; +pub const BLOCK_8X32: usize = 18; +pub const BLOCK_8X4: usize = 2; +pub const BLOCK_8X8: usize = 3; +pub const BLOCK_INVALID: usize = 22; +pub const BLOCK_SIZES: usize = 22; +pub const BLOCK_SIZE_GROUPS: usize = 4; +pub const BR_CDF_SIZE: usize = 4; +pub const BUFFER_POOL_MAX_SIZE: usize = 10; +pub const BWDREF_FRAME: usize = 5; +pub const BWD_REFS: usize = 3; +pub const CFL_ALPHABET_SIZE: usize = 16; +pub const CFL_ALPHA_CONTEXTS: usize = 6; +pub const CFL_JOINT_SIGNS: usize = 8; +pub const CFL_SIGN_NEG: usize = 1; +pub const CFL_SIGN_POS: usize = 2; +pub const CFL_SIGN_ZERO: usize = 0; +pub const CLASS0_SIZE: usize = 2; +pub const COEFF_BASE_RANGE: usize = 12; +pub const COEFF_CDF_Q_CTXS: usize = 4; +pub const COMPOUND_AVERAGE: usize = 2; +pub const COMPOUND_DIFFWTD: usize = 1; +pub const COMPOUND_DISTANCE: usize = 4; +pub const COMPOUND_IDX_CONTEXTS: usize = 6; +pub const COMPOUND_INTRA: usize = 3; +pub const COMPOUND_MODES: usize = 8; +pub const COMPOUND_MODE_CONTEXTS: usize = 8; +pub const COMPOUND_REFERENCE: usize = 1; +pub const COMPOUND_TYPES: usize = 2; +pub const COMPOUND_WEDGE: usize = 0; +pub const COMP_GROUP_IDX_CONTEXTS: usize = 6; +pub const COMP_INTER_CONTEXTS: usize = 5; +pub const COMP_NEWMV_CTXS: usize = 5; +pub const COMP_REF_TYPE_CONTEXTS: usize = 5; +pub const CP_BT_2020: usize = 9; +pub const CP_BT_470_B_G: usize = 5; +pub const CP_BT_470_M: usize = 4; +pub const CP_BT_601: usize = 6; +pub const CP_BT_709: usize = 1; +pub const CP_EBU_3213: usize = 22; +pub const CP_GENERIC_FILM: usize = 8; +pub const CP_SMPTE_240: usize = 7; +pub const CP_SMPTE_431: usize = 11; +pub const CP_SMPTE_432: usize = 12; +pub const CP_UNSPECIFIED: usize = 2; +pub const CP_XYZ: usize = 10; +pub const CSP_COLOCATED: usize = 2; +pub const CSP_RESERVED: usize = 3; +pub const CSP_UNKNOWN: usize = 0; +pub const CSP_VERTICAL: usize = 1; +pub const D113_PRED: usize = 5; +pub const D135_PRED: usize = 4; +pub const D157_PRED: usize = 6; +pub const D203_PRED: usize = 7; +pub const D45_PRED: usize = 3; +pub const D67_PRED: usize = 8; +pub const DCT_ADST: usize = 2; +pub const DCT_DCT: usize = 0; +pub const DCT_FLIPADST: usize = 5; +pub const DC_PRED: usize = 0; +pub const DC_SIGN_CONTEXTS: usize = 3; +pub const DELTA_LF_SMALL: usize = 3; +pub const DELTA_Q_SMALL: usize = 3; +pub const DIRECTIONAL_MODES: usize = 8; +pub const DIV_LUT_BITS: usize = 8; +pub const DIV_LUT_NUM: usize = 257; +pub const DIV_LUT_PREC_BITS: usize = 14; +pub const DRL_MODE_CONTEXTS: usize = 3; +pub const EC_MIN_PROB: usize = 4; +pub const EC_PROB_SHIFT: usize = 6; +pub const EIGHTTAP: usize = 0; +pub const EIGHTTAP_SHARP: usize = 2; +pub const EIGHTTAP_SMOOTH: usize = 1; +pub const EOB_COEF_CONTEXTS: usize = 9; +pub const FILTER_BITS: usize = 7; +pub const FILTER_D157_PRED: usize = 3; +pub const FILTER_DC_PRED: usize = 0; +pub const FILTER_H_PRED: usize = 2; +pub const FILTER_PAETH_PRED: usize = 4; +pub const FILTER_V_PRED: usize = 1; +pub const FLIPADST_ADST: usize = 8; +pub const FLIPADST_DCT: usize = 4; +pub const FLIPADST_FLIPADST: usize = 6; +pub const FRAME_LF_COUNT: usize = 4; +pub const FWD_REFS: usize = 4; +pub const GLOBALMV: usize = 16; +pub const GLOBAL_GLOBALMV: usize = 24; +pub const GM_ABS_ALPHA_BITS: usize = 12; +pub const GM_ABS_TRANS_BITS: usize = 12; +pub const GM_ABS_TRANS_ONLY_BITS: usize = 9; +pub const GM_ALPHA_PREC_BITS: usize = 15; +pub const GM_TRANS_ONLY_PREC_BITS: usize = 3; +pub const GM_TRANS_PREC_BITS: usize = 6; +pub const GOLDEN_FRAME: usize = 4; +pub const H_ADST: usize = 13; +pub const H_DCT: usize = 11; +pub const H_FLIPADST: usize = 15; +pub const H_PRED: usize = 2; +pub const IDENTITY: usize = 0; +pub const IDTX: usize = 9; +pub const II_DC_PRED: usize = 0; +pub const II_H_PRED: usize = 2; +pub const II_SMOOTH_PRED: usize = 3; +pub const II_V_PRED: usize = 1; +pub const INTERINTRA_MODES: usize = 4; +pub const INTERP_FILTERS: usize = 3; +pub const INTERP_FILTER_CONTEXTS: usize = 16; +pub const INTER_FRAME: usize = 1; +pub const INTRABC_DELAY_PIXELS: usize = 256; +pub const INTRABC_DELAY_SB64: usize = 4; +pub const INTRA_EDGE_KERNELS: usize = 3; +pub const INTRA_EDGE_TAPS: usize = 5; +pub const INTRA_FILTER_MODES: usize = 5; +pub const INTRA_FILTER_SCALE_BITS: usize = 4; +pub const INTRA_FRAME: usize = 0; +pub const INTRA_MODES: usize = 13; +pub const INTRA_MODE_CONTEXTS: usize = 5; +pub const INTRA_ONLY_FRAME: usize = 2; +pub const IS_INTER_CONTEXTS: usize = 4; +pub const KEY_FRAME: usize = 0; +pub const LAST2_FRAME: usize = 2; +pub const LAST3_FRAME: usize = 3; +pub const LAST_FRAME: usize = 1; +pub const LEAST_SQUARES_SAMPLES_MAX: usize = 8; +pub const LEVEL_CONTEXTS: usize = 21; +pub const LOCALWARP: usize = 2; +pub const LS_MV_MAX: usize = 256; +pub const MASK_MASTER_SIZE: usize = 64; +pub const MAX_ANGLE_DELTA: usize = 3; +pub const MAX_FRAME_DISTANCE: usize = 31; +pub const MAX_LOOP_FILTER: usize = 63; +pub const MAX_OFFSET_HEIGHT: usize = 0; +pub const MAX_OFFSET_WIDTH: usize = 8; +pub const MAX_REF_MV_STACK_SIZE: usize = 8; +pub const MAX_SB_SIZE: usize = 128; +pub const MAX_SEGMENTS: usize = 8; +pub const MAX_TILE_AREA: usize = 9437184; +pub const MAX_TILE_COLS: usize = 64; +pub const MAX_TILE_ROWS: usize = 64; +pub const MAX_TILE_WIDTH: usize = 4096; +pub const MAX_TX_DEPTH: usize = 2; +pub const MAX_VARTX_DEPTH: usize = 2; +pub const MB_MODE_COUNT: usize = 17; +pub const MC_BT_2020_CL: usize = 10; +pub const MC_BT_2020_NCL: usize = 9; +pub const MC_BT_470_B_G: usize = 5; +pub const MC_BT_601: usize = 6; +pub const MC_BT_709: usize = 1; +pub const MC_CHROMAT_CL: usize = 13; +pub const MC_CHROMAT_NCL: usize = 12; +pub const MC_FCC: usize = 4; +pub const MC_ICTCP: usize = 14; +pub const MC_IDENTITY: usize = 0; +pub const MC_RESERVED_3: usize = 3; +pub const MC_SMPTE_2085: usize = 11; +pub const MC_SMPTE_240: usize = 7; +pub const MC_SMPTE_YCGCO: usize = 8; +pub const MC_UNSPECIFIED: usize = 2; +pub const METADATA_TYPE_HDR_CLL: usize = 1; +pub const METADATA_TYPE_HDR_MDCV: usize = 2; +pub const METADATA_TYPE_ITUT_T35: usize = 4; +pub const METADATA_TYPE_SCALABILITY: usize = 3; +pub const METADATA_TYPE_TIMECODE: usize = 5; +pub const MFMV_STACK_SIZE: usize = 3; +pub const MI_SIZE: usize = 4; +pub const MI_SIZE_LOG2: usize = 2; +pub const MOTION_MODES: usize = 3; +pub const MV_BORDER: usize = 128; +pub const MV_CLASSES: usize = 11; +pub const MV_CLASS_0: usize = 0; +pub const MV_CLASS_1: usize = 1; +pub const MV_CLASS_10: usize = 10; +pub const MV_CLASS_2: usize = 2; +pub const MV_CLASS_3: usize = 3; +pub const MV_CLASS_4: usize = 4; +pub const MV_CLASS_5: usize = 5; +pub const MV_CLASS_6: usize = 6; +pub const MV_CLASS_7: usize = 7; +pub const MV_CLASS_8: usize = 8; +pub const MV_CLASS_9: usize = 9; +pub const MV_CONTEXTS: usize = 2; +pub const MV_INTRABC_CONTEXT: usize = 1; +pub const MV_JOINTS: usize = 4; +pub const MV_JOINT_HNZVNZ: usize = 3; +pub const MV_JOINT_HNZVZ: usize = 1; +pub const MV_JOINT_HZVNZ: usize = 2; +pub const MV_JOINT_ZERO: usize = 0; +pub const MV_OFFSET_BITS: usize = 10; +pub const NEARESTMV: usize = 14; +pub const NEAREST_NEARESTMV: usize = 18; +pub const NEAREST_NEWMV: usize = 20; +pub const NEARMV: usize = 15; +pub const NEAR_NEARMV: usize = 19; +pub const NEAR_NEWMV: usize = 22; +pub const NEWMV: usize = 17; +pub const NEW_MV_CONTEXTS: usize = 6; +pub const NEW_NEARESTMV: usize = 21; +pub const NEW_NEARMV: usize = 23; +pub const NEW_NEWMV: usize = 25; +pub const NUM_BASE_LEVELS: usize = 2; +pub const NUM_REF_FRAMES: usize = 8; +pub const OBMC: usize = 1; +pub const OBU_FRAME: usize = 6; +pub const OBU_FRAME_HEADER: usize = 3; +pub const OBU_METADATA: usize = 5; +pub const OBU_PADDING: usize = 15; +pub const OBU_REDUNDANT_FRAME_HEADER: usize = 7; +pub const OBU_SEQUENCE_HEADER: usize = 1; +pub const OBU_TEMPORAL_DELIMITER: usize = 2; +pub const OBU_TILE_GROUP: usize = 4; +pub const OBU_TILE_LIST: usize = 8; +pub const ONLY_4X4: usize = 0; +pub const PAETH_PRED: usize = 12; +pub const PALETTE_BLOCK_SIZE_CONTEXTS: usize = 7; +pub const PALETTE_COLORS: usize = 8; +pub const PALETTE_COLOR_CONTEXTS: usize = 5; +pub const PALETTE_MAX_COLOR_CONTEXT_HASH: usize = 8; +pub const PALETTE_NUM_NEIGHBORS: usize = 3; +pub const PALETTE_SIZES: usize = 7; +pub const PALETTE_UV_MODE_CONTEXTS: usize = 2; +pub const PALETTE_Y_MODE_CONTEXTS: usize = 3; +pub const PARTITION_CONTEXTS: usize = 4; +pub const PARTITION_HORZ: usize = 1; +pub const PARTITION_HORZ_4: usize = 8; +pub const PARTITION_HORZ_A: usize = 4; +pub const PARTITION_HORZ_B: usize = 5; +pub const PARTITION_NONE: usize = 0; +pub const PARTITION_SPLIT: usize = 3; +pub const PARTITION_VERT: usize = 2; +pub const PARTITION_VERT_4: usize = 9; +pub const PARTITION_VERT_A: usize = 6; +pub const PARTITION_VERT_B: usize = 7; +pub const PLANE_TYPES: usize = 2; +pub const PRIMARY_REF_NONE: usize = 7; +pub const QM_TOTAL_SIZE: usize = 3344; +pub const REFMVS_LIMIT: usize = 4095; +pub const REFS_PER_FRAME: usize = 7; +pub const REF_CAT_LEVEL: usize = 640; +pub const REF_CONTEXTS: usize = 3; +pub const REF_MV_CONTEXTS: usize = 6; +pub const REF_SCALE_SHIFT: usize = 14; +pub const RESTORATION_TILESIZE_MAX: usize = 256; +pub const RESTORE_NONE: usize = 0; +pub const RESTORE_SGRPROJ: usize = 2; +pub const RESTORE_SWITCHABLE: usize = 3; +pub const RESTORE_WIENER: usize = 1; +pub const ROTZOOM: usize = 2; +pub const SCALABILITY_L1T2: usize = 0; +pub const SCALABILITY_L1T3: usize = 1; +pub const SCALABILITY_L2T1: usize = 2; +pub const SCALABILITY_L2T2: usize = 3; +pub const SCALABILITY_L2T3: usize = 4; +pub const SCALABILITY_L3T1: usize = 15; +pub const SCALABILITY_L3T2: usize = 16; +pub const SCALABILITY_L3T2_KEY: usize = 21; +pub const SCALABILITY_L3T2_KEY_SHIFT: usize = 25; +pub const SCALABILITY_L3T3: usize = 17; +pub const SCALABILITY_L3T3_KEY: usize = 22; +pub const SCALABILITY_L3T3_KEY_SHIFT: usize = 26; +pub const SCALABILITY_L4T5_KEY: usize = 23; +pub const SCALABILITY_L4T5_KEY_SHIFT: usize = 27; +pub const SCALABILITY_L4T7_KEY: usize = 24; +pub const SCALABILITY_L4T7_KEY_SHIFT: usize = 28; +pub const SCALABILITY_S2T1: usize = 5; +pub const SCALABILITY_S2T2: usize = 6; +pub const SCALABILITY_S2T3: usize = 7; +pub const SCALABILITY_S3T1: usize = 18; +pub const SCALABILITY_S3T2: usize = 19; +pub const SCALABILITY_S3T3: usize = 20; +pub const SCALABILITY_SS: usize = 14; +pub const SCALE_SUBPEL_BITS: usize = 10; +pub const SEGMENT_ID_CONTEXTS: usize = 3; +pub const SEGMENT_ID_PREDICTED_CONTEXTS: usize = 3; +pub const SEG_LVL_ALT_LF_Y_V: usize = 1; +pub const SEG_LVL_ALT_Q: usize = 0; +pub const SEG_LVL_GLOBALMV: usize = 7; +pub const SEG_LVL_MAX: usize = 8; +pub const SEG_LVL_REF_FRAME: usize = 5; +pub const SEG_LVL_SKIP: usize = 6; +pub const SELECT_INTEGER_MV: usize = 2; +pub const SELECT_SCREEN_CONTENT_TOOLS: usize = 2; +pub const SGRPROJ_MTABLE_BITS: usize = 20; +pub const SGRPROJ_PARAMS_BITS: usize = 4; +pub const SGRPROJ_PRJ_BITS: usize = 7; +pub const SGRPROJ_PRJ_SUBEXP_K: usize = 4; +pub const SGRPROJ_RECIP_BITS: usize = 12; +pub const SGRPROJ_RST_BITS: usize = 4; +pub const SGRPROJ_SGR_BITS: usize = 8; +pub const SIG_COEF_CONTEXTS: usize = 42; +pub const SIG_COEF_CONTEXTS_2D: usize = 26; +pub const SIG_COEF_CONTEXTS_EOB: usize = 4; +pub const SIG_REF_DIFF_OFFSET_NUM: usize = 5; +pub const SIMPLE: usize = 0; +pub const SINGLE_REFERENCE: usize = 0; +pub const SINGLE_REFS: usize = 7; +pub const SINPI_1_9: usize = 1321; +pub const SINPI_2_9: usize = 2482; +pub const SINPI_3_9: usize = 3344; +pub const SINPI_4_9: usize = 3803; +pub const SKIP_CONTEXTS: usize = 3; +pub const SKIP_MODE_CONTEXTS: usize = 3; +pub const SMOOTH_H_PRED: usize = 11; +pub const SMOOTH_PRED: usize = 9; +pub const SMOOTH_V_PRED: usize = 10; +pub const SUBPEL_BITS: usize = 4; +pub const SUBPEL_MASK: usize = 15; +pub const SUPERRES_DENOM_BITS: usize = 3; +pub const SUPERRES_DENOM_MIN: usize = 9; +pub const SUPERRES_EXTRA_BITS: usize = 8; +pub const SUPERRES_FILTER_BITS: usize = 6; +pub const SUPERRES_FILTER_OFFSET: usize = 3; +pub const SUPERRES_FILTER_SHIFTS: usize = 64; +pub const SUPERRES_FILTER_TAPS: usize = 8; +pub const SUPERRES_NUM: usize = 8; +pub const SUPERRES_SCALE_BITS: usize = 14; +pub const SUPERRES_SCALE_MASK: usize = 16383; +pub const SWITCHABLE: usize = 4; +pub const SWITCH_FRAME: usize = 3; +pub const TC_BT_1361: usize = 12; +pub const TC_BT_2020_10_BIT: usize = 14; +pub const TC_BT_2020_12_BIT: usize = 15; +pub const TC_BT_470_B_G: usize = 5; +pub const TC_BT_470_M: usize = 4; +pub const TC_BT_601: usize = 6; +pub const TC_BT_709: usize = 1; +pub const TC_HLG: usize = 18; +pub const TC_IEC_61966: usize = 11; +pub const TC_LINEAR: usize = 8; +pub const TC_LOG_100: usize = 9; +pub const TC_LOG_100_SQRT10: usize = 10; +pub const TC_RESERVED_0: usize = 0; +pub const TC_RESERVED_3: usize = 3; +pub const TC_SMPTE_2084: usize = 16; +pub const TC_SMPTE_240: usize = 7; +pub const TC_SMPTE_428: usize = 17; +pub const TC_SRGB: usize = 13; +pub const TC_UNSPECIFIED: usize = 2; +pub const TOTAL_REFS_PER_FRAME: usize = 8; +pub const TRANSLATION: usize = 1; +pub const TXB_SKIP_CONTEXTS: usize = 13; +pub const TXFM_PARTITION_CONTEXTS: usize = 21; +pub const TX_16X16: usize = 2; +pub const TX_16X32: usize = 9; +pub const TX_16X4: usize = 14; +pub const TX_16X64: usize = 17; +pub const TX_16X8: usize = 8; +pub const TX_32X16: usize = 10; +pub const TX_32X32: usize = 3; +pub const TX_32X64: usize = 11; +pub const TX_32X8: usize = 16; +pub const TX_4X16: usize = 13; +pub const TX_4X4: usize = 0; +pub const TX_4X8: usize = 5; +pub const TX_64X16: usize = 18; +pub const TX_64X32: usize = 12; +pub const TX_64X64: usize = 4; +pub const TX_8X16: usize = 7; +pub const TX_8X32: usize = 15; +pub const TX_8X4: usize = 6; +pub const TX_8X8: usize = 1; +pub const TX_CLASS_2D: usize = 0; +pub const TX_CLASS_HORIZ: usize = 1; +pub const TX_CLASS_VERT: usize = 2; +pub const TX_MODES: usize = 3; +pub const TX_MODE_LARGEST: usize = 1; +pub const TX_MODE_SELECT: usize = 2; +pub const TX_SET_DCTONLY: usize = 0; +pub const TX_SET_INTER_1: usize = 1; +pub const TX_SET_INTER_2: usize = 2; +pub const TX_SET_INTER_3: usize = 3; +pub const TX_SET_INTRA_1: usize = 1; +pub const TX_SET_INTRA_2: usize = 2; +pub const TX_SET_TYPES_INTER: usize = 4; +pub const TX_SET_TYPES_INTRA: usize = 3; +pub const TX_SIZES: usize = 5; +pub const TX_SIZES_ALL: usize = 19; +pub const TX_SIZE_CONTEXTS: usize = 3; +pub const TX_TYPES: usize = 16; +pub const UNIDIR_COMP_REFERENCE: usize = 0; +pub const UNIDIR_COMP_REFS: usize = 4; +pub const UNIFORM_45: usize = 0; +pub const UNIFORM_45_INV: usize = 1; +pub const UV_CFL_PRED: usize = 13; +pub const UV_INTRA_MODES_CFL_ALLOWED: usize = 14; +pub const UV_INTRA_MODES_CFL_NOT_ALLOWED: usize = 13; +pub const V_ADST: usize = 12; +pub const V_DCT: usize = 10; +pub const V_FLIPADST: usize = 14; +pub const V_PRED: usize = 1; +pub const WARPEDDIFF_PREC_BITS: usize = 10; +pub const WARPEDMODEL_NONDIAGAFFINE_CLAMP: usize = 8192; +pub const WARPEDMODEL_PREC_BITS: usize = 16; +pub const WARPEDMODEL_TRANS_CLAMP: usize = 8388608; +pub const WARPEDPIXEL_PREC_SHIFTS: usize = 64; +pub const WARP_PARAM_REDUCE_BITS: usize = 6; +pub const WEDGE_HORIZONTAL: usize = 0; +pub const WEDGE_OBLIQUE117: usize = 4; +pub const WEDGE_OBLIQUE153: usize = 5; +pub const WEDGE_OBLIQUE27: usize = 2; +pub const WEDGE_OBLIQUE63: usize = 3; +pub const WEDGE_TYPES: usize = 16; +pub const WEDGE_VERTICAL: usize = 1; +pub const WIENER_COEFFS: usize = 3; +pub const ZERO_MV_CONTEXTS: usize = 2; + +pub static AC_QLOOKUP: [[u16; 256]; 3] = [ + [ + 4, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, + 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, + 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 104, 106, 108, 110, 112, 114, 116, + 118, 120, 122, 124, 126, 128, 130, 132, 134, 136, 138, 140, 142, 144, 146, 148, 150, 152, 155, 158, 161, 164, 167, 170, 173, 176, 179, 182, + 185, 188, 191, 194, 197, 200, 203, 207, 211, 215, 219, 223, 227, 231, 235, 239, 243, 247, 251, 255, 260, 265, 270, 275, 280, 285, 290, 295, + 300, 305, 311, 317, 323, 329, 335, 341, 347, 353, 359, 366, 373, 380, 387, 394, 401, 408, 416, 424, 432, 440, 448, 456, 465, 474, 483, 492, + 501, 510, 520, 530, 540, 550, 560, 571, 582, 593, 604, 615, 627, 639, 651, 663, 676, 689, 702, 715, 729, 743, 757, 771, 786, 801, 816, 832, + 848, 864, 881, 898, 915, 933, 951, 969, 988, 1007, 1026, 1046, 1066, 1087, 1108, 1129, 1151, 1173, 1196, 1219, 1243, 1267, 1292, 1317, 1343, + 1369, 1396, 1423, 1451, 1479, 1508, 1537, 1567, 1597, 1628, 1660, 1692, 1725, 1759, 1793, 1828, + ], + [ + 4, 9, 11, 13, 16, 18, 21, 24, 27, 30, 33, 37, 40, 44, 48, 51, 55, 59, 63, 67, 71, 75, 79, 83, 88, 92, 96, 100, 105, 109, 114, 118, 122, 127, + 131, 136, 140, 145, 149, 154, 158, 163, 168, 172, 177, 181, 186, 190, 195, 199, 204, 208, 213, 217, 222, 226, 231, 235, 240, 244, 249, 253, + 258, 262, 267, 271, 275, 280, 284, 289, 293, 297, 302, 306, 311, 315, 319, 324, 328, 332, 337, 341, 345, 349, 354, 358, 362, 367, 371, 375, + 379, 384, 388, 392, 396, 401, 409, 417, 425, 433, 441, 449, 458, 466, 474, 482, 490, 498, 506, 514, 523, 531, 539, 547, 555, 563, 571, 579, + 588, 596, 604, 616, 628, 640, 652, 664, 676, 688, 700, 713, 725, 737, 749, 761, 773, 785, 797, 809, 825, 841, 857, 873, 889, 905, 922, 938, + 954, 970, 986, 1002, 1018, 1038, 1058, 1078, 1098, 1118, 1138, 1158, 1178, 1198, 1218, 1242, 1266, 1290, 1314, 1338, 1362, 1386, 1411, 1435, + 1463, 1491, 1519, 1547, 1575, 1603, 1631, 1663, 1695, 1727, 1759, 1791, 1823, 1859, 1895, 1931, 1967, 2003, 2039, 2079, 2119, 2159, 2199, + 2239, 2283, 2327, 2371, 2415, 2459, 2507, 2555, 2603, 2651, 2703, 2755, 2807, 2859, 2915, 2971, 3027, 3083, 3143, 3203, 3263, 3327, 3391, + 3455, 3523, 3591, 3659, 3731, 3803, 3876, 3952, 4028, 4104, 4184, 4264, 4348, 4432, 4516, 4604, 4692, 4784, 4876, 4972, 5068, 5168, 5268, + 5372, 5476, 5584, 5692, 5804, 5916, 6032, 6148, 6268, 6388, 6512, 6640, 6768, 6900, 7036, 7172, 7312, + ], + [ + 4, 13, 19, 27, 35, 44, 54, 64, 75, 87, 99, 112, 126, 139, 154, 168, 183, 199, 214, 230, 247, 263, 280, 297, 314, 331, 349, 366, 384, 402, + 420, 438, 456, 475, 493, 511, 530, 548, 567, 586, 604, 623, 642, 660, 679, 698, 716, 735, 753, 772, 791, 809, 828, 846, 865, 884, 902, 920, + 939, 957, 976, 994, 1012, 1030, 1049, 1067, 1085, 1103, 1121, 1139, 1157, 1175, 1193, 1211, 1229, 1246, 1264, 1282, 1299, 1317, 1335, 1352, + 1370, 1387, 1405, 1422, 1440, 1457, 1474, 1491, 1509, 1526, 1543, 1560, 1577, 1595, 1627, 1660, 1693, 1725, 1758, 1791, 1824, 1856, 1889, + 1922, 1954, 1987, 2020, 2052, 2085, 2118, 2150, 2183, 2216, 2248, 2281, 2313, 2346, 2378, 2411, 2459, 2508, 2556, 2605, 2653, 2701, 2750, + 2798, 2847, 2895, 2943, 2992, 3040, 3088, 3137, 3185, 3234, 3298, 3362, 3426, 3491, 3555, 3619, 3684, 3748, 3812, 3876, 3941, 4005, 4069, + 4149, 4230, 4310, 4390, 4470, 4550, 4631, 4711, 4791, 4871, 4967, 5064, 5160, 5256, 5352, 5448, 5544, 5641, 5737, 5849, 5961, 6073, 6185, + 6297, 6410, 6522, 6650, 6778, 6906, 7034, 7162, 7290, 7435, 7579, 7723, 7867, 8011, 8155, 8315, 8475, 8635, 8795, 8956, 9132, 9308, 9484, + 9660, 9836, 10028, 10220, 10412, 10604, 10812, 11020, 11228, 11437, 11661, 11885, 12109, 12333, 12573, 12813, 13053, 13309, 13565, 13821, + 14093, 14365, 14637, 14925, 15213, 15502, 15806, 16110, 16414, 16734, 17054, 17390, 17726, 18062, 18414, 18766, 19134, 19502, 19886, 20270, + 20670, 21070, 21486, 21902, 22334, 22766, 23214, 23662, 24126, 24590, 25070, 25551, 26047, 26559, 27071, 27599, 28143, 28687, 29247, + ], +]; + +pub static ADJUSTED_TX_SIZE: [u8; 19] = [0, 1, 2, 3, 3, 5, 6, 7, 8, 9, 10, 3, 3, 13, 14, 15, 16, 9, 10]; + +pub static COEFF_BASE_CTX_OFFSET: [[[u8; 5]; 5]; 19] = [ + [[0, 1, 6, 6, 0], [1, 6, 6, 21, 0], [6, 6, 21, 21, 0], [6, 21, 21, 21, 0], [0, 0, 0, 0, 0]], + [[0, 1, 6, 6, 21], [1, 6, 6, 21, 21], [6, 6, 21, 21, 21], [6, 21, 21, 21, 21], [21, 21, 21, 21, 21]], + [[0, 1, 6, 6, 21], [1, 6, 6, 21, 21], [6, 6, 21, 21, 21], [6, 21, 21, 21, 21], [21, 21, 21, 21, 21]], + [[0, 1, 6, 6, 21], [1, 6, 6, 21, 21], [6, 6, 21, 21, 21], [6, 21, 21, 21, 21], [21, 21, 21, 21, 21]], + [[0, 1, 6, 6, 21], [1, 6, 6, 21, 21], [6, 6, 21, 21, 21], [6, 21, 21, 21, 21], [21, 21, 21, 21, 21]], + [[0, 11, 11, 11, 0], [11, 11, 11, 11, 0], [6, 6, 21, 21, 0], [6, 21, 21, 21, 0], [21, 21, 21, 21, 0]], + [[0, 16, 6, 6, 21], [16, 16, 6, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21], [0, 0, 0, 0, 0]], + [[0, 11, 11, 11, 11], [11, 11, 11, 11, 11], [6, 6, 21, 21, 21], [6, 21, 21, 21, 21], [21, 21, 21, 21, 21]], + [[0, 16, 6, 6, 21], [16, 16, 6, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21]], + [[0, 11, 11, 11, 11], [11, 11, 11, 11, 11], [6, 6, 21, 21, 21], [6, 21, 21, 21, 21], [21, 21, 21, 21, 21]], + [[0, 16, 6, 6, 21], [16, 16, 6, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21]], + [[0, 11, 11, 11, 11], [11, 11, 11, 11, 11], [6, 6, 21, 21, 21], [6, 21, 21, 21, 21], [21, 21, 21, 21, 21]], + [[0, 16, 6, 6, 21], [16, 16, 6, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21]], + [[0, 11, 11, 11, 0], [11, 11, 11, 11, 0], [6, 6, 21, 21, 0], [6, 21, 21, 21, 0], [21, 21, 21, 21, 0]], + [[0, 16, 6, 6, 21], [16, 16, 6, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21], [0, 0, 0, 0, 0]], + [[0, 11, 11, 11, 11], [11, 11, 11, 11, 11], [6, 6, 21, 21, 21], [6, 21, 21, 21, 21], [21, 21, 21, 21, 21]], + [[0, 16, 6, 6, 21], [16, 16, 6, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21]], + [[0, 11, 11, 11, 11], [11, 11, 11, 11, 11], [6, 6, 21, 21, 21], [6, 21, 21, 21, 21], [21, 21, 21, 21, 21]], + [[0, 16, 6, 6, 21], [16, 16, 6, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21], [16, 16, 21, 21, 21]], +]; + +pub static COEFF_BASE_POS_CTX_OFFSET: [u8; 3] = [26, 31, 36]; + +pub static COS128_LOOKUP: [u16; 65] = [ + 4096, 4095, 4091, 4085, 4076, 4065, 4052, 4036, 4017, 3996, 3973, 3948, 3920, 3889, 3857, 3822, 3784, 3745, 3703, 3659, 3612, 3564, 3513, 3461, + 3406, 3349, 3290, 3229, 3166, 3102, 3035, 2967, 2896, 2824, 2751, 2675, 2598, 2520, 2440, 2359, 2276, 2191, 2106, 2019, 1931, 1842, 1751, 1660, + 1567, 1474, 1380, 1285, 1189, 1092, 995, 897, 799, 700, 601, 501, 401, 301, 201, 101, 0, +]; + +pub static DC_QLOOKUP: [[u16; 256]; 3] = [ + [ + 4, 8, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 23, 24, 25, 26, 26, 27, 28, 29, 30, 31, 32, 32, 33, 34, 35, 36, 37, + 38, 38, 39, 40, 41, 42, 43, 43, 44, 45, 46, 47, 48, 48, 49, 50, 51, 52, 53, 53, 54, 55, 56, 57, 57, 58, 59, 60, 61, 62, 62, 63, 64, 65, 66, + 66, 67, 68, 69, 70, 70, 71, 72, 73, 74, 74, 75, 76, 77, 78, 78, 79, 80, 81, 81, 82, 83, 84, 85, 85, 87, 88, 90, 92, 93, 95, 96, 98, 99, 101, + 102, 104, 105, 107, 108, 110, 111, 113, 114, 116, 117, 118, 120, 121, 123, 125, 127, 129, 131, 134, 136, 138, 140, 142, 144, 146, 148, 150, + 152, 154, 156, 158, 161, 164, 166, 169, 172, 174, 177, 180, 182, 185, 187, 190, 192, 195, 199, 202, 205, 208, 211, 214, 217, 220, 223, 226, + 230, 233, 237, 240, 243, 247, 250, 253, 257, 261, 265, 269, 272, 276, 280, 284, 288, 292, 296, 300, 304, 309, 313, 317, 322, 326, 330, 335, + 340, 344, 349, 354, 359, 364, 369, 374, 379, 384, 389, 395, 400, 406, 411, 417, 423, 429, 435, 441, 447, 454, 461, 467, 475, 482, 489, 497, + 505, 513, 522, 530, 539, 549, 559, 569, 579, 590, 602, 614, 626, 640, 654, 668, 684, 700, 717, 736, 755, 775, 796, 819, 843, 869, 896, 925, + 955, 988, 1022, 1058, 1098, 1139, 1184, 1232, 1282, 1336, + ], + [ + 4, 9, 10, 13, 15, 17, 20, 22, 25, 28, 31, 34, 37, 40, 43, 47, 50, 53, 57, 60, 64, 68, 71, 75, 78, 82, 86, 90, 93, 97, 101, 105, 109, 113, + 116, 120, 124, 128, 132, 136, 140, 143, 147, 151, 155, 159, 163, 166, 170, 174, 178, 182, 185, 189, 193, 197, 200, 204, 208, 212, 215, 219, + 223, 226, 230, 233, 237, 241, 244, 248, 251, 255, 259, 262, 266, 269, 273, 276, 280, 283, 287, 290, 293, 297, 300, 304, 307, 310, 314, 317, + 321, 324, 327, 331, 334, 337, 343, 350, 356, 362, 369, 375, 381, 387, 394, 400, 406, 412, 418, 424, 430, 436, 442, 448, 454, 460, 466, 472, + 478, 484, 490, 499, 507, 516, 525, 533, 542, 550, 559, 567, 576, 584, 592, 601, 609, 617, 625, 634, 644, 655, 666, 676, 687, 698, 708, 718, + 729, 739, 749, 759, 770, 782, 795, 807, 819, 831, 844, 856, 868, 880, 891, 906, 920, 933, 947, 961, 975, 988, 1001, 1015, 1030, 1045, 1061, + 1076, 1090, 1105, 1120, 1137, 1153, 1170, 1186, 1202, 1218, 1236, 1253, 1271, 1288, 1306, 1323, 1342, 1361, 1379, 1398, 1416, 1436, 1456, + 1476, 1496, 1516, 1537, 1559, 1580, 1601, 1624, 1647, 1670, 1692, 1717, 1741, 1766, 1791, 1817, 1844, 1871, 1900, 1929, 1958, 1990, 2021, + 2054, 2088, 2123, 2159, 2197, 2236, 2276, 2319, 2363, 2410, 2458, 2508, 2561, 2616, 2675, 2737, 2802, 2871, 2944, 3020, 3102, 3188, 3280, + 3375, 3478, 3586, 3702, 3823, 3953, 4089, 4236, 4394, 4559, 4737, 4929, 5130, 5347, + ], + [ + 4, 12, 18, 25, 33, 41, 50, 60, 70, 80, 91, 103, 115, 127, 140, 153, 166, 180, 194, 208, 222, 237, 251, 266, 281, 296, 312, 327, 343, 358, + 374, 390, 405, 421, 437, 453, 469, 484, 500, 516, 532, 548, 564, 580, 596, 611, 627, 643, 659, 674, 690, 706, 721, 737, 752, 768, 783, 798, + 814, 829, 844, 859, 874, 889, 904, 919, 934, 949, 964, 978, 993, 1008, 1022, 1037, 1051, 1065, 1080, 1094, 1108, 1122, 1136, 1151, 1165, + 1179, 1192, 1206, 1220, 1234, 1248, 1261, 1275, 1288, 1302, 1315, 1329, 1342, 1368, 1393, 1419, 1444, 1469, 1494, 1519, 1544, 1569, 1594, + 1618, 1643, 1668, 1692, 1717, 1741, 1765, 1789, 1814, 1838, 1862, 1885, 1909, 1933, 1957, 1992, 2027, 2061, 2096, 2130, 2165, 2199, 2233, + 2267, 2300, 2334, 2367, 2400, 2434, 2467, 2499, 2532, 2575, 2618, 2661, 2704, 2746, 2788, 2830, 2872, 2913, 2954, 2995, 3036, 3076, 3127, + 3177, 3226, 3275, 3324, 3373, 3421, 3469, 3517, 3565, 3621, 3677, 3733, 3788, 3843, 3897, 3951, 4005, 4058, 4119, 4181, 4241, 4301, 4361, + 4420, 4479, 4546, 4612, 4677, 4742, 4807, 4871, 4942, 5013, 5083, 5153, 5222, 5291, 5367, 5442, 5517, 5591, 5665, 5745, 5825, 5905, 5984, + 6063, 6149, 6234, 6319, 6404, 6495, 6587, 6678, 6769, 6867, 6966, 7064, 7163, 7269, 7376, 7483, 7599, 7715, 7832, 7958, 8085, 8214, 8352, + 8492, 8635, 8788, 8945, 9104, 9275, 9450, 9639, 9832, 10031, 10245, 10465, 10702, 10946, 11210, 11482, 11776, 12081, 12409, 12750, 13118, + 13501, 13913, 14343, 14807, 15290, 15812, 16356, 16943, 17575, 18237, 18949, 19718, 20521, 21387, + ], +]; + +pub static DEFAULT_ANGLE_DELTA_CDF: [[u16; 8]; 8] = [ + [2180, 5032, 7567, 22776, 26989, 30217, 32768, 0], + [2301, 5608, 8801, 23487, 26974, 30330, 32768, 0], + [3780, 11018, 13699, 19354, 23083, 31286, 32768, 0], + [4581, 11226, 15147, 17138, 21834, 28397, 32768, 0], + [1737, 10927, 14509, 19588, 22745, 28823, 32768, 0], + [2664, 10176, 12485, 17650, 21600, 30495, 32768, 0], + [2240, 11096, 15453, 20341, 22561, 28917, 32768, 0], + [3605, 10428, 12459, 17676, 21244, 30655, 32768, 0], +]; + +pub static DEFAULT_COEFF_BASE_CDF: [[[[[u16; 5]; 42]; 2]; 5]; 4] = [ + [ + [ + [ + [4034, 8930, 12727, 32768, 0], + [18082, 29741, 31877, 32768, 0], + [12596, 26124, 30493, 32768, 0], + [9446, 21118, 27005, 32768, 0], + [6308, 15141, 21279, 32768, 0], + [2463, 6357, 9783, 32768, 0], + [20667, 30546, 31929, 32768, 0], + [13043, 26123, 30134, 32768, 0], + [8151, 18757, 24778, 32768, 0], + [5255, 12839, 18632, 32768, 0], + [2820, 7206, 11161, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [15736, 27553, 30604, 32768, 0], + [11210, 23794, 28787, 32768, 0], + [5947, 13874, 19701, 32768, 0], + [4215, 9323, 13891, 32768, 0], + [2833, 6462, 10059, 32768, 0], + [19605, 30393, 31582, 32768, 0], + [13523, 26252, 30248, 32768, 0], + [8446, 18622, 24512, 32768, 0], + [3818, 10343, 15974, 32768, 0], + [1481, 4117, 6796, 32768, 0], + [22649, 31302, 32190, 32768, 0], + [14829, 27127, 30449, 32768, 0], + [8313, 17702, 23304, 32768, 0], + [3022, 8301, 12786, 32768, 0], + [1536, 4412, 7184, 32768, 0], + [22354, 29774, 31372, 32768, 0], + [14723, 25472, 29214, 32768, 0], + [6673, 13745, 18662, 32768, 0], + [2068, 5766, 9322, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [6302, 16444, 21761, 32768, 0], + [23040, 31538, 32475, 32768, 0], + [15196, 28452, 31496, 32768, 0], + [10020, 22946, 28514, 32768, 0], + [6533, 16862, 23501, 32768, 0], + [3538, 9816, 15076, 32768, 0], + [24444, 31875, 32525, 32768, 0], + [15881, 28924, 31635, 32768, 0], + [9922, 22873, 28466, 32768, 0], + [6527, 16966, 23691, 32768, 0], + [4114, 11303, 17220, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [20201, 30770, 32209, 32768, 0], + [14754, 28071, 31258, 32768, 0], + [8378, 20186, 26517, 32768, 0], + [5916, 15299, 21978, 32768, 0], + [4268, 11583, 17901, 32768, 0], + [24361, 32025, 32581, 32768, 0], + [18673, 30105, 31943, 32768, 0], + [10196, 22244, 27576, 32768, 0], + [5495, 14349, 20417, 32768, 0], + [2676, 7415, 11498, 32768, 0], + [24678, 31958, 32585, 32768, 0], + [18629, 29906, 31831, 32768, 0], + [9364, 20724, 26315, 32768, 0], + [4641, 12318, 18094, 32768, 0], + [2758, 7387, 11579, 32768, 0], + [25433, 31842, 32469, 32768, 0], + [18795, 29289, 31411, 32768, 0], + [7644, 17584, 23592, 32768, 0], + [3408, 9014, 15047, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [4536, 10072, 14001, 32768, 0], + [25459, 31416, 32206, 32768, 0], + [16605, 28048, 30818, 32768, 0], + [11008, 22857, 27719, 32768, 0], + [6915, 16268, 22315, 32768, 0], + [2625, 6812, 10537, 32768, 0], + [24257, 31788, 32499, 32768, 0], + [16880, 29454, 31879, 32768, 0], + [11958, 25054, 29778, 32768, 0], + [7916, 18718, 25084, 32768, 0], + [3383, 8777, 13446, 32768, 0], + [22720, 31603, 32393, 32768, 0], + [14960, 28125, 31335, 32768, 0], + [9731, 22210, 27928, 32768, 0], + [6304, 15832, 22277, 32768, 0], + [2910, 7818, 12166, 32768, 0], + [20375, 30627, 32131, 32768, 0], + [13904, 27284, 30887, 32768, 0], + [9368, 21558, 27144, 32768, 0], + [5937, 14966, 21119, 32768, 0], + [2667, 7225, 11319, 32768, 0], + [23970, 31470, 32378, 32768, 0], + [17173, 29734, 32018, 32768, 0], + [12795, 25441, 29965, 32768, 0], + [8981, 19680, 25893, 32768, 0], + [4728, 11372, 16902, 32768, 0], + [24287, 31797, 32439, 32768, 0], + [16703, 29145, 31696, 32768, 0], + [10833, 23554, 28725, 32768, 0], + [6468, 16566, 23057, 32768, 0], + [2415, 6562, 10278, 32768, 0], + [26610, 32395, 32659, 32768, 0], + [18590, 30498, 32117, 32768, 0], + [12420, 25756, 29950, 32768, 0], + [7639, 18746, 24710, 32768, 0], + [3001, 8086, 12347, 32768, 0], + [25076, 32064, 32580, 32768, 0], + [17946, 30128, 32028, 32768, 0], + [12024, 24985, 29378, 32768, 0], + [7517, 18390, 24304, 32768, 0], + [3243, 8781, 13331, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [6037, 16771, 21957, 32768, 0], + [24774, 31704, 32426, 32768, 0], + [16830, 28589, 31056, 32768, 0], + [10602, 22828, 27760, 32768, 0], + [6733, 16829, 23071, 32768, 0], + [3250, 8914, 13556, 32768, 0], + [25582, 32220, 32668, 32768, 0], + [18659, 30342, 32223, 32768, 0], + [12546, 26149, 30515, 32768, 0], + [8420, 20451, 26801, 32768, 0], + [4636, 12420, 18344, 32768, 0], + [27581, 32362, 32639, 32768, 0], + [18987, 30083, 31978, 32768, 0], + [11327, 24248, 29084, 32768, 0], + [7264, 17719, 24120, 32768, 0], + [3995, 10768, 16169, 32768, 0], + [25893, 31831, 32487, 32768, 0], + [16577, 28587, 31379, 32768, 0], + [10189, 22748, 28182, 32768, 0], + [6832, 17094, 23556, 32768, 0], + [3708, 10110, 15334, 32768, 0], + [25904, 32282, 32656, 32768, 0], + [19721, 30792, 32276, 32768, 0], + [12819, 26243, 30411, 32768, 0], + [8572, 20614, 26891, 32768, 0], + [5364, 14059, 20467, 32768, 0], + [26580, 32438, 32677, 32768, 0], + [20852, 31225, 32340, 32768, 0], + [12435, 25700, 29967, 32768, 0], + [8691, 20825, 26976, 32768, 0], + [4446, 12209, 17269, 32768, 0], + [27350, 32429, 32696, 32768, 0], + [21372, 30977, 32272, 32768, 0], + [12673, 25270, 29853, 32768, 0], + [9208, 20925, 26640, 32768, 0], + [5018, 13351, 18732, 32768, 0], + [27351, 32479, 32713, 32768, 0], + [21398, 31209, 32387, 32768, 0], + [12162, 25047, 29842, 32768, 0], + [7896, 18691, 25319, 32768, 0], + [4670, 12882, 18881, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [5487, 10460, 13708, 32768, 0], + [21597, 28303, 30674, 32768, 0], + [11037, 21953, 26476, 32768, 0], + [8147, 17962, 22952, 32768, 0], + [5242, 13061, 18532, 32768, 0], + [1889, 5208, 8182, 32768, 0], + [26774, 32133, 32590, 32768, 0], + [17844, 29564, 31767, 32768, 0], + [11690, 24438, 29171, 32768, 0], + [7542, 18215, 24459, 32768, 0], + [2993, 8050, 12319, 32768, 0], + [28023, 32328, 32591, 32768, 0], + [18651, 30126, 31954, 32768, 0], + [12164, 25146, 29589, 32768, 0], + [7762, 18530, 24771, 32768, 0], + [3492, 9183, 13920, 32768, 0], + [27591, 32008, 32491, 32768, 0], + [17149, 28853, 31510, 32768, 0], + [11485, 24003, 28860, 32768, 0], + [7697, 18086, 24210, 32768, 0], + [3075, 7999, 12218, 32768, 0], + [28268, 32482, 32654, 32768, 0], + [19631, 31051, 32404, 32768, 0], + [13860, 27260, 31020, 32768, 0], + [9605, 21613, 27594, 32768, 0], + [4876, 12162, 17908, 32768, 0], + [27248, 32316, 32576, 32768, 0], + [18955, 30457, 32075, 32768, 0], + [11824, 23997, 28795, 32768, 0], + [7346, 18196, 24647, 32768, 0], + [3403, 9247, 14111, 32768, 0], + [29711, 32655, 32735, 32768, 0], + [21169, 31394, 32417, 32768, 0], + [13487, 27198, 30957, 32768, 0], + [8828, 21683, 27614, 32768, 0], + [4270, 11451, 17038, 32768, 0], + [28708, 32578, 32731, 32768, 0], + [20120, 31241, 32482, 32768, 0], + [13692, 27550, 31321, 32768, 0], + [9418, 22514, 28439, 32768, 0], + [4999, 13283, 19462, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [5673, 14302, 19711, 32768, 0], + [26251, 30701, 31834, 32768, 0], + [12782, 23783, 27803, 32768, 0], + [9127, 20657, 25808, 32768, 0], + [6368, 16208, 21462, 32768, 0], + [2465, 7177, 10822, 32768, 0], + [29961, 32563, 32719, 32768, 0], + [18318, 29891, 31949, 32768, 0], + [11361, 24514, 29357, 32768, 0], + [7900, 19603, 25607, 32768, 0], + [4002, 10590, 15546, 32768, 0], + [29637, 32310, 32595, 32768, 0], + [18296, 29913, 31809, 32768, 0], + [10144, 21515, 26871, 32768, 0], + [5358, 14322, 20394, 32768, 0], + [3067, 8362, 13346, 32768, 0], + [28652, 32470, 32676, 32768, 0], + [17538, 30771, 32209, 32768, 0], + [13924, 26882, 30494, 32768, 0], + [10496, 22837, 27869, 32768, 0], + [7236, 16396, 21621, 32768, 0], + [30743, 32687, 32746, 32768, 0], + [23006, 31676, 32489, 32768, 0], + [14494, 27828, 31120, 32768, 0], + [10174, 22801, 28352, 32768, 0], + [6242, 15281, 21043, 32768, 0], + [25817, 32243, 32720, 32768, 0], + [18618, 31367, 32325, 32768, 0], + [13997, 28318, 31878, 32768, 0], + [12255, 26534, 31383, 32768, 0], + [9561, 21588, 28450, 32768, 0], + [28188, 32635, 32724, 32768, 0], + [22060, 32365, 32728, 32768, 0], + [18102, 30690, 32528, 32768, 0], + [14196, 28864, 31999, 32768, 0], + [12262, 25792, 30865, 32768, 0], + [24176, 32109, 32628, 32768, 0], + [18280, 29681, 31963, 32768, 0], + [10205, 23703, 29664, 32768, 0], + [7889, 20025, 27676, 32768, 0], + [6060, 16743, 23970, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [5141, 7096, 8260, 32768, 0], + [27186, 29022, 29789, 32768, 0], + [6668, 12568, 15682, 32768, 0], + [2172, 6181, 8638, 32768, 0], + [1126, 3379, 4531, 32768, 0], + [443, 1361, 2254, 32768, 0], + [26083, 31153, 32436, 32768, 0], + [13486, 24603, 28483, 32768, 0], + [6508, 14840, 19910, 32768, 0], + [3386, 8800, 13286, 32768, 0], + [1530, 4322, 7054, 32768, 0], + [29639, 32080, 32548, 32768, 0], + [15897, 27552, 30290, 32768, 0], + [8588, 20047, 25383, 32768, 0], + [4889, 13339, 19269, 32768, 0], + [2240, 6871, 10498, 32768, 0], + [28165, 32197, 32517, 32768, 0], + [20735, 30427, 31568, 32768, 0], + [14325, 24671, 27692, 32768, 0], + [5119, 12554, 17805, 32768, 0], + [1810, 5441, 8261, 32768, 0], + [31212, 32724, 32748, 32768, 0], + [23352, 31766, 32545, 32768, 0], + [14669, 27570, 31059, 32768, 0], + [8492, 20894, 27272, 32768, 0], + [3644, 10194, 15204, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [2461, 7013, 9371, 32768, 0], + [24749, 29600, 30986, 32768, 0], + [9466, 19037, 22417, 32768, 0], + [3584, 9280, 14400, 32768, 0], + [1505, 3929, 5433, 32768, 0], + [677, 1500, 2736, 32768, 0], + [23987, 30702, 32117, 32768, 0], + [13554, 24571, 29263, 32768, 0], + [6211, 14556, 21155, 32768, 0], + [3135, 10972, 15625, 32768, 0], + [2435, 7127, 11427, 32768, 0], + [31300, 32532, 32550, 32768, 0], + [14757, 30365, 31954, 32768, 0], + [4405, 11612, 18553, 32768, 0], + [580, 4132, 7322, 32768, 0], + [1695, 10169, 14124, 32768, 0], + [30008, 32282, 32591, 32768, 0], + [19244, 30108, 31748, 32768, 0], + [11180, 24158, 29555, 32768, 0], + [5650, 14972, 19209, 32768, 0], + [2114, 5109, 8456, 32768, 0], + [31856, 32716, 32748, 32768, 0], + [23012, 31664, 32572, 32768, 0], + [13694, 26656, 30636, 32768, 0], + [8142, 19508, 26093, 32768, 0], + [4253, 10955, 16724, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [601, 983, 1311, 32768, 0], + [18725, 23406, 28087, 32768, 0], + [5461, 8192, 10923, 32768, 0], + [3781, 15124, 21425, 32768, 0], + [2587, 7761, 12072, 32768, 0], + [106, 458, 810, 32768, 0], + [22282, 29710, 31894, 32768, 0], + [8508, 20926, 25984, 32768, 0], + [3726, 12713, 18083, 32768, 0], + [1620, 7112, 10893, 32768, 0], + [729, 2236, 3495, 32768, 0], + [30163, 32474, 32684, 32768, 0], + [18304, 30464, 32000, 32768, 0], + [11443, 26526, 29647, 32768, 0], + [6007, 15292, 21299, 32768, 0], + [2234, 6703, 8937, 32768, 0], + [30954, 32177, 32571, 32768, 0], + [17363, 29562, 31076, 32768, 0], + [9686, 22464, 27410, 32768, 0], + [8192, 16384, 21390, 32768, 0], + [1755, 8046, 11264, 32768, 0], + [31168, 32734, 32748, 32768, 0], + [22486, 31441, 32471, 32768, 0], + [12833, 25627, 29738, 32768, 0], + [6980, 17379, 23122, 32768, 0], + [3111, 8887, 13479, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + ], + [ + [ + [ + [6041, 11854, 15927, 32768, 0], + [20326, 30905, 32251, 32768, 0], + [14164, 26831, 30725, 32768, 0], + [9760, 20647, 26585, 32768, 0], + [6416, 14953, 21219, 32768, 0], + [2966, 7151, 10891, 32768, 0], + [23567, 31374, 32254, 32768, 0], + [14978, 27416, 30946, 32768, 0], + [9434, 20225, 26254, 32768, 0], + [6658, 14558, 20535, 32768, 0], + [3916, 8677, 12989, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [18088, 29545, 31587, 32768, 0], + [13062, 25843, 30073, 32768, 0], + [8940, 16827, 22251, 32768, 0], + [7654, 13220, 17973, 32768, 0], + [5733, 10316, 14456, 32768, 0], + [22879, 31388, 32114, 32768, 0], + [15215, 27993, 30955, 32768, 0], + [9397, 19445, 24978, 32768, 0], + [3442, 9813, 15344, 32768, 0], + [1368, 3936, 6532, 32768, 0], + [25494, 32033, 32406, 32768, 0], + [16772, 27963, 30718, 32768, 0], + [9419, 18165, 23260, 32768, 0], + [2677, 7501, 11797, 32768, 0], + [1516, 4344, 7170, 32768, 0], + [26556, 31454, 32101, 32768, 0], + [17128, 27035, 30108, 32768, 0], + [8324, 15344, 20249, 32768, 0], + [1903, 5696, 9469, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8455, 19003, 24368, 32768, 0], + [23563, 32021, 32604, 32768, 0], + [16237, 29446, 31935, 32768, 0], + [10724, 23999, 29358, 32768, 0], + [6725, 17528, 24416, 32768, 0], + [3927, 10927, 16825, 32768, 0], + [26313, 32288, 32634, 32768, 0], + [17430, 30095, 32095, 32768, 0], + [11116, 24606, 29679, 32768, 0], + [7195, 18384, 25269, 32768, 0], + [4726, 12852, 19315, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [22822, 31648, 32483, 32768, 0], + [16724, 29633, 31929, 32768, 0], + [10261, 23033, 28725, 32768, 0], + [7029, 17840, 24528, 32768, 0], + [4867, 13886, 21502, 32768, 0], + [25298, 31892, 32491, 32768, 0], + [17809, 29330, 31512, 32768, 0], + [9668, 21329, 26579, 32768, 0], + [4774, 12956, 18976, 32768, 0], + [2322, 7030, 11540, 32768, 0], + [25472, 31920, 32543, 32768, 0], + [17957, 29387, 31632, 32768, 0], + [9196, 20593, 26400, 32768, 0], + [4680, 12705, 19202, 32768, 0], + [2917, 8456, 13436, 32768, 0], + [26471, 32059, 32574, 32768, 0], + [18458, 29783, 31909, 32768, 0], + [8400, 19464, 25956, 32768, 0], + [3812, 10973, 17206, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [6779, 13743, 17678, 32768, 0], + [24806, 31797, 32457, 32768, 0], + [17616, 29047, 31372, 32768, 0], + [11063, 23175, 28003, 32768, 0], + [6521, 16110, 22324, 32768, 0], + [2764, 7504, 11654, 32768, 0], + [25266, 32367, 32637, 32768, 0], + [19054, 30553, 32175, 32768, 0], + [12139, 25212, 29807, 32768, 0], + [7311, 18162, 24704, 32768, 0], + [3397, 9164, 14074, 32768, 0], + [25988, 32208, 32522, 32768, 0], + [16253, 28912, 31526, 32768, 0], + [9151, 21387, 27372, 32768, 0], + [5688, 14915, 21496, 32768, 0], + [2717, 7627, 12004, 32768, 0], + [23144, 31855, 32443, 32768, 0], + [16070, 28491, 31325, 32768, 0], + [8702, 20467, 26517, 32768, 0], + [5243, 13956, 20367, 32768, 0], + [2621, 7335, 11567, 32768, 0], + [26636, 32340, 32630, 32768, 0], + [19990, 31050, 32341, 32768, 0], + [13243, 26105, 30315, 32768, 0], + [8588, 19521, 25918, 32768, 0], + [4717, 11585, 17304, 32768, 0], + [25844, 32292, 32582, 32768, 0], + [19090, 30635, 32097, 32768, 0], + [11963, 24546, 28939, 32768, 0], + [6218, 16087, 22354, 32768, 0], + [2340, 6608, 10426, 32768, 0], + [28046, 32576, 32694, 32768, 0], + [21178, 31313, 32296, 32768, 0], + [13486, 26184, 29870, 32768, 0], + [7149, 17871, 23723, 32768, 0], + [2833, 7958, 12259, 32768, 0], + [27710, 32528, 32686, 32768, 0], + [20674, 31076, 32268, 32768, 0], + [12413, 24955, 29243, 32768, 0], + [6676, 16927, 23097, 32768, 0], + [2966, 8333, 12919, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8639, 19339, 24429, 32768, 0], + [24404, 31837, 32525, 32768, 0], + [16997, 29425, 31784, 32768, 0], + [11253, 24234, 29149, 32768, 0], + [6751, 17394, 24028, 32768, 0], + [3490, 9830, 15191, 32768, 0], + [26283, 32471, 32714, 32768, 0], + [19599, 31168, 32442, 32768, 0], + [13146, 26954, 30893, 32768, 0], + [8214, 20588, 26890, 32768, 0], + [4699, 13081, 19300, 32768, 0], + [28212, 32458, 32669, 32768, 0], + [18594, 30316, 32100, 32768, 0], + [11219, 24408, 29234, 32768, 0], + [6865, 17656, 24149, 32768, 0], + [3678, 10362, 16006, 32768, 0], + [25825, 32136, 32616, 32768, 0], + [17313, 29853, 32021, 32768, 0], + [11197, 24471, 29472, 32768, 0], + [6947, 17781, 24405, 32768, 0], + [3768, 10660, 16261, 32768, 0], + [27352, 32500, 32706, 32768, 0], + [20850, 31468, 32469, 32768, 0], + [14021, 27707, 31133, 32768, 0], + [8964, 21748, 27838, 32768, 0], + [5437, 14665, 21187, 32768, 0], + [26304, 32492, 32698, 32768, 0], + [20409, 31380, 32385, 32768, 0], + [13682, 27222, 30632, 32768, 0], + [8974, 21236, 26685, 32768, 0], + [4234, 11665, 16934, 32768, 0], + [26273, 32357, 32711, 32768, 0], + [20672, 31242, 32441, 32768, 0], + [14172, 27254, 30902, 32768, 0], + [9870, 21898, 27275, 32768, 0], + [5164, 13506, 19270, 32768, 0], + [26725, 32459, 32728, 32768, 0], + [20991, 31442, 32527, 32768, 0], + [13071, 26434, 30811, 32768, 0], + [8184, 20090, 26742, 32768, 0], + [4803, 13255, 19895, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [7555, 14942, 18501, 32768, 0], + [24410, 31178, 32287, 32768, 0], + [14394, 26738, 30253, 32768, 0], + [8413, 19554, 25195, 32768, 0], + [4766, 12924, 18785, 32768, 0], + [2029, 5806, 9207, 32768, 0], + [26776, 32364, 32663, 32768, 0], + [18732, 29967, 31931, 32768, 0], + [11005, 23786, 28852, 32768, 0], + [6466, 16909, 23510, 32768, 0], + [3044, 8638, 13419, 32768, 0], + [29208, 32582, 32704, 32768, 0], + [20068, 30857, 32208, 32768, 0], + [12003, 25085, 29595, 32768, 0], + [6947, 17750, 24189, 32768, 0], + [3245, 9103, 14007, 32768, 0], + [27359, 32465, 32669, 32768, 0], + [19421, 30614, 32174, 32768, 0], + [11915, 25010, 29579, 32768, 0], + [6950, 17676, 24074, 32768, 0], + [3007, 8473, 13096, 32768, 0], + [29002, 32676, 32735, 32768, 0], + [22102, 31849, 32576, 32768, 0], + [14408, 28009, 31405, 32768, 0], + [9027, 21679, 27931, 32768, 0], + [4694, 12678, 18748, 32768, 0], + [28216, 32528, 32682, 32768, 0], + [20849, 31264, 32318, 32768, 0], + [12756, 25815, 29751, 32768, 0], + [7565, 18801, 24923, 32768, 0], + [3509, 9533, 14477, 32768, 0], + [30133, 32687, 32739, 32768, 0], + [23063, 31910, 32515, 32768, 0], + [14588, 28051, 31132, 32768, 0], + [9085, 21649, 27457, 32768, 0], + [4261, 11654, 17264, 32768, 0], + [29518, 32691, 32748, 32768, 0], + [22451, 31959, 32613, 32768, 0], + [14864, 28722, 31700, 32768, 0], + [9695, 22964, 28716, 32768, 0], + [4932, 13358, 19502, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [6465, 16958, 21688, 32768, 0], + [25199, 31514, 32360, 32768, 0], + [14774, 27149, 30607, 32768, 0], + [9257, 21438, 26972, 32768, 0], + [5723, 15183, 21882, 32768, 0], + [3150, 8879, 13731, 32768, 0], + [26989, 32262, 32682, 32768, 0], + [17396, 29937, 32085, 32768, 0], + [11387, 24901, 29784, 32768, 0], + [7289, 18821, 25548, 32768, 0], + [3734, 10577, 16086, 32768, 0], + [29728, 32501, 32695, 32768, 0], + [17431, 29701, 31903, 32768, 0], + [9921, 22826, 28300, 32768, 0], + [5896, 15434, 22068, 32768, 0], + [3430, 9646, 14757, 32768, 0], + [28614, 32511, 32705, 32768, 0], + [19364, 30638, 32263, 32768, 0], + [13129, 26254, 30402, 32768, 0], + [8754, 20484, 26440, 32768, 0], + [4378, 11607, 17110, 32768, 0], + [30292, 32671, 32744, 32768, 0], + [21780, 31603, 32501, 32768, 0], + [14314, 27829, 31291, 32768, 0], + [9611, 22327, 28263, 32768, 0], + [4890, 13087, 19065, 32768, 0], + [25862, 32567, 32733, 32768, 0], + [20794, 32050, 32567, 32768, 0], + [17243, 30625, 32254, 32768, 0], + [13283, 27628, 31474, 32768, 0], + [9669, 22532, 28918, 32768, 0], + [27435, 32697, 32748, 32768, 0], + [24922, 32390, 32714, 32768, 0], + [21449, 31504, 32536, 32768, 0], + [16392, 29729, 31832, 32768, 0], + [11692, 24884, 29076, 32768, 0], + [24193, 32290, 32735, 32768, 0], + [18909, 31104, 32563, 32768, 0], + [12236, 26841, 31403, 32768, 0], + [8171, 21840, 29082, 32768, 0], + [7224, 17280, 25275, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [3078, 6839, 9890, 32768, 0], + [13837, 20450, 24479, 32768, 0], + [5914, 14222, 19328, 32768, 0], + [3866, 10267, 14762, 32768, 0], + [2612, 7208, 11042, 32768, 0], + [1067, 2991, 4776, 32768, 0], + [25817, 31646, 32529, 32768, 0], + [13708, 26338, 30385, 32768, 0], + [7328, 18585, 24870, 32768, 0], + [4691, 13080, 19276, 32768, 0], + [1825, 5253, 8352, 32768, 0], + [29386, 32315, 32624, 32768, 0], + [17160, 29001, 31360, 32768, 0], + [9602, 21862, 27396, 32768, 0], + [5915, 15772, 22148, 32768, 0], + [2786, 7779, 12047, 32768, 0], + [29246, 32450, 32663, 32768, 0], + [18696, 29929, 31818, 32768, 0], + [10510, 23369, 28560, 32768, 0], + [6229, 16499, 23125, 32768, 0], + [2608, 7448, 11705, 32768, 0], + [30753, 32710, 32748, 32768, 0], + [21638, 31487, 32503, 32768, 0], + [12937, 26854, 30870, 32768, 0], + [8182, 20596, 26970, 32768, 0], + [3637, 10269, 15497, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [5244, 12150, 16906, 32768, 0], + [20486, 26858, 29701, 32768, 0], + [7756, 18317, 23735, 32768, 0], + [3452, 9256, 13146, 32768, 0], + [2020, 5206, 8229, 32768, 0], + [1801, 4993, 7903, 32768, 0], + [27051, 31858, 32531, 32768, 0], + [15988, 27531, 30619, 32768, 0], + [9188, 21484, 26719, 32768, 0], + [6273, 17186, 23800, 32768, 0], + [3108, 9355, 14764, 32768, 0], + [31076, 32520, 32680, 32768, 0], + [18119, 30037, 31850, 32768, 0], + [10244, 22969, 27472, 32768, 0], + [4692, 14077, 19273, 32768, 0], + [3694, 11677, 17556, 32768, 0], + [30060, 32581, 32720, 32768, 0], + [21011, 30775, 32120, 32768, 0], + [11931, 24820, 29289, 32768, 0], + [7119, 17662, 24356, 32768, 0], + [3833, 10706, 16304, 32768, 0], + [31954, 32731, 32748, 32768, 0], + [23913, 31724, 32489, 32768, 0], + [15520, 28060, 31286, 32768, 0], + [11517, 23008, 28571, 32768, 0], + [6193, 14508, 20629, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [1035, 2807, 4156, 32768, 0], + [13162, 18138, 20939, 32768, 0], + [2696, 6633, 8755, 32768, 0], + [1373, 4161, 6853, 32768, 0], + [1099, 2746, 4716, 32768, 0], + [340, 1021, 1599, 32768, 0], + [22826, 30419, 32135, 32768, 0], + [10395, 21762, 26942, 32768, 0], + [4726, 12407, 17361, 32768, 0], + [2447, 7080, 10593, 32768, 0], + [1227, 3717, 6011, 32768, 0], + [28156, 31424, 31934, 32768, 0], + [16915, 27754, 30373, 32768, 0], + [9148, 20990, 26431, 32768, 0], + [5950, 15515, 21148, 32768, 0], + [2492, 7327, 11526, 32768, 0], + [30602, 32477, 32670, 32768, 0], + [20026, 29955, 31568, 32768, 0], + [11220, 23628, 28105, 32768, 0], + [6652, 17019, 22973, 32768, 0], + [3064, 8536, 13043, 32768, 0], + [31769, 32724, 32748, 32768, 0], + [22230, 30887, 32373, 32768, 0], + [12234, 25079, 29731, 32768, 0], + [7326, 18816, 25353, 32768, 0], + [3933, 10907, 16616, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + ], + [ + [ + [ + [8896, 16227, 20630, 32768, 0], + [23629, 31782, 32527, 32768, 0], + [15173, 27755, 31321, 32768, 0], + [10158, 21233, 27382, 32768, 0], + [6420, 14857, 21558, 32768, 0], + [3269, 8155, 12646, 32768, 0], + [24835, 32009, 32496, 32768, 0], + [16509, 28421, 31579, 32768, 0], + [10957, 21514, 27418, 32768, 0], + [7881, 15930, 22096, 32768, 0], + [5388, 10960, 15918, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [20745, 30773, 32093, 32768, 0], + [15200, 27221, 30861, 32768, 0], + [13032, 20873, 25667, 32768, 0], + [12285, 18663, 23494, 32768, 0], + [11563, 17481, 21489, 32768, 0], + [26260, 31982, 32320, 32768, 0], + [15397, 28083, 31100, 32768, 0], + [9742, 19217, 24824, 32768, 0], + [3261, 9629, 15362, 32768, 0], + [1480, 4322, 7499, 32768, 0], + [27599, 32256, 32460, 32768, 0], + [16857, 27659, 30774, 32768, 0], + [9551, 18290, 23748, 32768, 0], + [3052, 8933, 14103, 32768, 0], + [2021, 5910, 9787, 32768, 0], + [29005, 32015, 32392, 32768, 0], + [17677, 27694, 30863, 32768, 0], + [9204, 17356, 23219, 32768, 0], + [2403, 7516, 12814, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [10808, 22056, 26896, 32768, 0], + [25739, 32313, 32676, 32768, 0], + [17288, 30203, 32221, 32768, 0], + [11359, 24878, 29896, 32768, 0], + [6949, 17767, 24893, 32768, 0], + [4287, 11796, 18071, 32768, 0], + [27880, 32521, 32705, 32768, 0], + [19038, 31004, 32414, 32768, 0], + [12564, 26345, 30768, 32768, 0], + [8269, 19947, 26779, 32768, 0], + [5674, 14657, 21674, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [25742, 32319, 32671, 32768, 0], + [19557, 31164, 32454, 32768, 0], + [13381, 26381, 30755, 32768, 0], + [10101, 21466, 26722, 32768, 0], + [9209, 19650, 26825, 32768, 0], + [27107, 31917, 32432, 32768, 0], + [18056, 28893, 31203, 32768, 0], + [10200, 21434, 26764, 32768, 0], + [4660, 12913, 19502, 32768, 0], + [2368, 6930, 12504, 32768, 0], + [26960, 32158, 32613, 32768, 0], + [18628, 30005, 32031, 32768, 0], + [10233, 22442, 28232, 32768, 0], + [5471, 14630, 21516, 32768, 0], + [3235, 10767, 17109, 32768, 0], + [27696, 32440, 32692, 32768, 0], + [20032, 31167, 32438, 32768, 0], + [8700, 21341, 28442, 32768, 0], + [5662, 14831, 21795, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [9704, 17294, 21132, 32768, 0], + [26762, 32278, 32633, 32768, 0], + [18382, 29620, 31819, 32768, 0], + [10891, 23475, 28723, 32768, 0], + [6358, 16583, 23309, 32768, 0], + [3248, 9118, 14141, 32768, 0], + [27204, 32573, 32699, 32768, 0], + [19818, 30824, 32329, 32768, 0], + [11772, 25120, 30041, 32768, 0], + [6995, 18033, 25039, 32768, 0], + [3752, 10442, 16098, 32768, 0], + [27222, 32256, 32559, 32768, 0], + [15356, 28399, 31475, 32768, 0], + [8821, 20635, 27057, 32768, 0], + [5511, 14404, 21239, 32768, 0], + [2935, 8222, 13051, 32768, 0], + [24875, 32120, 32529, 32768, 0], + [15233, 28265, 31445, 32768, 0], + [8605, 20570, 26932, 32768, 0], + [5431, 14413, 21196, 32768, 0], + [2994, 8341, 13223, 32768, 0], + [28201, 32604, 32700, 32768, 0], + [21041, 31446, 32456, 32768, 0], + [13221, 26213, 30475, 32768, 0], + [8255, 19385, 26037, 32768, 0], + [4930, 12585, 18830, 32768, 0], + [28768, 32448, 32627, 32768, 0], + [19705, 30561, 32021, 32768, 0], + [11572, 23589, 28220, 32768, 0], + [5532, 15034, 21446, 32768, 0], + [2460, 7150, 11456, 32768, 0], + [29874, 32619, 32699, 32768, 0], + [21621, 31071, 32201, 32768, 0], + [12511, 24747, 28992, 32768, 0], + [6281, 16395, 22748, 32768, 0], + [3246, 9278, 14497, 32768, 0], + [29715, 32625, 32712, 32768, 0], + [20958, 31011, 32283, 32768, 0], + [11233, 23671, 28806, 32768, 0], + [6012, 16128, 22868, 32768, 0], + [3427, 9851, 15414, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [11016, 22111, 26794, 32768, 0], + [25946, 32357, 32677, 32768, 0], + [17890, 30452, 32252, 32768, 0], + [11678, 25142, 29816, 32768, 0], + [6720, 17534, 24584, 32768, 0], + [4230, 11665, 17820, 32768, 0], + [28400, 32623, 32747, 32768, 0], + [21164, 31668, 32575, 32768, 0], + [13572, 27388, 31182, 32768, 0], + [8234, 20750, 27358, 32768, 0], + [5065, 14055, 20897, 32768, 0], + [28981, 32547, 32705, 32768, 0], + [18681, 30543, 32239, 32768, 0], + [10919, 24075, 29286, 32768, 0], + [6431, 17199, 24077, 32768, 0], + [3819, 10464, 16618, 32768, 0], + [26870, 32467, 32693, 32768, 0], + [19041, 30831, 32347, 32768, 0], + [11794, 25211, 30016, 32768, 0], + [6888, 18019, 24970, 32768, 0], + [4370, 12363, 18992, 32768, 0], + [29578, 32670, 32744, 32768, 0], + [23159, 32007, 32613, 32768, 0], + [15315, 28669, 31676, 32768, 0], + [9298, 22607, 28782, 32768, 0], + [6144, 15913, 22968, 32768, 0], + [28110, 32499, 32669, 32768, 0], + [21574, 30937, 32015, 32768, 0], + [12759, 24818, 28727, 32768, 0], + [6545, 16761, 23042, 32768, 0], + [3649, 10597, 16833, 32768, 0], + [28163, 32552, 32728, 32768, 0], + [22101, 31469, 32464, 32768, 0], + [13160, 25472, 30143, 32768, 0], + [7303, 18684, 25468, 32768, 0], + [5241, 13975, 20955, 32768, 0], + [28400, 32631, 32744, 32768, 0], + [22104, 31793, 32603, 32768, 0], + [13557, 26571, 30846, 32768, 0], + [7749, 19861, 26675, 32768, 0], + [4873, 14030, 21234, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [9800, 17635, 21073, 32768, 0], + [26153, 31885, 32527, 32768, 0], + [15038, 27852, 31006, 32768, 0], + [8718, 20564, 26486, 32768, 0], + [5128, 14076, 20514, 32768, 0], + [2636, 7566, 11925, 32768, 0], + [27551, 32504, 32701, 32768, 0], + [18310, 30054, 32100, 32768, 0], + [10211, 23420, 29082, 32768, 0], + [6222, 16876, 23916, 32768, 0], + [3462, 9954, 15498, 32768, 0], + [29991, 32633, 32721, 32768, 0], + [19883, 30751, 32201, 32768, 0], + [11141, 24184, 29285, 32768, 0], + [6420, 16940, 23774, 32768, 0], + [3392, 9753, 15118, 32768, 0], + [28465, 32616, 32712, 32768, 0], + [19850, 30702, 32244, 32768, 0], + [10983, 24024, 29223, 32768, 0], + [6294, 16770, 23582, 32768, 0], + [3244, 9283, 14509, 32768, 0], + [30023, 32717, 32748, 32768, 0], + [22940, 32032, 32626, 32768, 0], + [14282, 27928, 31473, 32768, 0], + [8562, 21327, 27914, 32768, 0], + [4846, 13393, 19919, 32768, 0], + [29981, 32590, 32695, 32768, 0], + [20465, 30963, 32166, 32768, 0], + [11479, 23579, 28195, 32768, 0], + [5916, 15648, 22073, 32768, 0], + [3031, 8605, 13398, 32768, 0], + [31146, 32691, 32739, 32768, 0], + [23106, 31724, 32444, 32768, 0], + [13783, 26738, 30439, 32768, 0], + [7852, 19468, 25807, 32768, 0], + [3860, 11124, 16853, 32768, 0], + [31014, 32724, 32748, 32768, 0], + [23629, 32109, 32628, 32768, 0], + [14747, 28115, 31403, 32768, 0], + [8545, 21242, 27478, 32768, 0], + [4574, 12781, 19067, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [9185, 19694, 24688, 32768, 0], + [26081, 31985, 32621, 32768, 0], + [16015, 29000, 31787, 32768, 0], + [10542, 23690, 29206, 32768, 0], + [6732, 17945, 24677, 32768, 0], + [3916, 11039, 16722, 32768, 0], + [28224, 32566, 32744, 32768, 0], + [19100, 31138, 32485, 32768, 0], + [12528, 26620, 30879, 32768, 0], + [7741, 20277, 26885, 32768, 0], + [4566, 12845, 18990, 32768, 0], + [29933, 32593, 32718, 32768, 0], + [17670, 30333, 32155, 32768, 0], + [10385, 23600, 28909, 32768, 0], + [6243, 16236, 22407, 32768, 0], + [3976, 10389, 16017, 32768, 0], + [28377, 32561, 32738, 32768, 0], + [19366, 31175, 32482, 32768, 0], + [13327, 27175, 31094, 32768, 0], + [8258, 20769, 27143, 32768, 0], + [4703, 13198, 19527, 32768, 0], + [31086, 32706, 32748, 32768, 0], + [22853, 31902, 32583, 32768, 0], + [14759, 28186, 31419, 32768, 0], + [9284, 22382, 28348, 32768, 0], + [5585, 15192, 21868, 32768, 0], + [28291, 32652, 32746, 32768, 0], + [19849, 32107, 32571, 32768, 0], + [14834, 26818, 29214, 32768, 0], + [10306, 22594, 28672, 32768, 0], + [6615, 17384, 23384, 32768, 0], + [28947, 32604, 32745, 32768, 0], + [25625, 32289, 32646, 32768, 0], + [18758, 28672, 31403, 32768, 0], + [10017, 23430, 28523, 32768, 0], + [6862, 15269, 22131, 32768, 0], + [23933, 32509, 32739, 32768, 0], + [19927, 31495, 32631, 32768, 0], + [11903, 26023, 30621, 32768, 0], + [7026, 20094, 27252, 32768, 0], + [5998, 18106, 24437, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [4456, 11274, 15533, 32768, 0], + [21219, 29079, 31616, 32768, 0], + [11173, 23774, 28567, 32768, 0], + [7282, 18293, 24263, 32768, 0], + [4890, 13286, 19115, 32768, 0], + [1890, 5508, 8659, 32768, 0], + [26651, 32136, 32647, 32768, 0], + [14630, 28254, 31455, 32768, 0], + [8716, 21287, 27395, 32768, 0], + [5615, 15331, 22008, 32768, 0], + [2675, 7700, 12150, 32768, 0], + [29954, 32526, 32690, 32768, 0], + [16126, 28982, 31633, 32768, 0], + [9030, 21361, 27352, 32768, 0], + [5411, 14793, 21271, 32768, 0], + [2943, 8422, 13163, 32768, 0], + [29539, 32601, 32730, 32768, 0], + [18125, 30385, 32201, 32768, 0], + [10422, 24090, 29468, 32768, 0], + [6468, 17487, 24438, 32768, 0], + [2970, 8653, 13531, 32768, 0], + [30912, 32715, 32748, 32768, 0], + [20666, 31373, 32497, 32768, 0], + [12509, 26640, 30917, 32768, 0], + [8058, 20629, 27290, 32768, 0], + [4231, 12006, 18052, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [10202, 20633, 25484, 32768, 0], + [27336, 31445, 32352, 32768, 0], + [12420, 24384, 28552, 32768, 0], + [7648, 18115, 23856, 32768, 0], + [5662, 14341, 19902, 32768, 0], + [3611, 10328, 15390, 32768, 0], + [30945, 32616, 32736, 32768, 0], + [18682, 30505, 32253, 32768, 0], + [11513, 25336, 30203, 32768, 0], + [7449, 19452, 26148, 32768, 0], + [4482, 13051, 18886, 32768, 0], + [32022, 32690, 32747, 32768, 0], + [18578, 30501, 32146, 32768, 0], + [11249, 23368, 28631, 32768, 0], + [5645, 16958, 22158, 32768, 0], + [5009, 11444, 16637, 32768, 0], + [31357, 32710, 32748, 32768, 0], + [21552, 31494, 32504, 32768, 0], + [13891, 27677, 31340, 32768, 0], + [9051, 22098, 28172, 32768, 0], + [5190, 13377, 19486, 32768, 0], + [32364, 32740, 32748, 32768, 0], + [24839, 31907, 32551, 32768, 0], + [17160, 28779, 31696, 32768, 0], + [12452, 24137, 29602, 32768, 0], + [6165, 15389, 22477, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [2575, 7281, 11077, 32768, 0], + [14002, 20866, 25402, 32768, 0], + [6343, 15056, 19658, 32768, 0], + [4474, 11858, 17041, 32768, 0], + [2865, 8299, 12534, 32768, 0], + [1344, 3949, 6391, 32768, 0], + [24720, 31239, 32459, 32768, 0], + [12585, 25356, 29968, 32768, 0], + [7181, 18246, 24444, 32768, 0], + [5025, 13667, 19885, 32768, 0], + [2521, 7304, 11605, 32768, 0], + [29908, 32252, 32584, 32768, 0], + [17421, 29156, 31575, 32768, 0], + [9889, 22188, 27782, 32768, 0], + [5878, 15647, 22123, 32768, 0], + [2814, 8665, 13323, 32768, 0], + [30183, 32568, 32713, 32768, 0], + [18528, 30195, 32049, 32768, 0], + [10982, 24606, 29657, 32768, 0], + [6957, 18165, 25231, 32768, 0], + [3508, 10118, 15468, 32768, 0], + [31761, 32736, 32748, 32768, 0], + [21041, 31328, 32546, 32768, 0], + [12568, 26732, 31166, 32768, 0], + [8052, 20720, 27733, 32768, 0], + [4336, 12192, 18396, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + ], + [ + [ + [ + [7062, 16472, 22319, 32768, 0], + [24538, 32261, 32674, 32768, 0], + [13675, 28041, 31779, 32768, 0], + [8590, 20674, 27631, 32768, 0], + [5685, 14675, 22013, 32768, 0], + [3655, 9898, 15731, 32768, 0], + [26493, 32418, 32658, 32768, 0], + [16376, 29342, 32090, 32768, 0], + [10594, 22649, 28970, 32768, 0], + [8176, 17170, 24303, 32768, 0], + [5605, 12694, 19139, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [23888, 31902, 32542, 32768, 0], + [18612, 29687, 31987, 32768, 0], + [16245, 24852, 29249, 32768, 0], + [15765, 22608, 27559, 32768, 0], + [19895, 24699, 27510, 32768, 0], + [28401, 32212, 32457, 32768, 0], + [15274, 27825, 30980, 32768, 0], + [9364, 18128, 24332, 32768, 0], + [2283, 8193, 15082, 32768, 0], + [1228, 3972, 7881, 32768, 0], + [29455, 32469, 32620, 32768, 0], + [17981, 28245, 31388, 32768, 0], + [10921, 20098, 26240, 32768, 0], + [3743, 11829, 18657, 32768, 0], + [2374, 9593, 15715, 32768, 0], + [31068, 32466, 32635, 32768, 0], + [20321, 29572, 31971, 32768, 0], + [10771, 20255, 27119, 32768, 0], + [2795, 10410, 17361, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [9320, 22102, 27840, 32768, 0], + [27057, 32464, 32724, 32768, 0], + [16331, 30268, 32309, 32768, 0], + [10319, 23935, 29720, 32768, 0], + [6189, 16448, 24106, 32768, 0], + [3589, 10884, 18808, 32768, 0], + [29026, 32624, 32748, 32768, 0], + [19226, 31507, 32587, 32768, 0], + [12692, 26921, 31203, 32768, 0], + [7049, 19532, 27635, 32768, 0], + [7727, 15669, 23252, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [28056, 32625, 32748, 32768, 0], + [22383, 32075, 32669, 32768, 0], + [15417, 27098, 31749, 32768, 0], + [18127, 26493, 27190, 32768, 0], + [5461, 16384, 21845, 32768, 0], + [27982, 32091, 32584, 32768, 0], + [19045, 29868, 31972, 32768, 0], + [10397, 22266, 27932, 32768, 0], + [5990, 13697, 21500, 32768, 0], + [1792, 6912, 15104, 32768, 0], + [28198, 32501, 32718, 32768, 0], + [21534, 31521, 32569, 32768, 0], + [11109, 25217, 30017, 32768, 0], + [5671, 15124, 26151, 32768, 0], + [4681, 14043, 18725, 32768, 0], + [28688, 32580, 32741, 32768, 0], + [22576, 32079, 32661, 32768, 0], + [10627, 22141, 28340, 32768, 0], + [9362, 14043, 28087, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [7754, 16948, 22142, 32768, 0], + [25670, 32330, 32691, 32768, 0], + [15663, 29225, 31994, 32768, 0], + [9878, 23288, 29158, 32768, 0], + [6419, 17088, 24336, 32768, 0], + [3859, 11003, 17039, 32768, 0], + [27562, 32595, 32725, 32768, 0], + [17575, 30588, 32399, 32768, 0], + [10819, 24838, 30309, 32768, 0], + [7124, 18686, 25916, 32768, 0], + [4479, 12688, 19340, 32768, 0], + [28385, 32476, 32673, 32768, 0], + [15306, 29005, 31938, 32768, 0], + [8937, 21615, 28322, 32768, 0], + [5982, 15603, 22786, 32768, 0], + [3620, 10267, 16136, 32768, 0], + [27280, 32464, 32667, 32768, 0], + [15607, 29160, 32004, 32768, 0], + [9091, 22135, 28740, 32768, 0], + [6232, 16632, 24020, 32768, 0], + [4047, 11377, 17672, 32768, 0], + [29220, 32630, 32718, 32768, 0], + [19650, 31220, 32462, 32768, 0], + [13050, 26312, 30827, 32768, 0], + [9228, 20870, 27468, 32768, 0], + [6146, 15149, 21971, 32768, 0], + [30169, 32481, 32623, 32768, 0], + [17212, 29311, 31554, 32768, 0], + [9911, 21311, 26882, 32768, 0], + [4487, 13314, 20372, 32768, 0], + [2570, 7772, 12889, 32768, 0], + [30924, 32613, 32708, 32768, 0], + [19490, 30206, 32107, 32768, 0], + [11232, 23998, 29276, 32768, 0], + [6769, 17955, 25035, 32768, 0], + [4398, 12623, 19214, 32768, 0], + [30609, 32627, 32722, 32768, 0], + [19370, 30582, 32287, 32768, 0], + [10457, 23619, 29409, 32768, 0], + [6443, 17637, 24834, 32768, 0], + [4645, 13236, 20106, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8626, 20271, 26216, 32768, 0], + [26707, 32406, 32711, 32768, 0], + [16999, 30329, 32286, 32768, 0], + [11445, 25123, 30286, 32768, 0], + [6411, 18828, 25601, 32768, 0], + [6801, 12458, 20248, 32768, 0], + [29918, 32682, 32748, 32768, 0], + [20649, 31739, 32618, 32768, 0], + [12879, 27773, 31581, 32768, 0], + [7896, 21751, 28244, 32768, 0], + [5260, 14870, 23698, 32768, 0], + [29252, 32593, 32731, 32768, 0], + [17072, 30460, 32294, 32768, 0], + [10653, 24143, 29365, 32768, 0], + [6536, 17490, 23983, 32768, 0], + [4929, 13170, 20085, 32768, 0], + [28137, 32518, 32715, 32768, 0], + [18171, 30784, 32407, 32768, 0], + [11437, 25436, 30459, 32768, 0], + [7252, 18534, 26176, 32768, 0], + [4126, 13353, 20978, 32768, 0], + [31162, 32726, 32748, 32768, 0], + [23017, 32222, 32701, 32768, 0], + [15629, 29233, 32046, 32768, 0], + [9387, 22621, 29480, 32768, 0], + [6922, 17616, 25010, 32768, 0], + [28838, 32265, 32614, 32768, 0], + [19701, 30206, 31920, 32768, 0], + [11214, 22410, 27933, 32768, 0], + [5320, 14177, 23034, 32768, 0], + [5049, 12881, 17827, 32768, 0], + [27484, 32471, 32734, 32768, 0], + [21076, 31526, 32561, 32768, 0], + [12707, 26303, 31211, 32768, 0], + [8169, 21722, 28219, 32768, 0], + [6045, 19406, 27042, 32768, 0], + [27753, 32572, 32745, 32768, 0], + [20832, 31878, 32653, 32768, 0], + [13250, 27356, 31674, 32768, 0], + [7718, 21508, 29858, 32768, 0], + [7209, 18350, 25559, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [7876, 16901, 21741, 32768, 0], + [24001, 31898, 32625, 32768, 0], + [14529, 27959, 31451, 32768, 0], + [8273, 20818, 27258, 32768, 0], + [5278, 14673, 21510, 32768, 0], + [2983, 8843, 14039, 32768, 0], + [28016, 32574, 32732, 32768, 0], + [17471, 30306, 32301, 32768, 0], + [10224, 24063, 29728, 32768, 0], + [6602, 17954, 25052, 32768, 0], + [4002, 11585, 17759, 32768, 0], + [30190, 32634, 32739, 32768, 0], + [17497, 30282, 32270, 32768, 0], + [10229, 23729, 29538, 32768, 0], + [6344, 17211, 24440, 32768, 0], + [3849, 11189, 17108, 32768, 0], + [28570, 32583, 32726, 32768, 0], + [17521, 30161, 32238, 32768, 0], + [10153, 23565, 29378, 32768, 0], + [6455, 17341, 24443, 32768, 0], + [3907, 11042, 17024, 32768, 0], + [30689, 32715, 32748, 32768, 0], + [21546, 31840, 32610, 32768, 0], + [13547, 27581, 31459, 32768, 0], + [8912, 21757, 28309, 32768, 0], + [5548, 15080, 22046, 32768, 0], + [30783, 32540, 32685, 32768, 0], + [17540, 29528, 31668, 32768, 0], + [10160, 21468, 26783, 32768, 0], + [4724, 13393, 20054, 32768, 0], + [2702, 8174, 13102, 32768, 0], + [31648, 32686, 32742, 32768, 0], + [20954, 31094, 32337, 32768, 0], + [12420, 25698, 30179, 32768, 0], + [7304, 19320, 26248, 32768, 0], + [4366, 12261, 18864, 32768, 0], + [31581, 32723, 32748, 32768, 0], + [21373, 31586, 32525, 32768, 0], + [12744, 26625, 30885, 32768, 0], + [7431, 20322, 26950, 32768, 0], + [4692, 13323, 20111, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [7833, 18369, 24095, 32768, 0], + [26650, 32273, 32702, 32768, 0], + [16371, 29961, 32191, 32768, 0], + [11055, 24082, 29629, 32768, 0], + [6892, 18644, 25400, 32768, 0], + [5006, 13057, 19240, 32768, 0], + [29834, 32666, 32748, 32768, 0], + [19577, 31335, 32570, 32768, 0], + [12253, 26509, 31122, 32768, 0], + [7991, 20772, 27711, 32768, 0], + [5677, 15910, 23059, 32768, 0], + [30109, 32532, 32720, 32768, 0], + [16747, 30166, 32252, 32768, 0], + [10134, 23542, 29184, 32768, 0], + [5791, 16176, 23556, 32768, 0], + [4362, 10414, 17284, 32768, 0], + [29492, 32626, 32748, 32768, 0], + [19894, 31402, 32525, 32768, 0], + [12942, 27071, 30869, 32768, 0], + [8346, 21216, 27405, 32768, 0], + [6572, 17087, 23859, 32768, 0], + [32035, 32735, 32748, 32768, 0], + [22957, 31838, 32618, 32768, 0], + [14724, 28572, 31772, 32768, 0], + [10364, 23999, 29553, 32768, 0], + [7004, 18433, 25655, 32768, 0], + [27528, 32277, 32681, 32768, 0], + [16959, 31171, 32096, 32768, 0], + [10486, 23593, 27962, 32768, 0], + [8192, 16384, 23211, 32768, 0], + [8937, 17873, 20852, 32768, 0], + [27715, 32002, 32615, 32768, 0], + [15073, 29491, 31676, 32768, 0], + [11264, 24576, 28672, 32768, 0], + [2341, 18725, 23406, 32768, 0], + [7282, 18204, 25486, 32768, 0], + [28547, 32213, 32657, 32768, 0], + [20788, 29773, 32239, 32768, 0], + [6780, 21469, 30508, 32768, 0], + [5958, 14895, 23831, 32768, 0], + [16384, 21845, 27307, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [5992, 14304, 19765, 32768, 0], + [22612, 31238, 32456, 32768, 0], + [13456, 27162, 31087, 32768, 0], + [8001, 20062, 26504, 32768, 0], + [5168, 14105, 20764, 32768, 0], + [2632, 7771, 12385, 32768, 0], + [27034, 32344, 32709, 32768, 0], + [15850, 29415, 31997, 32768, 0], + [9494, 22776, 28841, 32768, 0], + [6151, 16830, 23969, 32768, 0], + [3461, 10039, 15722, 32768, 0], + [30134, 32569, 32731, 32768, 0], + [15638, 29422, 31945, 32768, 0], + [9150, 21865, 28218, 32768, 0], + [5647, 15719, 22676, 32768, 0], + [3402, 9772, 15477, 32768, 0], + [28530, 32586, 32735, 32768, 0], + [17139, 30298, 32292, 32768, 0], + [10200, 24039, 29685, 32768, 0], + [6419, 17674, 24786, 32768, 0], + [3544, 10225, 15824, 32768, 0], + [31333, 32726, 32748, 32768, 0], + [20618, 31487, 32544, 32768, 0], + [12901, 27217, 31232, 32768, 0], + [8624, 21734, 28171, 32768, 0], + [5104, 14191, 20748, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [11206, 21090, 26561, 32768, 0], + [28759, 32279, 32671, 32768, 0], + [14171, 27952, 31569, 32768, 0], + [9743, 22907, 29141, 32768, 0], + [6871, 17886, 24868, 32768, 0], + [4960, 13152, 19315, 32768, 0], + [31077, 32661, 32748, 32768, 0], + [19400, 31195, 32515, 32768, 0], + [12752, 26858, 31040, 32768, 0], + [8370, 22098, 28591, 32768, 0], + [5457, 15373, 22298, 32768, 0], + [31697, 32706, 32748, 32768, 0], + [17860, 30657, 32333, 32768, 0], + [12510, 24812, 29261, 32768, 0], + [6180, 19124, 24722, 32768, 0], + [5041, 13548, 17959, 32768, 0], + [31552, 32716, 32748, 32768, 0], + [21908, 31769, 32623, 32768, 0], + [14470, 28201, 31565, 32768, 0], + [9493, 22982, 28608, 32768, 0], + [6858, 17240, 24137, 32768, 0], + [32543, 32752, 32756, 32768, 0], + [24286, 32097, 32666, 32768, 0], + [15958, 29217, 32024, 32768, 0], + [10207, 24234, 29958, 32768, 0], + [6929, 18305, 25652, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + [ + [ + [4137, 10847, 15682, 32768, 0], + [17824, 27001, 30058, 32768, 0], + [10204, 22796, 28291, 32768, 0], + [6076, 15935, 22125, 32768, 0], + [3852, 10937, 16816, 32768, 0], + [2252, 6324, 10131, 32768, 0], + [25840, 32016, 32662, 32768, 0], + [15109, 28268, 31531, 32768, 0], + [9385, 22231, 28340, 32768, 0], + [6082, 16672, 23479, 32768, 0], + [3318, 9427, 14681, 32768, 0], + [30594, 32574, 32718, 32768, 0], + [16836, 29552, 31859, 32768, 0], + [9556, 22542, 28356, 32768, 0], + [6305, 16725, 23540, 32768, 0], + [3376, 9895, 15184, 32768, 0], + [29383, 32617, 32745, 32768, 0], + [18891, 30809, 32401, 32768, 0], + [11688, 25942, 30687, 32768, 0], + [7468, 19469, 26651, 32768, 0], + [3909, 11358, 17012, 32768, 0], + [31564, 32736, 32748, 32768, 0], + [20906, 31611, 32600, 32768, 0], + [13191, 27621, 31537, 32768, 0], + [8768, 22029, 28676, 32768, 0], + [5079, 14109, 20906, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + ], +]; + +pub static DEFAULT_COEFF_BASE_EOB_CDF: [[[[[u16; 4]; 4]; 2]; 5]; 4] = [ + [ + [ + [[17837, 29055, 32768, 0], [29600, 31446, 32768, 0], [30844, 31878, 32768, 0], [24926, 28948, 32768, 0]], + [[21365, 30026, 32768, 0], [30512, 32423, 32768, 0], [31658, 32621, 32768, 0], [29630, 31881, 32768, 0]], + ], + [ + [[5717, 26477, 32768, 0], [30491, 31703, 32768, 0], [31550, 32158, 32768, 0], [29648, 31491, 32768, 0]], + [[12608, 27820, 32768, 0], [30680, 32225, 32768, 0], [30809, 32335, 32768, 0], [31299, 32423, 32768, 0]], + ], + [ + [[1786, 12612, 32768, 0], [30663, 31625, 32768, 0], [32339, 32468, 32768, 0], [31148, 31833, 32768, 0]], + [[18857, 23865, 32768, 0], [31428, 32428, 32768, 0], [31744, 32373, 32768, 0], [31775, 32526, 32768, 0]], + ], + [ + [[1787, 2532, 32768, 0], [30832, 31662, 32768, 0], [31824, 32682, 32768, 0], [32133, 32569, 32768, 0]], + [[13751, 22235, 32768, 0], [32089, 32409, 32768, 0], [27084, 27920, 32768, 0], [29291, 32594, 32768, 0]], + ], + [ + [[1725, 3449, 32768, 0], [31102, 31935, 32768, 0], [32457, 32613, 32768, 0], [32412, 32649, 32768, 0]], + [[10923, 21845, 32768, 0], [10923, 21845, 32768, 0], [10923, 21845, 32768, 0], [10923, 21845, 32768, 0]], + ], + ], + [ + [ + [[17560, 29888, 32768, 0], [29671, 31549, 32768, 0], [31007, 32056, 32768, 0], [27286, 30006, 32768, 0]], + [[26594, 31212, 32768, 0], [31208, 32582, 32768, 0], [31835, 32637, 32768, 0], [30595, 32206, 32768, 0]], + ], + [ + [[15239, 29932, 32768, 0], [31315, 32095, 32768, 0], [32130, 32434, 32768, 0], [30864, 31996, 32768, 0]], + [[26279, 30968, 32768, 0], [31142, 32495, 32768, 0], [31713, 32540, 32768, 0], [31929, 32594, 32768, 0]], + ], + [ + [[2644, 25198, 32768, 0], [32038, 32451, 32768, 0], [32639, 32695, 32768, 0], [32166, 32518, 32768, 0]], + [[17187, 27668, 32768, 0], [31714, 32550, 32768, 0], [32283, 32678, 32768, 0], [31930, 32563, 32768, 0]], + ], + [ + [[1044, 2257, 32768, 0], [30755, 31923, 32768, 0], [32208, 32693, 32768, 0], [32244, 32615, 32768, 0]], + [[21317, 26207, 32768, 0], [29133, 30868, 32768, 0], [29311, 31231, 32768, 0], [29657, 31087, 32768, 0]], + ], + [ + [[478, 1834, 32768, 0], [31005, 31987, 32768, 0], [32317, 32724, 32768, 0], [30865, 32648, 32768, 0]], + [[10923, 21845, 32768, 0], [10923, 21845, 32768, 0], [10923, 21845, 32768, 0], [10923, 21845, 32768, 0]], + ], + ], + [ + [ + [[20092, 30774, 32768, 0], [30695, 32020, 32768, 0], [31131, 32103, 32768, 0], [28666, 30870, 32768, 0]], + [[27258, 31095, 32768, 0], [31804, 32623, 32768, 0], [31763, 32528, 32768, 0], [31438, 32506, 32768, 0]], + ], + [ + [[18049, 30489, 32768, 0], [31706, 32286, 32768, 0], [32163, 32473, 32768, 0], [31550, 32184, 32768, 0]], + [[27116, 30842, 32768, 0], [31971, 32598, 32768, 0], [32088, 32576, 32768, 0], [32067, 32664, 32768, 0]], + ], + [ + [[12854, 29093, 32768, 0], [32272, 32558, 32768, 0], [32667, 32729, 32768, 0], [32306, 32585, 32768, 0]], + [[25476, 30366, 32768, 0], [32169, 32687, 32768, 0], [32479, 32689, 32768, 0], [31673, 32634, 32768, 0]], + ], + [ + [[2809, 19301, 32768, 0], [32205, 32622, 32768, 0], [32338, 32730, 32768, 0], [31786, 32616, 32768, 0]], + [[22737, 29105, 32768, 0], [30810, 32362, 32768, 0], [30014, 32627, 32768, 0], [30528, 32574, 32768, 0]], + ], + [ + [[935, 3382, 32768, 0], [30789, 31909, 32768, 0], [32466, 32756, 32768, 0], [30860, 32513, 32768, 0]], + [[10923, 21845, 32768, 0], [10923, 21845, 32768, 0], [10923, 21845, 32768, 0], [10923, 21845, 32768, 0]], + ], + ], + [ + [ + [[22497, 31198, 32768, 0], [31715, 32495, 32768, 0], [31606, 32337, 32768, 0], [30388, 31990, 32768, 0]], + [[27877, 31584, 32768, 0], [32170, 32728, 32768, 0], [32155, 32688, 32768, 0], [32219, 32702, 32768, 0]], + ], + [ + [[21457, 31043, 32768, 0], [31951, 32483, 32768, 0], [32153, 32562, 32768, 0], [31473, 32215, 32768, 0]], + [[27558, 31151, 32768, 0], [32020, 32640, 32768, 0], [32097, 32575, 32768, 0], [32242, 32719, 32768, 0]], + ], + [ + [[19980, 30591, 32768, 0], [32219, 32597, 32768, 0], [32581, 32706, 32768, 0], [31803, 32287, 32768, 0]], + [[26473, 30507, 32768, 0], [32431, 32723, 32768, 0], [32196, 32611, 32768, 0], [31588, 32528, 32768, 0]], + ], + [ + [[24647, 30463, 32768, 0], [32412, 32695, 32768, 0], [32468, 32720, 32768, 0], [31269, 32523, 32768, 0]], + [[28482, 31505, 32768, 0], [32152, 32701, 32768, 0], [31732, 32598, 32768, 0], [31767, 32712, 32768, 0]], + ], + [ + [[12358, 24977, 32768, 0], [31331, 32385, 32768, 0], [32634, 32756, 32768, 0], [30411, 32548, 32768, 0]], + [[10923, 21845, 32768, 0], [10923, 21845, 32768, 0], [10923, 21845, 32768, 0], [10923, 21845, 32768, 0]], + ], + ], +]; + +pub static DEFAULT_COEFF_BR_CDF: [[[[[u16; 5]; 21]; 2]; 5]; 4] = [ + [ + [ + [ + [14298, 20718, 24174, 32768, 0], + [12536, 19601, 23789, 32768, 0], + [8712, 15051, 19503, 32768, 0], + [6170, 11327, 15434, 32768, 0], + [4742, 8926, 12538, 32768, 0], + [3803, 7317, 10546, 32768, 0], + [1696, 3317, 4871, 32768, 0], + [14392, 19951, 22756, 32768, 0], + [15978, 23218, 26818, 32768, 0], + [12187, 19474, 23889, 32768, 0], + [9176, 15640, 20259, 32768, 0], + [7068, 12655, 17028, 32768, 0], + [5656, 10442, 14472, 32768, 0], + [2580, 4992, 7244, 32768, 0], + [12136, 18049, 21426, 32768, 0], + [13784, 20721, 24481, 32768, 0], + [10836, 17621, 21900, 32768, 0], + [8372, 14444, 18847, 32768, 0], + [6523, 11779, 16000, 32768, 0], + [5337, 9898, 13760, 32768, 0], + [3034, 5860, 8462, 32768, 0], + ], + [ + [15967, 22905, 26286, 32768, 0], + [13534, 20654, 24579, 32768, 0], + [9504, 16092, 20535, 32768, 0], + [6975, 12568, 16903, 32768, 0], + [5364, 10091, 14020, 32768, 0], + [4357, 8370, 11857, 32768, 0], + [2506, 4934, 7218, 32768, 0], + [23032, 28815, 30936, 32768, 0], + [19540, 26704, 29719, 32768, 0], + [15158, 22969, 27097, 32768, 0], + [11408, 18865, 23650, 32768, 0], + [8885, 15448, 20250, 32768, 0], + [7108, 12853, 17416, 32768, 0], + [4231, 8041, 11480, 32768, 0], + [19823, 26490, 29156, 32768, 0], + [18890, 25929, 28932, 32768, 0], + [15660, 23491, 27433, 32768, 0], + [12147, 19776, 24488, 32768, 0], + [9728, 16774, 21649, 32768, 0], + [7919, 14277, 19066, 32768, 0], + [5440, 10170, 14185, 32768, 0], + ], + ], + [ + [ + [14406, 20862, 24414, 32768, 0], + [11824, 18907, 23109, 32768, 0], + [8257, 14393, 18803, 32768, 0], + [5860, 10747, 14778, 32768, 0], + [4475, 8486, 11984, 32768, 0], + [3606, 6954, 10043, 32768, 0], + [1736, 3410, 5048, 32768, 0], + [14430, 20046, 22882, 32768, 0], + [15593, 22899, 26709, 32768, 0], + [12102, 19368, 23811, 32768, 0], + [9059, 15584, 20262, 32768, 0], + [6999, 12603, 17048, 32768, 0], + [5684, 10497, 14553, 32768, 0], + [2822, 5438, 7862, 32768, 0], + [15785, 21585, 24359, 32768, 0], + [18347, 25229, 28266, 32768, 0], + [14974, 22487, 26389, 32768, 0], + [11423, 18681, 23271, 32768, 0], + [8863, 15350, 20008, 32768, 0], + [7153, 12852, 17278, 32768, 0], + [3707, 7036, 9982, 32768, 0], + ], + [ + [15460, 21696, 25469, 32768, 0], + [12170, 19249, 23191, 32768, 0], + [8723, 15027, 19332, 32768, 0], + [6428, 11704, 15874, 32768, 0], + [4922, 9292, 13052, 32768, 0], + [4139, 7695, 11010, 32768, 0], + [2291, 4508, 6598, 32768, 0], + [19856, 26920, 29828, 32768, 0], + [17923, 25289, 28792, 32768, 0], + [14278, 21968, 26297, 32768, 0], + [10910, 18136, 22950, 32768, 0], + [8423, 14815, 19627, 32768, 0], + [6771, 12283, 16774, 32768, 0], + [4074, 7750, 11081, 32768, 0], + [19852, 26074, 28672, 32768, 0], + [19371, 26110, 28989, 32768, 0], + [16265, 23873, 27663, 32768, 0], + [12758, 20378, 24952, 32768, 0], + [10095, 17098, 21961, 32768, 0], + [8250, 14628, 19451, 32768, 0], + [5205, 9745, 13622, 32768, 0], + ], + ], + [ + [ + [10563, 16233, 19763, 32768, 0], + [9794, 16022, 19804, 32768, 0], + [6750, 11945, 15759, 32768, 0], + [4963, 9186, 12752, 32768, 0], + [3845, 7435, 10627, 32768, 0], + [3051, 6085, 8834, 32768, 0], + [1311, 2596, 3830, 32768, 0], + [11246, 16404, 19689, 32768, 0], + [12315, 18911, 22731, 32768, 0], + [10557, 17095, 21289, 32768, 0], + [8136, 14006, 18249, 32768, 0], + [6348, 11474, 15565, 32768, 0], + [5196, 9655, 13400, 32768, 0], + [2349, 4526, 6587, 32768, 0], + [13337, 18730, 21569, 32768, 0], + [19306, 26071, 28882, 32768, 0], + [15952, 23540, 27254, 32768, 0], + [12409, 19934, 24430, 32768, 0], + [9760, 16706, 21389, 32768, 0], + [8004, 14220, 18818, 32768, 0], + [4138, 7794, 10961, 32768, 0], + ], + [ + [10870, 16684, 20949, 32768, 0], + [9664, 15230, 18680, 32768, 0], + [6886, 12109, 15408, 32768, 0], + [4825, 8900, 12305, 32768, 0], + [3630, 7162, 10314, 32768, 0], + [3036, 6429, 9387, 32768, 0], + [1671, 3296, 4940, 32768, 0], + [13819, 19159, 23026, 32768, 0], + [11984, 19108, 23120, 32768, 0], + [10690, 17210, 21663, 32768, 0], + [7984, 14154, 18333, 32768, 0], + [6868, 12294, 16124, 32768, 0], + [5274, 8994, 12868, 32768, 0], + [2988, 5771, 8424, 32768, 0], + [19736, 26647, 29141, 32768, 0], + [18933, 26070, 28984, 32768, 0], + [15779, 23048, 27200, 32768, 0], + [12638, 20061, 24532, 32768, 0], + [10692, 17545, 22220, 32768, 0], + [9217, 15251, 20054, 32768, 0], + [5078, 9284, 12594, 32768, 0], + ], + ], + [ + [ + [2331, 3662, 5244, 32768, 0], + [2891, 4771, 6145, 32768, 0], + [4598, 7623, 9729, 32768, 0], + [3520, 6845, 9199, 32768, 0], + [3417, 6119, 9324, 32768, 0], + [2601, 5412, 7385, 32768, 0], + [600, 1173, 1744, 32768, 0], + [7672, 13286, 17469, 32768, 0], + [4232, 7792, 10793, 32768, 0], + [2915, 5317, 7397, 32768, 0], + [2318, 4356, 6152, 32768, 0], + [2127, 4000, 5554, 32768, 0], + [1850, 3478, 5275, 32768, 0], + [977, 1933, 2843, 32768, 0], + [18280, 24387, 27989, 32768, 0], + [15852, 22671, 26185, 32768, 0], + [13845, 20951, 24789, 32768, 0], + [11055, 17966, 22129, 32768, 0], + [9138, 15422, 19801, 32768, 0], + [7454, 13145, 17456, 32768, 0], + [3370, 6393, 9013, 32768, 0], + ], + [ + [5842, 9229, 10838, 32768, 0], + [2313, 3491, 4276, 32768, 0], + [2998, 6104, 7496, 32768, 0], + [2420, 7447, 9868, 32768, 0], + [3034, 8495, 10923, 32768, 0], + [4076, 8937, 10975, 32768, 0], + [1086, 2370, 3299, 32768, 0], + [9714, 17254, 20444, 32768, 0], + [8543, 13698, 17123, 32768, 0], + [4918, 9007, 11910, 32768, 0], + [4129, 7532, 10553, 32768, 0], + [2364, 5533, 8058, 32768, 0], + [1834, 3546, 5563, 32768, 0], + [1473, 2908, 4133, 32768, 0], + [15405, 21193, 25619, 32768, 0], + [15691, 21952, 26561, 32768, 0], + [12962, 19194, 24165, 32768, 0], + [10272, 17855, 22129, 32768, 0], + [8588, 15270, 20718, 32768, 0], + [8682, 14669, 19500, 32768, 0], + [4870, 9636, 13205, 32768, 0], + ], + ], + [ + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + ], + [ + [ + [ + [14995, 21341, 24749, 32768, 0], + [13158, 20289, 24601, 32768, 0], + [8941, 15326, 19876, 32768, 0], + [6297, 11541, 15807, 32768, 0], + [4817, 9029, 12776, 32768, 0], + [3731, 7273, 10627, 32768, 0], + [1847, 3617, 5354, 32768, 0], + [14472, 19659, 22343, 32768, 0], + [16806, 24162, 27533, 32768, 0], + [12900, 20404, 24713, 32768, 0], + [9411, 16112, 20797, 32768, 0], + [7056, 12697, 17148, 32768, 0], + [5544, 10339, 14460, 32768, 0], + [2954, 5704, 8319, 32768, 0], + [12464, 18071, 21354, 32768, 0], + [15482, 22528, 26034, 32768, 0], + [12070, 19269, 23624, 32768, 0], + [8953, 15406, 20106, 32768, 0], + [7027, 12730, 17220, 32768, 0], + [5887, 10913, 15140, 32768, 0], + [3793, 7278, 10447, 32768, 0], + ], + [ + [15571, 22232, 25749, 32768, 0], + [14506, 21575, 25374, 32768, 0], + [10189, 17089, 21569, 32768, 0], + [7316, 13301, 17915, 32768, 0], + [5783, 10912, 15190, 32768, 0], + [4760, 9155, 13088, 32768, 0], + [2993, 5966, 8774, 32768, 0], + [23424, 28903, 30778, 32768, 0], + [20775, 27666, 30290, 32768, 0], + [16474, 24410, 28299, 32768, 0], + [12471, 20180, 24987, 32768, 0], + [9410, 16487, 21439, 32768, 0], + [7536, 13614, 18529, 32768, 0], + [5048, 9586, 13549, 32768, 0], + [21090, 27290, 29756, 32768, 0], + [20796, 27402, 30026, 32768, 0], + [17819, 25485, 28969, 32768, 0], + [13860, 21909, 26462, 32768, 0], + [11002, 18494, 23529, 32768, 0], + [8953, 15929, 20897, 32768, 0], + [6448, 11918, 16454, 32768, 0], + ], + ], + [ + [ + [15999, 22208, 25449, 32768, 0], + [13050, 19988, 24122, 32768, 0], + [8594, 14864, 19378, 32768, 0], + [6033, 11079, 15238, 32768, 0], + [4554, 8683, 12347, 32768, 0], + [3672, 7139, 10337, 32768, 0], + [1900, 3771, 5576, 32768, 0], + [15788, 21340, 23949, 32768, 0], + [16825, 24235, 27758, 32768, 0], + [12873, 20402, 24810, 32768, 0], + [9590, 16363, 21094, 32768, 0], + [7352, 13209, 17733, 32768, 0], + [5960, 10989, 15184, 32768, 0], + [3232, 6234, 9007, 32768, 0], + [15761, 20716, 23224, 32768, 0], + [19318, 25989, 28759, 32768, 0], + [15529, 23094, 26929, 32768, 0], + [11662, 18989, 23641, 32768, 0], + [8955, 15568, 20366, 32768, 0], + [7281, 13106, 17708, 32768, 0], + [4248, 8059, 11440, 32768, 0], + ], + [ + [14899, 21217, 24503, 32768, 0], + [13519, 20283, 24047, 32768, 0], + [9429, 15966, 20365, 32768, 0], + [6700, 12355, 16652, 32768, 0], + [5088, 9704, 13716, 32768, 0], + [4243, 8154, 11731, 32768, 0], + [2702, 5364, 7861, 32768, 0], + [22745, 28388, 30454, 32768, 0], + [20235, 27146, 29922, 32768, 0], + [15896, 23715, 27637, 32768, 0], + [11840, 19350, 24131, 32768, 0], + [9122, 15932, 20880, 32768, 0], + [7488, 13581, 18362, 32768, 0], + [5114, 9568, 13370, 32768, 0], + [20845, 26553, 28932, 32768, 0], + [20981, 27372, 29884, 32768, 0], + [17781, 25335, 28785, 32768, 0], + [13760, 21708, 26297, 32768, 0], + [10975, 18415, 23365, 32768, 0], + [9045, 15789, 20686, 32768, 0], + [6130, 11199, 15423, 32768, 0], + ], + ], + [ + [ + [13549, 19724, 23158, 32768, 0], + [11844, 18382, 22246, 32768, 0], + [7919, 13619, 17773, 32768, 0], + [5486, 10143, 13946, 32768, 0], + [4166, 7983, 11324, 32768, 0], + [3364, 6506, 9427, 32768, 0], + [1598, 3160, 4674, 32768, 0], + [15281, 20979, 23781, 32768, 0], + [14939, 22119, 25952, 32768, 0], + [11363, 18407, 22812, 32768, 0], + [8609, 14857, 19370, 32768, 0], + [6737, 12184, 16480, 32768, 0], + [5506, 10263, 14262, 32768, 0], + [2990, 5786, 8380, 32768, 0], + [20249, 25253, 27417, 32768, 0], + [21070, 27518, 30001, 32768, 0], + [16854, 24469, 28074, 32768, 0], + [12864, 20486, 25000, 32768, 0], + [9962, 16978, 21778, 32768, 0], + [8074, 14338, 19048, 32768, 0], + [4494, 8479, 11906, 32768, 0], + ], + [ + [13960, 19617, 22829, 32768, 0], + [11150, 17341, 21228, 32768, 0], + [7150, 12964, 17190, 32768, 0], + [5331, 10002, 13867, 32768, 0], + [4167, 7744, 11057, 32768, 0], + [3480, 6629, 9646, 32768, 0], + [1883, 3784, 5686, 32768, 0], + [18752, 25660, 28912, 32768, 0], + [16968, 24586, 28030, 32768, 0], + [13520, 21055, 25313, 32768, 0], + [10453, 17626, 22280, 32768, 0], + [8386, 14505, 19116, 32768, 0], + [6742, 12595, 17008, 32768, 0], + [4273, 8140, 11499, 32768, 0], + [22120, 27827, 30233, 32768, 0], + [20563, 27358, 29895, 32768, 0], + [17076, 24644, 28153, 32768, 0], + [13362, 20942, 25309, 32768, 0], + [10794, 17965, 22695, 32768, 0], + [9014, 15652, 20319, 32768, 0], + [5708, 10512, 14497, 32768, 0], + ], + ], + [ + [ + [5705, 10930, 15725, 32768, 0], + [7946, 12765, 16115, 32768, 0], + [6801, 12123, 16226, 32768, 0], + [5462, 10135, 14200, 32768, 0], + [4189, 8011, 11507, 32768, 0], + [3191, 6229, 9408, 32768, 0], + [1057, 2137, 3212, 32768, 0], + [10018, 17067, 21491, 32768, 0], + [7380, 12582, 16453, 32768, 0], + [6068, 10845, 14339, 32768, 0], + [5098, 9198, 12555, 32768, 0], + [4312, 8010, 11119, 32768, 0], + [3700, 6966, 9781, 32768, 0], + [1693, 3326, 4887, 32768, 0], + [18757, 24930, 27774, 32768, 0], + [17648, 24596, 27817, 32768, 0], + [14707, 22052, 26026, 32768, 0], + [11720, 18852, 23292, 32768, 0], + [9357, 15952, 20525, 32768, 0], + [7810, 13753, 18210, 32768, 0], + [3879, 7333, 10328, 32768, 0], + ], + [ + [8278, 13242, 15922, 32768, 0], + [10547, 15867, 18919, 32768, 0], + [9106, 15842, 20609, 32768, 0], + [6833, 13007, 17218, 32768, 0], + [4811, 9712, 13923, 32768, 0], + [3985, 7352, 11128, 32768, 0], + [1688, 3458, 5262, 32768, 0], + [12951, 21861, 26510, 32768, 0], + [9788, 16044, 20276, 32768, 0], + [6309, 11244, 14870, 32768, 0], + [5183, 9349, 12566, 32768, 0], + [4389, 8229, 11492, 32768, 0], + [3633, 6945, 10620, 32768, 0], + [3600, 6847, 9907, 32768, 0], + [21748, 28137, 30255, 32768, 0], + [19436, 26581, 29560, 32768, 0], + [16359, 24201, 27953, 32768, 0], + [13961, 21693, 25871, 32768, 0], + [11544, 18686, 23322, 32768, 0], + [9372, 16462, 20952, 32768, 0], + [6138, 11210, 15390, 32768, 0], + ], + ], + [ + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + ], + [ + [ + [ + [16138, 22223, 25509, 32768, 0], + [15347, 22430, 26332, 32768, 0], + [9614, 16736, 21332, 32768, 0], + [6600, 12275, 16907, 32768, 0], + [4811, 9424, 13547, 32768, 0], + [3748, 7809, 11420, 32768, 0], + [2254, 4587, 6890, 32768, 0], + [15196, 20284, 23177, 32768, 0], + [18317, 25469, 28451, 32768, 0], + [13918, 21651, 25842, 32768, 0], + [10052, 17150, 21995, 32768, 0], + [7499, 13630, 18587, 32768, 0], + [6158, 11417, 16003, 32768, 0], + [4014, 7785, 11252, 32768, 0], + [15048, 21067, 24384, 32768, 0], + [18202, 25346, 28553, 32768, 0], + [14302, 22019, 26356, 32768, 0], + [10839, 18139, 23166, 32768, 0], + [8715, 15744, 20806, 32768, 0], + [7536, 13576, 18544, 32768, 0], + [5413, 10335, 14498, 32768, 0], + ], + [ + [17394, 24501, 27895, 32768, 0], + [15889, 23420, 27185, 32768, 0], + [11561, 19133, 23870, 32768, 0], + [8285, 14812, 19844, 32768, 0], + [6496, 12043, 16550, 32768, 0], + [4771, 9574, 13677, 32768, 0], + [3603, 6830, 10144, 32768, 0], + [21656, 27704, 30200, 32768, 0], + [21324, 27915, 30511, 32768, 0], + [17327, 25336, 28997, 32768, 0], + [13417, 21381, 26033, 32768, 0], + [10132, 17425, 22338, 32768, 0], + [8580, 15016, 19633, 32768, 0], + [5694, 11477, 16411, 32768, 0], + [24116, 29780, 31450, 32768, 0], + [23853, 29695, 31591, 32768, 0], + [20085, 27614, 30428, 32768, 0], + [15326, 24335, 28575, 32768, 0], + [11814, 19472, 24810, 32768, 0], + [10221, 18611, 24767, 32768, 0], + [7689, 14558, 20321, 32768, 0], + ], + ], + [ + [ + [16214, 22380, 25770, 32768, 0], + [14213, 21304, 25295, 32768, 0], + [9213, 15823, 20455, 32768, 0], + [6395, 11758, 16139, 32768, 0], + [4779, 9187, 13066, 32768, 0], + [3821, 7501, 10953, 32768, 0], + [2293, 4567, 6795, 32768, 0], + [15859, 21283, 23820, 32768, 0], + [18404, 25602, 28726, 32768, 0], + [14325, 21980, 26206, 32768, 0], + [10669, 17937, 22720, 32768, 0], + [8297, 14642, 19447, 32768, 0], + [6746, 12389, 16893, 32768, 0], + [4324, 8251, 11770, 32768, 0], + [16532, 21631, 24475, 32768, 0], + [20667, 27150, 29668, 32768, 0], + [16728, 24510, 28175, 32768, 0], + [12861, 20645, 25332, 32768, 0], + [10076, 17361, 22417, 32768, 0], + [8395, 14940, 19963, 32768, 0], + [5731, 10683, 14912, 32768, 0], + ], + [ + [14433, 21155, 24938, 32768, 0], + [14658, 21716, 25545, 32768, 0], + [9923, 16824, 21557, 32768, 0], + [6982, 13052, 17721, 32768, 0], + [5419, 10503, 15050, 32768, 0], + [4852, 9162, 13014, 32768, 0], + [3271, 6395, 9630, 32768, 0], + [22210, 27833, 30109, 32768, 0], + [20750, 27368, 29821, 32768, 0], + [16894, 24828, 28573, 32768, 0], + [13247, 21276, 25757, 32768, 0], + [10038, 17265, 22563, 32768, 0], + [8587, 14947, 20327, 32768, 0], + [5645, 11371, 15252, 32768, 0], + [22027, 27526, 29714, 32768, 0], + [23098, 29146, 31221, 32768, 0], + [19886, 27341, 30272, 32768, 0], + [15609, 23747, 28046, 32768, 0], + [11993, 20065, 24939, 32768, 0], + [9637, 18267, 23671, 32768, 0], + [7625, 13801, 19144, 32768, 0], + ], + ], + [ + [ + [14438, 20798, 24089, 32768, 0], + [12621, 19203, 23097, 32768, 0], + [8177, 14125, 18402, 32768, 0], + [5674, 10501, 14456, 32768, 0], + [4236, 8239, 11733, 32768, 0], + [3447, 6750, 9806, 32768, 0], + [1986, 3950, 5864, 32768, 0], + [16208, 22099, 24930, 32768, 0], + [16537, 24025, 27585, 32768, 0], + [12780, 20381, 24867, 32768, 0], + [9767, 16612, 21416, 32768, 0], + [7686, 13738, 18398, 32768, 0], + [6333, 11614, 15964, 32768, 0], + [3941, 7571, 10836, 32768, 0], + [22819, 27422, 29202, 32768, 0], + [22224, 28514, 30721, 32768, 0], + [17660, 25433, 28913, 32768, 0], + [13574, 21482, 26002, 32768, 0], + [10629, 17977, 22938, 32768, 0], + [8612, 15298, 20265, 32768, 0], + [5607, 10491, 14596, 32768, 0], + ], + [ + [13569, 19800, 23206, 32768, 0], + [13128, 19924, 23869, 32768, 0], + [8329, 14841, 19403, 32768, 0], + [6130, 10976, 15057, 32768, 0], + [4682, 8839, 12518, 32768, 0], + [3656, 7409, 10588, 32768, 0], + [2577, 5099, 7412, 32768, 0], + [22427, 28684, 30585, 32768, 0], + [20913, 27750, 30139, 32768, 0], + [15840, 24109, 27834, 32768, 0], + [12308, 20029, 24569, 32768, 0], + [10216, 16785, 21458, 32768, 0], + [8309, 14203, 19113, 32768, 0], + [6043, 11168, 15307, 32768, 0], + [23166, 28901, 30998, 32768, 0], + [21899, 28405, 30751, 32768, 0], + [18413, 26091, 29443, 32768, 0], + [15233, 23114, 27352, 32768, 0], + [12683, 20472, 25288, 32768, 0], + [10702, 18259, 23409, 32768, 0], + [8125, 14464, 19226, 32768, 0], + ], + ], + [ + [ + [9040, 14786, 18360, 32768, 0], + [9979, 15718, 19415, 32768, 0], + [7913, 13918, 18311, 32768, 0], + [5859, 10889, 15184, 32768, 0], + [4593, 8677, 12510, 32768, 0], + [3820, 7396, 10791, 32768, 0], + [1730, 3471, 5192, 32768, 0], + [11803, 18365, 22709, 32768, 0], + [11419, 18058, 22225, 32768, 0], + [9418, 15774, 20243, 32768, 0], + [7539, 13325, 17657, 32768, 0], + [6233, 11317, 15384, 32768, 0], + [5137, 9656, 13545, 32768, 0], + [2977, 5774, 8349, 32768, 0], + [21207, 27246, 29640, 32768, 0], + [19547, 26578, 29497, 32768, 0], + [16169, 23871, 27690, 32768, 0], + [12820, 20458, 25018, 32768, 0], + [10224, 17332, 22214, 32768, 0], + [8526, 15048, 19884, 32768, 0], + [5037, 9410, 13118, 32768, 0], + ], + [ + [12339, 17329, 20140, 32768, 0], + [13505, 19895, 23225, 32768, 0], + [9847, 16944, 21564, 32768, 0], + [7280, 13256, 18348, 32768, 0], + [4712, 10009, 14454, 32768, 0], + [4361, 7914, 12477, 32768, 0], + [2870, 5628, 7995, 32768, 0], + [20061, 25504, 28526, 32768, 0], + [15235, 22878, 26145, 32768, 0], + [12985, 19958, 24155, 32768, 0], + [9782, 16641, 21403, 32768, 0], + [9456, 16360, 20760, 32768, 0], + [6855, 12940, 18557, 32768, 0], + [5661, 10564, 15002, 32768, 0], + [25656, 30602, 31894, 32768, 0], + [22570, 29107, 31092, 32768, 0], + [18917, 26423, 29541, 32768, 0], + [15940, 23649, 27754, 32768, 0], + [12803, 20581, 25219, 32768, 0], + [11082, 18695, 23376, 32768, 0], + [7939, 14373, 19005, 32768, 0], + ], + ], + [ + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + ], + [ + [ + [ + [18315, 24289, 27551, 32768, 0], + [16854, 24068, 27835, 32768, 0], + [10140, 17927, 23173, 32768, 0], + [6722, 12982, 18267, 32768, 0], + [4661, 9826, 14706, 32768, 0], + [3832, 8165, 12294, 32768, 0], + [2795, 6098, 9245, 32768, 0], + [17145, 23326, 26672, 32768, 0], + [20733, 27680, 30308, 32768, 0], + [16032, 24461, 28546, 32768, 0], + [11653, 20093, 25081, 32768, 0], + [9290, 16429, 22086, 32768, 0], + [7796, 14598, 19982, 32768, 0], + [6502, 12378, 17441, 32768, 0], + [21681, 27732, 30320, 32768, 0], + [22389, 29044, 31261, 32768, 0], + [19027, 26731, 30087, 32768, 0], + [14739, 23755, 28624, 32768, 0], + [11358, 20778, 25511, 32768, 0], + [10995, 18073, 24190, 32768, 0], + [9162, 14990, 20617, 32768, 0], + ], + [ + [21425, 27952, 30388, 32768, 0], + [18062, 25838, 29034, 32768, 0], + [11956, 19881, 24808, 32768, 0], + [7718, 15000, 20980, 32768, 0], + [5702, 11254, 16143, 32768, 0], + [4898, 9088, 16864, 32768, 0], + [3679, 6776, 11907, 32768, 0], + [23294, 30160, 31663, 32768, 0], + [24397, 29896, 31836, 32768, 0], + [19245, 27128, 30593, 32768, 0], + [13202, 19825, 26404, 32768, 0], + [11578, 19297, 23957, 32768, 0], + [8073, 13297, 21370, 32768, 0], + [5461, 10923, 19745, 32768, 0], + [27367, 30521, 31934, 32768, 0], + [24904, 30671, 31940, 32768, 0], + [23075, 28460, 31299, 32768, 0], + [14400, 23658, 30417, 32768, 0], + [13885, 23882, 28325, 32768, 0], + [14746, 22938, 27853, 32768, 0], + [5461, 16384, 27307, 32768, 0], + ], + ], + [ + [ + [18274, 24813, 27890, 32768, 0], + [15537, 23149, 27003, 32768, 0], + [9449, 16740, 21827, 32768, 0], + [6700, 12498, 17261, 32768, 0], + [4988, 9866, 14198, 32768, 0], + [4236, 8147, 11902, 32768, 0], + [2867, 5860, 8654, 32768, 0], + [17124, 23171, 26101, 32768, 0], + [20396, 27477, 30148, 32768, 0], + [16573, 24629, 28492, 32768, 0], + [12749, 20846, 25674, 32768, 0], + [10233, 17878, 22818, 32768, 0], + [8525, 15332, 20363, 32768, 0], + [6283, 11632, 16255, 32768, 0], + [20466, 26511, 29286, 32768, 0], + [23059, 29174, 31191, 32768, 0], + [19481, 27263, 30241, 32768, 0], + [15458, 23631, 28137, 32768, 0], + [12416, 20608, 25693, 32768, 0], + [10261, 18011, 23261, 32768, 0], + [8016, 14655, 19666, 32768, 0], + ], + [ + [17616, 24586, 28112, 32768, 0], + [15809, 23299, 27155, 32768, 0], + [10767, 18890, 23793, 32768, 0], + [7727, 14255, 18865, 32768, 0], + [6129, 11926, 16882, 32768, 0], + [4482, 9704, 14861, 32768, 0], + [3277, 7452, 11522, 32768, 0], + [22956, 28551, 30730, 32768, 0], + [22724, 28937, 30961, 32768, 0], + [18467, 26324, 29580, 32768, 0], + [13234, 20713, 25649, 32768, 0], + [11181, 17592, 22481, 32768, 0], + [8291, 18358, 24576, 32768, 0], + [7568, 11881, 14984, 32768, 0], + [24948, 29001, 31147, 32768, 0], + [25674, 30619, 32151, 32768, 0], + [20841, 26793, 29603, 32768, 0], + [14669, 24356, 28666, 32768, 0], + [11334, 23593, 28219, 32768, 0], + [8922, 14762, 22873, 32768, 0], + [8301, 13544, 20535, 32768, 0], + ], + ], + [ + [ + [17113, 23733, 27081, 32768, 0], + [14139, 21406, 25452, 32768, 0], + [8552, 15002, 19776, 32768, 0], + [5871, 11120, 15378, 32768, 0], + [4455, 8616, 12253, 32768, 0], + [3469, 6910, 10386, 32768, 0], + [2255, 4553, 6782, 32768, 0], + [18224, 24376, 27053, 32768, 0], + [19290, 26710, 29614, 32768, 0], + [14936, 22991, 27184, 32768, 0], + [11238, 18951, 23762, 32768, 0], + [8786, 15617, 20588, 32768, 0], + [7317, 13228, 18003, 32768, 0], + [5101, 9512, 13493, 32768, 0], + [22639, 28222, 30210, 32768, 0], + [23216, 29331, 31307, 32768, 0], + [19075, 26762, 29895, 32768, 0], + [15014, 23113, 27457, 32768, 0], + [11938, 19857, 24752, 32768, 0], + [9942, 17280, 22282, 32768, 0], + [7167, 13144, 17752, 32768, 0], + ], + [ + [15820, 22738, 26488, 32768, 0], + [13530, 20885, 25216, 32768, 0], + [8395, 15530, 20452, 32768, 0], + [6574, 12321, 16380, 32768, 0], + [5353, 10419, 14568, 32768, 0], + [4613, 8446, 12381, 32768, 0], + [3440, 7158, 9903, 32768, 0], + [24247, 29051, 31224, 32768, 0], + [22118, 28058, 30369, 32768, 0], + [16498, 24768, 28389, 32768, 0], + [12920, 21175, 26137, 32768, 0], + [10730, 18619, 25352, 32768, 0], + [10187, 16279, 22791, 32768, 0], + [9310, 14631, 22127, 32768, 0], + [24970, 30558, 32057, 32768, 0], + [24801, 29942, 31698, 32768, 0], + [22432, 28453, 30855, 32768, 0], + [19054, 25680, 29580, 32768, 0], + [14392, 23036, 28109, 32768, 0], + [12495, 20947, 26650, 32768, 0], + [12442, 20326, 26214, 32768, 0], + ], + ], + [ + [ + [12162, 18785, 22648, 32768, 0], + [12749, 19697, 23806, 32768, 0], + [8580, 15297, 20346, 32768, 0], + [6169, 11749, 16543, 32768, 0], + [4836, 9391, 13448, 32768, 0], + [3821, 7711, 11613, 32768, 0], + [2228, 4601, 7070, 32768, 0], + [16319, 24725, 28280, 32768, 0], + [15698, 23277, 27168, 32768, 0], + [12726, 20368, 25047, 32768, 0], + [9912, 17015, 21976, 32768, 0], + [7888, 14220, 19179, 32768, 0], + [6777, 12284, 17018, 32768, 0], + [4492, 8590, 12252, 32768, 0], + [23249, 28904, 30947, 32768, 0], + [21050, 27908, 30512, 32768, 0], + [17440, 25340, 28949, 32768, 0], + [14059, 22018, 26541, 32768, 0], + [11288, 18903, 23898, 32768, 0], + [9411, 16342, 21428, 32768, 0], + [6278, 11588, 15944, 32768, 0], + ], + [ + [13981, 20067, 23226, 32768, 0], + [16922, 23580, 26783, 32768, 0], + [11005, 19039, 24487, 32768, 0], + [7389, 14218, 19798, 32768, 0], + [5598, 11505, 17206, 32768, 0], + [6090, 11213, 15659, 32768, 0], + [3820, 7371, 10119, 32768, 0], + [21082, 26925, 29675, 32768, 0], + [21262, 28627, 31128, 32768, 0], + [18392, 26454, 30437, 32768, 0], + [14870, 22910, 27096, 32768, 0], + [12620, 19484, 24908, 32768, 0], + [9290, 16553, 22802, 32768, 0], + [6668, 14288, 20004, 32768, 0], + [27704, 31055, 31949, 32768, 0], + [24709, 29978, 31788, 32768, 0], + [21668, 29264, 31657, 32768, 0], + [18295, 26968, 30074, 32768, 0], + [16399, 24422, 29313, 32768, 0], + [14347, 23026, 28104, 32768, 0], + [12370, 19806, 24477, 32768, 0], + ], + ], + [ + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + [ + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + [8192, 16384, 24576, 32768, 0], + ], + ], + ], +]; + +pub static DEFAULT_DC_SIGN_CDF: [[[[u16; 3]; 3]; 2]; 4] = [ + [[[16000, 32768, 0], [13056, 32768, 0], [18816, 32768, 0]], [[15232, 32768, 0], [12928, 32768, 0], [17280, 32768, 0]]], + [[[16000, 32768, 0], [13056, 32768, 0], [18816, 32768, 0]], [[15232, 32768, 0], [12928, 32768, 0], [17280, 32768, 0]]], + [[[16000, 32768, 0], [13056, 32768, 0], [18816, 32768, 0]], [[15232, 32768, 0], [12928, 32768, 0], [17280, 32768, 0]]], + [[[16000, 32768, 0], [13056, 32768, 0], [18816, 32768, 0]], [[15232, 32768, 0], [12928, 32768, 0], [17280, 32768, 0]]], +]; + +pub static DEFAULT_DRL_MODE_CDF: [[u16; 3]; 3] = [[13104, 32768, 0], [24560, 32768, 0], [18945, 32768, 0]]; + +pub static DEFAULT_EOB_EXTRA_CDF: [[[[[u16; 3]; 9]; 2]; 5]; 4] = [ + [ + [ + [ + [16961, 32768, 0], + [17223, 32768, 0], + [7621, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [19069, 32768, 0], + [22525, 32768, 0], + [13377, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [20401, 32768, 0], + [17025, 32768, 0], + [12845, 32768, 0], + [12873, 32768, 0], + [14094, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [20681, 32768, 0], + [20701, 32768, 0], + [15250, 32768, 0], + [15017, 32768, 0], + [14928, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [23905, 32768, 0], + [17194, 32768, 0], + [16170, 32768, 0], + [17695, 32768, 0], + [13826, 32768, 0], + [15810, 32768, 0], + [12036, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [23959, 32768, 0], + [20799, 32768, 0], + [19021, 32768, 0], + [16203, 32768, 0], + [17886, 32768, 0], + [14144, 32768, 0], + [12010, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [27399, 32768, 0], + [16327, 32768, 0], + [18071, 32768, 0], + [19584, 32768, 0], + [20721, 32768, 0], + [18432, 32768, 0], + [19560, 32768, 0], + [10150, 32768, 0], + [8805, 32768, 0], + ], + [ + [24932, 32768, 0], + [20833, 32768, 0], + [12027, 32768, 0], + [16670, 32768, 0], + [19914, 32768, 0], + [15106, 32768, 0], + [17662, 32768, 0], + [13783, 32768, 0], + [28756, 32768, 0], + ], + ], + [ + [ + [23406, 32768, 0], + [21845, 32768, 0], + [18432, 32768, 0], + [16384, 32768, 0], + [17096, 32768, 0], + [12561, 32768, 0], + [17320, 32768, 0], + [22395, 32768, 0], + [21370, 32768, 0], + ], + [ + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + ], + [ + [ + [ + [17471, 32768, 0], + [20223, 32768, 0], + [11357, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [20335, 32768, 0], + [21667, 32768, 0], + [14818, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [20430, 32768, 0], + [20662, 32768, 0], + [15367, 32768, 0], + [16970, 32768, 0], + [14657, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [22117, 32768, 0], + [22028, 32768, 0], + [18650, 32768, 0], + [16042, 32768, 0], + [15885, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [22409, 32768, 0], + [21012, 32768, 0], + [15650, 32768, 0], + [17395, 32768, 0], + [15469, 32768, 0], + [20205, 32768, 0], + [19511, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [24220, 32768, 0], + [22480, 32768, 0], + [17737, 32768, 0], + [18916, 32768, 0], + [19268, 32768, 0], + [18412, 32768, 0], + [18844, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [25991, 32768, 0], + [20314, 32768, 0], + [17731, 32768, 0], + [19678, 32768, 0], + [18649, 32768, 0], + [17307, 32768, 0], + [21798, 32768, 0], + [17549, 32768, 0], + [15630, 32768, 0], + ], + [ + [26585, 32768, 0], + [21469, 32768, 0], + [20432, 32768, 0], + [17735, 32768, 0], + [19280, 32768, 0], + [15235, 32768, 0], + [20297, 32768, 0], + [22471, 32768, 0], + [28997, 32768, 0], + ], + ], + [ + [ + [26605, 32768, 0], + [11304, 32768, 0], + [16726, 32768, 0], + [16560, 32768, 0], + [20866, 32768, 0], + [23524, 32768, 0], + [19878, 32768, 0], + [13469, 32768, 0], + [23084, 32768, 0], + ], + [ + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + ], + [ + [ + [ + [18983, 32768, 0], + [20512, 32768, 0], + [14885, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [20090, 32768, 0], + [19444, 32768, 0], + [17286, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [19139, 32768, 0], + [21487, 32768, 0], + [18959, 32768, 0], + [20910, 32768, 0], + [19089, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [20536, 32768, 0], + [20664, 32768, 0], + [20625, 32768, 0], + [19123, 32768, 0], + [14862, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [19833, 32768, 0], + [21502, 32768, 0], + [17485, 32768, 0], + [20267, 32768, 0], + [18353, 32768, 0], + [23329, 32768, 0], + [21478, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [22041, 32768, 0], + [23434, 32768, 0], + [20001, 32768, 0], + [20554, 32768, 0], + [20951, 32768, 0], + [20145, 32768, 0], + [15562, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [23312, 32768, 0], + [21607, 32768, 0], + [16526, 32768, 0], + [18957, 32768, 0], + [18034, 32768, 0], + [18934, 32768, 0], + [24247, 32768, 0], + [16921, 32768, 0], + [17080, 32768, 0], + ], + [ + [26579, 32768, 0], + [24910, 32768, 0], + [18637, 32768, 0], + [19800, 32768, 0], + [20388, 32768, 0], + [9887, 32768, 0], + [15642, 32768, 0], + [30198, 32768, 0], + [24721, 32768, 0], + ], + ], + [ + [ + [26998, 32768, 0], + [16737, 32768, 0], + [17838, 32768, 0], + [18922, 32768, 0], + [19515, 32768, 0], + [18636, 32768, 0], + [17333, 32768, 0], + [15776, 32768, 0], + [22658, 32768, 0], + ], + [ + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + ], + [ + [ + [ + [20177, 32768, 0], + [20789, 32768, 0], + [20262, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [21416, 32768, 0], + [20855, 32768, 0], + [23410, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [20238, 32768, 0], + [21057, 32768, 0], + [19159, 32768, 0], + [22337, 32768, 0], + [20159, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [20125, 32768, 0], + [20559, 32768, 0], + [21707, 32768, 0], + [22296, 32768, 0], + [17333, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [19941, 32768, 0], + [20527, 32768, 0], + [21470, 32768, 0], + [22487, 32768, 0], + [19558, 32768, 0], + [22354, 32768, 0], + [20331, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [22752, 32768, 0], + [25006, 32768, 0], + [22075, 32768, 0], + [21576, 32768, 0], + [17740, 32768, 0], + [21690, 32768, 0], + [19211, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [21442, 32768, 0], + [22358, 32768, 0], + [18503, 32768, 0], + [20291, 32768, 0], + [19945, 32768, 0], + [21294, 32768, 0], + [21178, 32768, 0], + [19400, 32768, 0], + [10556, 32768, 0], + ], + [ + [24648, 32768, 0], + [24949, 32768, 0], + [20708, 32768, 0], + [23905, 32768, 0], + [20501, 32768, 0], + [9558, 32768, 0], + [9423, 32768, 0], + [30365, 32768, 0], + [19253, 32768, 0], + ], + ], + [ + [ + [26064, 32768, 0], + [22098, 32768, 0], + [19613, 32768, 0], + [20525, 32768, 0], + [17595, 32768, 0], + [16618, 32768, 0], + [20497, 32768, 0], + [18989, 32768, 0], + [15513, 32768, 0], + ], + [ + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + ], +]; + +pub static DEFAULT_EOB_PT_1024_CDF: [[[u16; 12]; 2]; 4] = [ + [ + [393, 421, 751, 1623, 3160, 6352, 13345, 18047, 22571, 25830, 32768, 0], + [1865, 1988, 2930, 4242, 10533, 16538, 21354, 27255, 28546, 31784, 32768, 0], + ], + [ + [696, 948, 3145, 5702, 9706, 13217, 17851, 21856, 25692, 28034, 32768, 0], + [2672, 3591, 9330, 17084, 22725, 24284, 26527, 28027, 28377, 30876, 32768, 0], + ], + [ + [2784, 3831, 7041, 10521, 14847, 18844, 23155, 26682, 29229, 31045, 32768, 0], + [9577, 12466, 17739, 20750, 22061, 23215, 24601, 25483, 25843, 32056, 32768, 0], + ], + [ + [6698, 8334, 11961, 15762, 20186, 23862, 27434, 29326, 31082, 32050, 32768, 0], + [20569, 22426, 25569, 26859, 28053, 28913, 29486, 29724, 29807, 32570, 32768, 0], + ], +]; + +pub static DEFAULT_EOB_PT_128_CDF: [[[[u16; 9]; 2]; 2]; 4] = [ + [ + [[219, 482, 1140, 2091, 3680, 6028, 12586, 32768, 0], [371, 699, 1254, 4830, 9479, 12562, 17497, 32768, 0]], + [[5245, 7456, 12880, 15852, 20033, 23932, 27608, 32768, 0], [2054, 3472, 5869, 14232, 18242, 20590, 26752, 32768, 0]], + ], + [ + [[685, 933, 1488, 2714, 4766, 8562, 19254, 32768, 0], [217, 352, 618, 2303, 5261, 9969, 17472, 32768, 0]], + [[8045, 11200, 15497, 19595, 23948, 27408, 30938, 32768, 0], [2310, 4160, 7471, 14997, 17931, 20768, 30240, 32768, 0]], + ], + [ + [[1366, 1738, 2527, 5016, 9355, 15797, 24643, 32768, 0], [354, 558, 944, 2760, 7287, 14037, 21779, 32768, 0]], + [[13627, 16246, 20173, 24429, 27948, 30415, 31863, 32768, 0], [6275, 9889, 14769, 23164, 27988, 30493, 32272, 32768, 0]], + ], + [ + [[3472, 4885, 7489, 12481, 18517, 24536, 29635, 32768, 0], [886, 1731, 3271, 8469, 15569, 22126, 28383, 32768, 0]], + [[24313, 26062, 28385, 30107, 31217, 31898, 32345, 32768, 0], [9165, 13282, 21150, 30286, 31894, 32571, 32712, 32768, 0]], + ], +]; + +pub static DEFAULT_EOB_PT_16_CDF: [[[[u16; 6]; 2]; 2]; 4] = [ + [ + [[840, 1039, 1980, 4895, 32768, 0], [370, 671, 1883, 4471, 32768, 0]], + [[3247, 4950, 9688, 14563, 32768, 0], [1904, 3354, 7763, 14647, 32768, 0]], + ], + [ + [[2125, 2551, 5165, 8946, 32768, 0], [513, 765, 1859, 6339, 32768, 0]], + [[7637, 9498, 14259, 19108, 32768, 0], [2497, 4096, 8866, 16993, 32768, 0]], + ], + [ + [[4016, 4897, 8881, 14968, 32768, 0], [716, 1105, 2646, 10056, 32768, 0]], + [[11139, 13270, 18241, 23566, 32768, 0], [3192, 5032, 10297, 19755, 32768, 0]], + ], + [ + [[6708, 8958, 14746, 22133, 32768, 0], [1222, 2074, 4783, 15410, 32768, 0]], + [[19575, 21766, 26044, 29709, 32768, 0], [7297, 10767, 19273, 28194, 32768, 0]], + ], +]; + +pub static DEFAULT_EOB_PT_256_CDF: [[[[u16; 10]; 2]; 2]; 4] = [ + [ + [[310, 584, 1887, 3589, 6168, 8611, 11352, 15652, 32768, 0], [998, 1850, 2998, 5604, 17341, 19888, 22899, 25583, 32768, 0]], + [[2520, 3240, 5952, 8870, 12577, 17558, 19954, 24168, 32768, 0], [2203, 4130, 7435, 10739, 20652, 23681, 25609, 27261, 32768, 0]], + ], + [ + [[1448, 2109, 4151, 6263, 9329, 13260, 17944, 23300, 32768, 0], [399, 1019, 1749, 3038, 10444, 15546, 22739, 27294, 32768, 0]], + [[6402, 8148, 12623, 15072, 18728, 22847, 26447, 29377, 32768, 0], [1674, 3252, 5734, 10159, 22397, 23802, 24821, 30940, 32768, 0]], + ], + [ + [[3089, 3920, 6038, 9460, 14266, 19881, 25766, 29176, 32768, 0], [1084, 2358, 3488, 5122, 11483, 18103, 26023, 29799, 32768, 0]], + [[11514, 13794, 17480, 20754, 24361, 27378, 29492, 31277, 32768, 0], [6571, 9610, 15516, 21826, 29092, 30829, 31842, 32708, 32768, 0]], + ], + [ + [[5348, 7113, 11820, 15924, 22106, 26777, 30334, 31757, 32768, 0], [2453, 4474, 6307, 8777, 16474, 22975, 29000, 31547, 32768, 0]], + [[23110, 24597, 27140, 28894, 30167, 30927, 31392, 32094, 32768, 0], [9998, 17661, 25178, 28097, 31308, 32038, 32403, 32695, 32768, 0]], + ], +]; + +pub static DEFAULT_EOB_PT_32_CDF: [[[[u16; 7]; 2]; 2]; 4] = [ + [ + [[400, 520, 977, 2102, 6542, 32768, 0], [210, 405, 1315, 3326, 7537, 32768, 0]], + [[2636, 4273, 7588, 11794, 20401, 32768, 0], [1786, 3179, 6902, 11357, 19054, 32768, 0]], + ], + [ + [[989, 1249, 2019, 4151, 10785, 32768, 0], [313, 441, 1099, 2917, 8562, 32768, 0]], + [[8394, 10352, 13932, 18855, 26014, 32768, 0], [2578, 4124, 8181, 13670, 24234, 32768, 0]], + ], + [ + [[2515, 3003, 4452, 8162, 16041, 32768, 0], [574, 821, 1836, 5089, 13128, 32768, 0]], + [[13468, 16303, 20361, 25105, 29281, 32768, 0], [3542, 5502, 10415, 16760, 25644, 32768, 0]], + ], + [ + [[4617, 5709, 8446, 13584, 23135, 32768, 0], [1156, 1702, 3675, 9274, 20539, 32768, 0]], + [[22086, 24282, 27010, 29770, 31743, 32768, 0], [7699, 10897, 20891, 26926, 31628, 32768, 0]], + ], +]; + +pub static DEFAULT_EOB_PT_512_CDF: [[[u16; 11]; 2]; 4] = [ + [[641, 983, 3707, 5430, 10234, 14958, 18788, 23412, 26061, 32768, 0], [5095, 6446, 9996, 13354, 16017, 17986, 20919, 26129, 29140, 32768, 0]], + [[1230, 2278, 5035, 7776, 11871, 15346, 19590, 24584, 28749, 32768, 0], [7265, 9979, 15819, 19250, 21780, 23846, 26478, 28396, 31811, 32768, 0]], + [ + [2624, 3936, 6480, 9686, 13979, 17726, 23267, 28410, 31078, 32768, 0], + [12015, 14769, 19588, 22052, 24222, 25812, 27300, 29219, 32114, 32768, 0], + ], + [ + [5927, 7809, 10923, 14597, 19439, 24135, 28456, 31142, 32060, 32768, 0], + [21093, 23043, 25742, 27658, 29097, 29716, 30073, 30820, 31956, 32768, 0], + ], +]; + +pub static DEFAULT_EOB_PT_64_CDF: [[[[u16; 8]; 2]; 2]; 4] = [ + [ + [[329, 498, 1101, 1784, 3265, 7758, 32768, 0], [335, 730, 1459, 5494, 8755, 12997, 32768, 0]], + [[3505, 5304, 10086, 13814, 17684, 23370, 32768, 0], [1563, 2700, 4876, 10911, 14706, 22480, 32768, 0]], + ], + [ + [[1260, 1446, 2253, 3712, 6652, 13369, 32768, 0], [401, 605, 1029, 2563, 5845, 12626, 32768, 0]], + [[8609, 10612, 14624, 18714, 22614, 29024, 32768, 0], [1923, 3127, 5867, 9703, 14277, 27100, 32768, 0]], + ], + [ + [[2374, 2772, 4583, 7276, 12288, 19706, 32768, 0], [497, 810, 1315, 3000, 7004, 15641, 32768, 0]], + [[15050, 17126, 21410, 24886, 28156, 30726, 32768, 0], [4034, 6290, 10235, 14982, 21214, 28491, 32768, 0]], + ], + [ + [[6307, 7541, 12060, 16358, 22553, 27865, 32768, 0], [1289, 2320, 3971, 7926, 14153, 24291, 32768, 0]], + [[24212, 25708, 28268, 30035, 31307, 32049, 32768, 0], [8726, 12378, 19409, 26450, 30038, 32462, 32768, 0]], + ], +]; + +pub static DEFAULT_INTER_TX_TYPE_SET3_CDF: [[u16; 3]; 4] = [[16384, 32768, 0], [4167, 32768, 0], [1998, 32768, 0], [748, 32768, 0]]; + +pub static DEFAULT_INTRA_FRAME_Y_MODE_CDF: [[[u16; 14]; 5]; 5] = [ + [ + [15588, 17027, 19338, 20218, 20682, 21110, 21825, 23244, 24189, 28165, 29093, 30466, 32768, 0], + [12016, 18066, 19516, 20303, 20719, 21444, 21888, 23032, 24434, 28658, 30172, 31409, 32768, 0], + [10052, 10771, 22296, 22788, 23055, 23239, 24133, 25620, 26160, 29336, 29929, 31567, 32768, 0], + [14091, 15406, 16442, 18808, 19136, 19546, 19998, 22096, 24746, 29585, 30958, 32462, 32768, 0], + [12122, 13265, 15603, 16501, 18609, 20033, 22391, 25583, 26437, 30261, 31073, 32475, 32768, 0], + ], + [ + [10023, 19585, 20848, 21440, 21832, 22760, 23089, 24023, 25381, 29014, 30482, 31436, 32768, 0], + [5983, 24099, 24560, 24886, 25066, 25795, 25913, 26423, 27610, 29905, 31276, 31794, 32768, 0], + [7444, 12781, 20177, 20728, 21077, 21607, 22170, 23405, 24469, 27915, 29090, 30492, 32768, 0], + [8537, 14689, 15432, 17087, 17408, 18172, 18408, 19825, 24649, 29153, 31096, 32210, 32768, 0], + [7543, 14231, 15496, 16195, 17905, 20717, 21984, 24516, 26001, 29675, 30981, 31994, 32768, 0], + ], + [ + [12613, 13591, 21383, 22004, 22312, 22577, 23401, 25055, 25729, 29538, 30305, 32077, 32768, 0], + [9687, 13470, 18506, 19230, 19604, 20147, 20695, 22062, 23219, 27743, 29211, 30907, 32768, 0], + [6183, 6505, 26024, 26252, 26366, 26434, 27082, 28354, 28555, 30467, 30794, 32086, 32768, 0], + [10718, 11734, 14954, 17224, 17565, 17924, 18561, 21523, 23878, 28975, 30287, 32252, 32768, 0], + [9194, 9858, 16501, 17263, 18424, 19171, 21563, 25961, 26561, 30072, 30737, 32463, 32768, 0], + ], + [ + [12602, 14399, 15488, 18381, 18778, 19315, 19724, 21419, 25060, 29696, 30917, 32409, 32768, 0], + [8203, 13821, 14524, 17105, 17439, 18131, 18404, 19468, 25225, 29485, 31158, 32342, 32768, 0], + [8451, 9731, 15004, 17643, 18012, 18425, 19070, 21538, 24605, 29118, 30078, 32018, 32768, 0], + [7714, 9048, 9516, 16667, 16817, 16994, 17153, 18767, 26743, 30389, 31536, 32528, 32768, 0], + [8843, 10280, 11496, 15317, 16652, 17943, 19108, 22718, 25769, 29953, 30983, 32485, 32768, 0], + ], + [ + [12578, 13671, 15979, 16834, 19075, 20913, 22989, 25449, 26219, 30214, 31150, 32477, 32768, 0], + [9563, 13626, 15080, 15892, 17756, 20863, 22207, 24236, 25380, 29653, 31143, 32277, 32768, 0], + [8356, 8901, 17616, 18256, 19350, 20106, 22598, 25947, 26466, 29900, 30523, 32261, 32768, 0], + [10835, 11815, 13124, 16042, 17018, 18039, 18947, 22753, 24615, 29489, 30883, 32482, 32768, 0], + [7618, 8288, 9859, 10509, 15386, 18657, 22903, 28776, 29180, 31355, 31802, 32593, 32768, 0], + ], +]; + +pub static DEFAULT_INTRA_TX_TYPE_SET2_CDF: [[[u16; 6]; 13]; 3] = [ + [ + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + ], + [ + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + [6554, 13107, 19661, 26214, 32768, 0], + ], + [ + [1127, 12814, 22772, 27483, 32768, 0], + [145, 6761, 11980, 26667, 32768, 0], + [362, 5887, 11678, 16725, 32768, 0], + [385, 15213, 18587, 30693, 32768, 0], + [25, 2914, 23134, 27903, 32768, 0], + [60, 4470, 11749, 23991, 32768, 0], + [37, 3332, 14511, 21448, 32768, 0], + [157, 6320, 13036, 17439, 32768, 0], + [119, 6719, 12906, 29396, 32768, 0], + [47, 5537, 12576, 21499, 32768, 0], + [269, 6076, 11258, 23115, 32768, 0], + [83, 5615, 12001, 17228, 32768, 0], + [1968, 5556, 12023, 18547, 32768, 0], + ], +]; + +pub static DEFAULT_IS_INTER_CDF: [[u16; 3]; 4] = [[806, 32768, 0], [16662, 32768, 0], [20186, 32768, 0], [26538, 32768, 0]]; + +pub static DEFAULT_MV_BIT_CDF: [[u16; 3]; 10] = [ + [17408, 32768, 0], + [17920, 32768, 0], + [18944, 32768, 0], + [20480, 32768, 0], + [22528, 32768, 0], + [24576, 32768, 0], + [28672, 32768, 0], + [29952, 32768, 0], + [29952, 32768, 0], + [30720, 32768, 0], +]; + +pub static DEFAULT_MV_CLASS0_BIT_CDF: [u16; 3] = [27648, 32768, 0]; + +pub static DEFAULT_MV_CLASS0_FR_CDF: [[[u16; 5]; 2]; 2] = + [[[16384, 24576, 26624, 32768, 0], [12288, 21248, 24128, 32768, 0]], [[16384, 24576, 26624, 32768, 0], [12288, 21248, 24128, 32768, 0]]]; + +pub static DEFAULT_MV_CLASS0_HP_CDF: [u16; 3] = [20480, 32768, 0]; + +pub static DEFAULT_MV_CLASS_CDF: [[u16; 12]; 2] = [ + [28672, 30976, 31858, 32320, 32551, 32656, 32740, 32757, 32762, 32767, 32768, 0], + [28672, 30976, 31858, 32320, 32551, 32656, 32740, 32757, 32762, 32767, 32768, 0], +]; + +pub static DEFAULT_MV_FR_CDF: [[u16; 5]; 2] = [[8192, 17408, 21248, 32768, 0], [8192, 17408, 21248, 32768, 0]]; + +pub static DEFAULT_MV_HP_CDF: [u16; 3] = [16384, 32768, 0]; + +pub static DEFAULT_MV_JOINT_CDF: [u16; 5] = [4096, 11264, 19328, 32768, 0]; + +pub static DEFAULT_MV_SIGN_CDF: [u16; 3] = [16384, 32768, 0]; + +pub static DEFAULT_NEW_MV_CDF: [[u16; 3]; 6] = + [[24035, 32768, 0], [16630, 32768, 0], [15339, 32768, 0], [8386, 32768, 0], [12222, 32768, 0], [4676, 32768, 0]]; + +pub static DEFAULT_PARTITION_W16_CDF: [[u16; 11]; 4] = [ + [15597, 20929, 24571, 26706, 27664, 28821, 29601, 30571, 31902, 32768, 0], + [7925, 11043, 16785, 22470, 23971, 25043, 26651, 28701, 29834, 32768, 0], + [5414, 13269, 15111, 20488, 22360, 24500, 25537, 26336, 32117, 32768, 0], + [2662, 6362, 8614, 20860, 23053, 24778, 26436, 27829, 31171, 32768, 0], +]; + +pub static DEFAULT_PARTITION_W32_CDF: [[u16; 11]; 4] = [ + [18462, 20920, 23124, 27647, 28227, 29049, 29519, 30178, 31544, 32768, 0], + [7689, 9060, 12056, 24992, 25660, 26182, 26951, 28041, 29052, 32768, 0], + [6015, 9009, 10062, 24544, 25409, 26545, 27071, 27526, 32047, 32768, 0], + [1394, 2208, 2796, 28614, 29061, 29466, 29840, 30185, 31899, 32768, 0], +]; + +pub static DEFAULT_PARTITION_W64_CDF: [[u16; 11]; 4] = [ + [20137, 21547, 23078, 29566, 29837, 30261, 30524, 30892, 31724, 32768, 0], + [6732, 7490, 9497, 27944, 28250, 28515, 28969, 29630, 30104, 32768, 0], + [5945, 7663, 8348, 28683, 29117, 29749, 30064, 30298, 32238, 32768, 0], + [870, 1212, 1487, 31198, 31394, 31574, 31743, 31881, 32332, 32768, 0], +]; + +pub static DEFAULT_PARTITION_W8_CDF: [[u16; 5]; 4] = + [[19132, 25510, 30392, 32768, 0], [13928, 19855, 28540, 32768, 0], [12522, 23679, 28629, 32768, 0], [9896, 18783, 25853, 32768, 0]]; + +pub static DEFAULT_REF_MV_CDF: [[u16; 3]; 6] = + [[23974, 32768, 0], [24188, 32768, 0], [17848, 32768, 0], [28622, 32768, 0], [24312, 32768, 0], [19923, 32768, 0]]; + +pub static DEFAULT_SCAN_16X16: [u16; 256] = [ + 0, 1, 16, 32, 17, 2, 3, 18, 33, 48, 64, 49, 34, 19, 4, 5, 20, 35, 50, 65, 80, 96, 81, 66, 51, 36, 21, 6, 7, 22, 37, 52, 67, 82, 97, 112, 128, + 113, 98, 83, 68, 53, 38, 23, 8, 9, 24, 39, 54, 69, 84, 99, 114, 129, 144, 160, 145, 130, 115, 100, 85, 70, 55, 40, 25, 10, 11, 26, 41, 56, 71, + 86, 101, 116, 131, 146, 161, 176, 192, 177, 162, 147, 132, 117, 102, 87, 72, 57, 42, 27, 12, 13, 28, 43, 58, 73, 88, 103, 118, 133, 148, 163, + 178, 193, 208, 224, 209, 194, 179, 164, 149, 134, 119, 104, 89, 74, 59, 44, 29, 14, 15, 30, 45, 60, 75, 90, 105, 120, 135, 150, 165, 180, 195, + 210, 225, 240, 241, 226, 211, 196, 181, 166, 151, 136, 121, 106, 91, 76, 61, 46, 31, 47, 62, 77, 92, 107, 122, 137, 152, 167, 182, 197, 212, 227, + 242, 243, 228, 213, 198, 183, 168, 153, 138, 123, 108, 93, 78, 63, 79, 94, 109, 124, 139, 154, 169, 184, 199, 214, 229, 244, 245, 230, 215, 200, + 185, 170, 155, 140, 125, 110, 95, 111, 126, 141, 156, 171, 186, 201, 216, 231, 246, 247, 232, 217, 202, 187, 172, 157, 142, 127, 143, 158, 173, + 188, 203, 218, 233, 248, 249, 234, 219, 204, 189, 174, 159, 175, 190, 205, 220, 235, 250, 251, 236, 221, 206, 191, 207, 222, 237, 252, 253, 238, + 223, 239, 254, 255, +]; + +pub static DEFAULT_SCAN_32X32: [u16; 1024] = [ + 0, 1, 32, 64, 33, 2, 3, 34, 65, 96, 128, 97, 66, 35, 4, 5, 36, 67, 98, 129, 160, 192, 161, 130, 99, 68, 37, 6, 7, 38, 69, 100, 131, 162, 193, + 224, 256, 225, 194, 163, 132, 101, 70, 39, 8, 9, 40, 71, 102, 133, 164, 195, 226, 257, 288, 320, 289, 258, 227, 196, 165, 134, 103, 72, 41, 10, + 11, 42, 73, 104, 135, 166, 197, 228, 259, 290, 321, 352, 384, 353, 322, 291, 260, 229, 198, 167, 136, 105, 74, 43, 12, 13, 44, 75, 106, 137, 168, + 199, 230, 261, 292, 323, 354, 385, 416, 448, 417, 386, 355, 324, 293, 262, 231, 200, 169, 138, 107, 76, 45, 14, 15, 46, 77, 108, 139, 170, 201, + 232, 263, 294, 325, 356, 387, 418, 449, 480, 512, 481, 450, 419, 388, 357, 326, 295, 264, 233, 202, 171, 140, 109, 78, 47, 16, 17, 48, 79, 110, + 141, 172, 203, 234, 265, 296, 327, 358, 389, 420, 451, 482, 513, 544, 576, 545, 514, 483, 452, 421, 390, 359, 328, 297, 266, 235, 204, 173, 142, + 111, 80, 49, 18, 19, 50, 81, 112, 143, 174, 205, 236, 267, 298, 329, 360, 391, 422, 453, 484, 515, 546, 577, 608, 640, 609, 578, 547, 516, 485, + 454, 423, 392, 361, 330, 299, 268, 237, 206, 175, 144, 113, 82, 51, 20, 21, 52, 83, 114, 145, 176, 207, 238, 269, 300, 331, 362, 393, 424, 455, + 486, 517, 548, 579, 610, 641, 672, 704, 673, 642, 611, 580, 549, 518, 487, 456, 425, 394, 363, 332, 301, 270, 239, 208, 177, 146, 115, 84, 53, + 22, 23, 54, 85, 116, 147, 178, 209, 240, 271, 302, 333, 364, 395, 426, 457, 488, 519, 550, 581, 612, 643, 674, 705, 736, 768, 737, 706, 675, 644, + 613, 582, 551, 520, 489, 458, 427, 396, 365, 334, 303, 272, 241, 210, 179, 148, 117, 86, 55, 24, 25, 56, 87, 118, 149, 180, 211, 242, 273, 304, + 335, 366, 397, 428, 459, 490, 521, 552, 583, 614, 645, 676, 707, 738, 769, 800, 832, 801, 770, 739, 708, 677, 646, 615, 584, 553, 522, 491, 460, + 429, 398, 367, 336, 305, 274, 243, 212, 181, 150, 119, 88, 57, 26, 27, 58, 89, 120, 151, 182, 213, 244, 275, 306, 337, 368, 399, 430, 461, 492, + 523, 554, 585, 616, 647, 678, 709, 740, 771, 802, 833, 864, 896, 865, 834, 803, 772, 741, 710, 679, 648, 617, 586, 555, 524, 493, 462, 431, 400, + 369, 338, 307, 276, 245, 214, 183, 152, 121, 90, 59, 28, 29, 60, 91, 122, 153, 184, 215, 246, 277, 308, 339, 370, 401, 432, 463, 494, 525, 556, + 587, 618, 649, 680, 711, 742, 773, 804, 835, 866, 897, 928, 960, 929, 898, 867, 836, 805, 774, 743, 712, 681, 650, 619, 588, 557, 526, 495, 464, + 433, 402, 371, 340, 309, 278, 247, 216, 185, 154, 123, 92, 61, 30, 31, 62, 93, 124, 155, 186, 217, 248, 279, 310, 341, 372, 403, 434, 465, 496, + 527, 558, 589, 620, 651, 682, 713, 744, 775, 806, 837, 868, 899, 930, 961, 992, 993, 962, 931, 900, 869, 838, 807, 776, 745, 714, 683, 652, 621, + 590, 559, 528, 497, 466, 435, 404, 373, 342, 311, 280, 249, 218, 187, 156, 125, 94, 63, 95, 126, 157, 188, 219, 250, 281, 312, 343, 374, 405, + 436, 467, 498, 529, 560, 591, 622, 653, 684, 715, 746, 777, 808, 839, 870, 901, 932, 963, 994, 995, 964, 933, 902, 871, 840, 809, 778, 747, 716, + 685, 654, 623, 592, 561, 530, 499, 468, 437, 406, 375, 344, 313, 282, 251, 220, 189, 158, 127, 159, 190, 221, 252, 283, 314, 345, 376, 407, 438, + 469, 500, 531, 562, 593, 624, 655, 686, 717, 748, 779, 810, 841, 872, 903, 934, 965, 996, 997, 966, 935, 904, 873, 842, 811, 780, 749, 718, 687, + 656, 625, 594, 563, 532, 501, 470, 439, 408, 377, 346, 315, 284, 253, 222, 191, 223, 254, 285, 316, 347, 378, 409, 440, 471, 502, 533, 564, 595, + 626, 657, 688, 719, 750, 781, 812, 843, 874, 905, 936, 967, 998, 999, 968, 937, 906, 875, 844, 813, 782, 751, 720, 689, 658, 627, 596, 565, 534, + 503, 472, 441, 410, 379, 348, 317, 286, 255, 287, 318, 349, 380, 411, 442, 473, 504, 535, 566, 597, 628, 659, 690, 721, 752, 783, 814, 845, 876, + 907, 938, 969, 1000, 1001, 970, 939, 908, 877, 846, 815, 784, 753, 722, 691, 660, 629, 598, 567, 536, 505, 474, 443, 412, 381, 350, 319, 351, + 382, 413, 444, 475, 506, 537, 568, 599, 630, 661, 692, 723, 754, 785, 816, 847, 878, 909, 940, 971, 1002, 1003, 972, 941, 910, 879, 848, 817, + 786, 755, 724, 693, 662, 631, 600, 569, 538, 507, 476, 445, 414, 383, 415, 446, 477, 508, 539, 570, 601, 632, 663, 694, 725, 756, 787, 818, 849, + 880, 911, 942, 973, 1004, 1005, 974, 943, 912, 881, 850, 819, 788, 757, 726, 695, 664, 633, 602, 571, 540, 509, 478, 447, 479, 510, 541, 572, + 603, 634, 665, 696, 727, 758, 789, 820, 851, 882, 913, 944, 975, 1006, 1007, 976, 945, 914, 883, 852, 821, 790, 759, 728, 697, 666, 635, 604, + 573, 542, 511, 543, 574, 605, 636, 667, 698, 729, 760, 791, 822, 853, 884, 915, 946, 977, 1008, 1009, 978, 947, 916, 885, 854, 823, 792, 761, + 730, 699, 668, 637, 606, 575, 607, 638, 669, 700, 731, 762, 793, 824, 855, 886, 917, 948, 979, 1010, 1011, 980, 949, 918, 887, 856, 825, 794, + 763, 732, 701, 670, 639, 671, 702, 733, 764, 795, 826, 857, 888, 919, 950, 981, 1012, 1013, 982, 951, 920, 889, 858, 827, 796, 765, 734, 703, + 735, 766, 797, 828, 859, 890, 921, 952, 983, 1014, 1015, 984, 953, 922, 891, 860, 829, 798, 767, 799, 830, 861, 892, 923, 954, 985, 1016, 1017, + 986, 955, 924, 893, 862, 831, 863, 894, 925, 956, 987, 1018, 1019, 988, 957, 926, 895, 927, 958, 989, 1020, 1021, 990, 959, 991, 1022, 1023, +]; + +pub static DEFAULT_SCAN_4X4: [u16; 16] = [0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15]; + +pub static DEFAULT_SCAN_8X8: [u16; 64] = [ + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, + 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63, +]; + +pub static DEFAULT_SINGLE_REF_CDF: [[[u16; 3]; 6]; 3] = [ + [[4897, 32768, 0], [1555, 32768, 0], [4236, 32768, 0], [8650, 32768, 0], [904, 32768, 0], [1444, 32768, 0]], + [[16973, 32768, 0], [16751, 32768, 0], [19647, 32768, 0], [24773, 32768, 0], [11014, 32768, 0], [15087, 32768, 0]], + [[29744, 32768, 0], [30279, 32768, 0], [31194, 32768, 0], [31895, 32768, 0], [26875, 32768, 0], [30304, 32768, 0]], +]; + +pub static DEFAULT_SKIP_CDF: [[u16; 3]; 3] = [[31671, 32768, 0], [16515, 32768, 0], [4576, 32768, 0]]; + +pub static DEFAULT_TXB_SKIP_CDF: [[[[u16; 3]; 13]; 5]; 4] = [ + [ + [ + [31849, 32768, 0], + [5892, 32768, 0], + [12112, 32768, 0], + [21935, 32768, 0], + [20289, 32768, 0], + [27473, 32768, 0], + [32487, 32768, 0], + [7654, 32768, 0], + [19473, 32768, 0], + [29984, 32768, 0], + [9961, 32768, 0], + [30242, 32768, 0], + [32117, 32768, 0], + ], + [ + [31548, 32768, 0], + [1549, 32768, 0], + [10130, 32768, 0], + [16656, 32768, 0], + [18591, 32768, 0], + [26308, 32768, 0], + [32537, 32768, 0], + [5403, 32768, 0], + [18096, 32768, 0], + [30003, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [29957, 32768, 0], + [5391, 32768, 0], + [18039, 32768, 0], + [23566, 32768, 0], + [22431, 32768, 0], + [25822, 32768, 0], + [32197, 32768, 0], + [3778, 32768, 0], + [15336, 32768, 0], + [28981, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [17920, 32768, 0], + [1818, 32768, 0], + [7282, 32768, 0], + [25273, 32768, 0], + [10923, 32768, 0], + [31554, 32768, 0], + [32624, 32768, 0], + [1366, 32768, 0], + [15628, 32768, 0], + [30462, 32768, 0], + [146, 32768, 0], + [5132, 32768, 0], + [31657, 32768, 0], + ], + [ + [6308, 32768, 0], + [117, 32768, 0], + [1638, 32768, 0], + [2161, 32768, 0], + [16384, 32768, 0], + [10923, 32768, 0], + [30247, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [30371, 32768, 0], + [7570, 32768, 0], + [13155, 32768, 0], + [20751, 32768, 0], + [20969, 32768, 0], + [27067, 32768, 0], + [32013, 32768, 0], + [5495, 32768, 0], + [17942, 32768, 0], + [28280, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [31782, 32768, 0], + [1836, 32768, 0], + [10689, 32768, 0], + [17604, 32768, 0], + [21622, 32768, 0], + [27518, 32768, 0], + [32399, 32768, 0], + [4419, 32768, 0], + [16294, 32768, 0], + [28345, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [31901, 32768, 0], + [10311, 32768, 0], + [18047, 32768, 0], + [24806, 32768, 0], + [23288, 32768, 0], + [27914, 32768, 0], + [32296, 32768, 0], + [4215, 32768, 0], + [15756, 32768, 0], + [28341, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [26726, 32768, 0], + [1045, 32768, 0], + [11703, 32768, 0], + [20590, 32768, 0], + [18554, 32768, 0], + [25970, 32768, 0], + [31938, 32768, 0], + [5583, 32768, 0], + [21313, 32768, 0], + [29390, 32768, 0], + [641, 32768, 0], + [22265, 32768, 0], + [31452, 32768, 0], + ], + [ + [26584, 32768, 0], + [188, 32768, 0], + [8847, 32768, 0], + [24519, 32768, 0], + [22938, 32768, 0], + [30583, 32768, 0], + [32608, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [29614, 32768, 0], + [9068, 32768, 0], + [12924, 32768, 0], + [19538, 32768, 0], + [17737, 32768, 0], + [24619, 32768, 0], + [30642, 32768, 0], + [4119, 32768, 0], + [16026, 32768, 0], + [25657, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [31957, 32768, 0], + [3230, 32768, 0], + [11153, 32768, 0], + [18123, 32768, 0], + [20143, 32768, 0], + [26536, 32768, 0], + [31986, 32768, 0], + [3050, 32768, 0], + [14603, 32768, 0], + [25155, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [32363, 32768, 0], + [10692, 32768, 0], + [19090, 32768, 0], + [24357, 32768, 0], + [24442, 32768, 0], + [28312, 32768, 0], + [32169, 32768, 0], + [3648, 32768, 0], + [15690, 32768, 0], + [26815, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [30669, 32768, 0], + [3832, 32768, 0], + [11663, 32768, 0], + [18889, 32768, 0], + [19782, 32768, 0], + [23313, 32768, 0], + [31330, 32768, 0], + [5124, 32768, 0], + [18719, 32768, 0], + [28468, 32768, 0], + [3082, 32768, 0], + [20982, 32768, 0], + [29443, 32768, 0], + ], + [ + [28573, 32768, 0], + [3183, 32768, 0], + [17802, 32768, 0], + [25977, 32768, 0], + [26677, 32768, 0], + [27832, 32768, 0], + [32387, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], + [ + [ + [26887, 32768, 0], + [6729, 32768, 0], + [10361, 32768, 0], + [17442, 32768, 0], + [15045, 32768, 0], + [22478, 32768, 0], + [29072, 32768, 0], + [2713, 32768, 0], + [11861, 32768, 0], + [20773, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [31903, 32768, 0], + [2044, 32768, 0], + [7528, 32768, 0], + [14618, 32768, 0], + [16182, 32768, 0], + [24168, 32768, 0], + [31037, 32768, 0], + [2786, 32768, 0], + [11194, 32768, 0], + [20155, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [32510, 32768, 0], + [8430, 32768, 0], + [17318, 32768, 0], + [24154, 32768, 0], + [23674, 32768, 0], + [28789, 32768, 0], + [32139, 32768, 0], + [3440, 32768, 0], + [13117, 32768, 0], + [22702, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + [ + [31671, 32768, 0], + [2056, 32768, 0], + [11746, 32768, 0], + [16852, 32768, 0], + [18635, 32768, 0], + [24715, 32768, 0], + [31484, 32768, 0], + [4656, 32768, 0], + [16074, 32768, 0], + [24704, 32768, 0], + [1806, 32768, 0], + [14645, 32768, 0], + [25336, 32768, 0], + ], + [ + [31539, 32768, 0], + [8433, 32768, 0], + [20576, 32768, 0], + [27904, 32768, 0], + [27852, 32768, 0], + [30026, 32768, 0], + [32441, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + [16384, 32768, 0], + ], + ], +]; + +pub static DEFAULT_UV_MODE_CFL_ALLOWED_CDF: [[u16; 15]; 13] = [ + [10407, 11208, 12900, 13181, 13823, 14175, 14899, 15656, 15986, 20086, 20995, 22455, 24212, 32768, 0], + [4532, 19780, 20057, 20215, 20428, 21071, 21199, 21451, 22099, 24228, 24693, 27032, 29472, 32768, 0], + [5273, 5379, 20177, 20270, 20385, 20439, 20949, 21695, 21774, 23138, 24256, 24703, 26679, 32768, 0], + [6740, 7167, 7662, 14152, 14536, 14785, 15034, 16741, 18371, 21520, 22206, 23389, 24182, 32768, 0], + [4987, 5368, 5928, 6068, 19114, 20315, 21857, 22253, 22411, 24911, 25380, 26027, 26376, 32768, 0], + [5370, 6889, 7247, 7393, 9498, 21114, 21402, 21753, 21981, 24780, 25386, 26517, 27176, 32768, 0], + [4816, 4961, 7204, 7326, 8765, 8930, 20169, 20682, 20803, 23188, 23763, 24455, 24940, 32768, 0], + [6608, 6740, 8529, 9049, 9257, 9356, 9735, 18827, 19059, 22336, 23204, 23964, 24793, 32768, 0], + [5998, 7419, 7781, 8933, 9255, 9549, 9753, 10417, 18898, 22494, 23139, 24764, 25989, 32768, 0], + [10660, 11298, 12550, 12957, 13322, 13624, 14040, 15004, 15534, 20714, 21789, 23443, 24861, 32768, 0], + [10522, 11530, 12552, 12963, 13378, 13779, 14245, 15235, 15902, 20102, 22696, 23774, 25838, 32768, 0], + [10099, 10691, 12639, 13049, 13386, 13665, 14125, 15163, 15636, 19676, 20474, 23519, 25208, 32768, 0], + [3144, 5087, 7382, 7504, 7593, 7690, 7801, 8064, 8232, 9248, 9875, 10521, 29048, 32768, 0], +]; + +pub static DEFAULT_UV_MODE_CFL_NOT_ALLOWED_CDF: [[u16; 14]; 13] = [ + [22631, 24152, 25378, 25661, 25986, 26520, 27055, 27923, 28244, 30059, 30941, 31961, 32768, 0], + [9513, 26881, 26973, 27046, 27118, 27664, 27739, 27824, 28359, 29505, 29800, 31796, 32768, 0], + [9845, 9915, 28663, 28704, 28757, 28780, 29198, 29822, 29854, 30764, 31777, 32029, 32768, 0], + [13639, 13897, 14171, 25331, 25606, 25727, 25953, 27148, 28577, 30612, 31355, 32493, 32768, 0], + [9764, 9835, 9930, 9954, 25386, 27053, 27958, 28148, 28243, 31101, 31744, 32363, 32768, 0], + [11825, 13589, 13677, 13720, 15048, 29213, 29301, 29458, 29711, 31161, 31441, 32550, 32768, 0], + [14175, 14399, 16608, 16821, 17718, 17775, 28551, 30200, 30245, 31837, 32342, 32667, 32768, 0], + [12885, 13038, 14978, 15590, 15673, 15748, 16176, 29128, 29267, 30643, 31961, 32461, 32768, 0], + [12026, 13661, 13874, 15305, 15490, 15726, 15995, 16273, 28443, 30388, 30767, 32416, 32768, 0], + [19052, 19840, 20579, 20916, 21150, 21467, 21885, 22719, 23174, 28861, 30379, 32175, 32768, 0], + [18627, 19649, 20974, 21219, 21492, 21816, 22199, 23119, 23527, 27053, 31397, 32148, 32768, 0], + [17026, 19004, 19997, 20339, 20586, 21103, 21349, 21907, 22482, 25896, 26541, 31819, 32768, 0], + [12124, 13759, 14959, 14992, 15007, 15051, 15078, 15166, 15255, 15753, 16039, 16606, 32768, 0], +]; + +pub static DEFAULT_Y_MODE_CDF: [[u16; 14]; 4] = [ + [22801, 23489, 24293, 24756, 25601, 26123, 26606, 27418, 27945, 29228, 29685, 30349, 32768, 0], + [18673, 19845, 22631, 23318, 23950, 24649, 25527, 27364, 28152, 29701, 29984, 30852, 32768, 0], + [19770, 20979, 23396, 23939, 24241, 24654, 25136, 27073, 27830, 29360, 29730, 30659, 32768, 0], + [20155, 21301, 22838, 23178, 23261, 23533, 23703, 24804, 25352, 26575, 27016, 28049, 32768, 0], +]; + +pub static DEFAULT_ZERO_MV_CDF: [[u16; 3]; 2] = [[2175, 32768, 0], [1054, 32768, 0]]; + +pub static INTRA_MODE_CONTEXT: [u8; 13] = [0, 1, 2, 3, 4, 4, 4, 4, 3, 0, 1, 2, 0]; + +pub static MAG_REF_OFFSET_WITH_TX_CLASS: [[[u8; 2]; 3]; 3] = [[[0, 1], [1, 0], [1, 1]], [[0, 1], [1, 0], [0, 2]], [[0, 1], [1, 0], [2, 0]]]; + +pub static MAX_TX_SIZE_RECT: [u8; 22] = [0, 5, 6, 1, 7, 8, 2, 9, 10, 3, 11, 12, 4, 4, 4, 4, 13, 14, 15, 16, 17, 18]; + +pub static MI_HEIGHT_LOG2: [u8; 22] = [0, 1, 0, 1, 2, 1, 2, 3, 2, 3, 4, 3, 4, 5, 4, 5, 2, 0, 3, 1, 4, 2]; + +pub static MI_WIDTH_LOG2: [u8; 22] = [0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 0, 2, 1, 3, 2, 4]; + +pub static MODE_TO_TXFM: [u8; 14] = [0, 1, 2, 0, 3, 1, 2, 2, 1, 3, 1, 2, 3, 0]; + +pub static NUM_4X4_BLOCKS_HIGH: [u8; 22] = [1, 2, 1, 2, 4, 2, 4, 8, 4, 8, 16, 8, 16, 32, 16, 32, 4, 1, 8, 2, 16, 4]; + +pub static NUM_4X4_BLOCKS_WIDE: [u8; 22] = [1, 1, 2, 2, 2, 4, 4, 4, 8, 8, 8, 16, 16, 16, 32, 32, 1, 4, 2, 8, 4, 16]; + +pub static PARTITION_SUBSIZE: [[u8; 22]; 10] = [ + [0, 22, 22, 3, 22, 22, 6, 22, 22, 9, 22, 22, 12, 22, 22, 15, 22, 22, 22, 22, 22, 22], + [22, 22, 22, 2, 22, 22, 5, 22, 22, 8, 22, 22, 11, 22, 22, 14, 22, 22, 22, 22, 22, 22], + [22, 22, 22, 1, 22, 22, 4, 22, 22, 7, 22, 22, 10, 22, 22, 13, 22, 22, 22, 22, 22, 22], + [22, 22, 22, 0, 22, 22, 3, 22, 22, 6, 22, 22, 9, 22, 22, 12, 22, 22, 22, 22, 22, 22], + [22, 22, 22, 2, 22, 22, 5, 22, 22, 8, 22, 22, 11, 22, 22, 14, 22, 22, 22, 22, 22, 22], + [22, 22, 22, 2, 22, 22, 5, 22, 22, 8, 22, 22, 11, 22, 22, 14, 22, 22, 22, 22, 22, 22], + [22, 22, 22, 1, 22, 22, 4, 22, 22, 7, 22, 22, 10, 22, 22, 13, 22, 22, 22, 22, 22, 22], + [22, 22, 22, 1, 22, 22, 4, 22, 22, 7, 22, 22, 10, 22, 22, 13, 22, 22, 22, 22, 22, 22], + [22, 22, 22, 22, 22, 22, 17, 22, 22, 19, 22, 22, 21, 22, 22, 22, 22, 22, 22, 22, 22, 22], + [22, 22, 22, 22, 22, 22, 16, 22, 22, 18, 22, 22, 20, 22, 22, 22, 22, 22, 22, 22, 22, 22], +]; + +pub static SIG_REF_DIFF_OFFSET: [[[u8; 2]; 5]; 3] = + [[[0, 1], [1, 0], [1, 1], [0, 2], [2, 0]], [[0, 1], [1, 0], [0, 2], [0, 3], [0, 4]], [[0, 1], [1, 0], [2, 0], [3, 0], [4, 0]]]; + +pub static SIZE_GROUP: [u8; 22] = [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 0, 0, 1, 1, 2, 2]; + +pub static SM_WEIGHTS_TX_16X16: [u8; 16] = [255, 225, 196, 170, 145, 123, 102, 84, 68, 54, 43, 33, 26, 20, 17, 16]; + +pub static SM_WEIGHTS_TX_32X32: [u8; 32] = + [255, 240, 225, 210, 196, 182, 169, 157, 145, 133, 122, 111, 101, 92, 83, 74, 66, 59, 52, 45, 39, 34, 29, 25, 21, 17, 14, 12, 10, 9, 8, 8]; + +pub static SM_WEIGHTS_TX_4X4: [u8; 4] = [255, 149, 85, 64]; + +pub static SM_WEIGHTS_TX_8X8: [u8; 8] = [255, 197, 146, 105, 73, 50, 37, 32]; + +pub static SUBPEL_FILTERS: [[[i16; 8]; 16]; 6] = [ + [ + [0, 0, 0, 128, 0, 0, 0, 0], + [0, 2, -6, 126, 8, -2, 0, 0], + [0, 2, -10, 122, 18, -4, 0, 0], + [0, 2, -12, 116, 28, -8, 2, 0], + [0, 2, -14, 110, 38, -10, 2, 0], + [0, 2, -14, 102, 48, -12, 2, 0], + [0, 2, -16, 94, 58, -12, 2, 0], + [0, 2, -14, 84, 66, -12, 2, 0], + [0, 2, -14, 76, 76, -14, 2, 0], + [0, 2, -12, 66, 84, -14, 2, 0], + [0, 2, -12, 58, 94, -16, 2, 0], + [0, 2, -12, 48, 102, -14, 2, 0], + [0, 2, -10, 38, 110, -14, 2, 0], + [0, 2, -8, 28, 116, -12, 2, 0], + [0, 0, -4, 18, 122, -10, 2, 0], + [0, 0, -2, 8, 126, -6, 2, 0], + ], + [ + [0, 0, 0, 128, 0, 0, 0, 0], + [0, 2, 28, 62, 34, 2, 0, 0], + [0, 0, 26, 62, 36, 4, 0, 0], + [0, 0, 22, 62, 40, 4, 0, 0], + [0, 0, 20, 60, 42, 6, 0, 0], + [0, 0, 18, 58, 44, 8, 0, 0], + [0, 0, 16, 56, 46, 10, 0, 0], + [0, -2, 16, 54, 48, 12, 0, 0], + [0, -2, 14, 52, 52, 14, -2, 0], + [0, 0, 12, 48, 54, 16, -2, 0], + [0, 0, 10, 46, 56, 16, 0, 0], + [0, 0, 8, 44, 58, 18, 0, 0], + [0, 0, 6, 42, 60, 20, 0, 0], + [0, 0, 4, 40, 62, 22, 0, 0], + [0, 0, 4, 36, 62, 26, 0, 0], + [0, 0, 2, 34, 62, 28, 2, 0], + ], + [ + [0, 0, 0, 128, 0, 0, 0, 0], + [-2, 2, -6, 126, 8, -2, 2, 0], + [-2, 6, -12, 124, 16, -6, 4, -2], + [-2, 8, -18, 120, 26, -10, 6, -2], + [-4, 10, -22, 116, 38, -14, 6, -2], + [-4, 10, -22, 108, 48, -18, 8, -2], + [-4, 10, -24, 100, 60, -20, 8, -2], + [-4, 10, -24, 90, 70, -22, 10, -2], + [-4, 12, -24, 80, 80, -24, 12, -4], + [-2, 10, -22, 70, 90, -24, 10, -4], + [-2, 8, -20, 60, 100, -24, 10, -4], + [-2, 8, -18, 48, 108, -22, 10, -4], + [-2, 6, -14, 38, 116, -22, 10, -4], + [-2, 6, -10, 26, 120, -18, 8, -2], + [-2, 4, -6, 16, 124, -12, 6, -2], + [0, 2, -2, 8, 126, -6, 2, -2], + ], + [ + [0, 0, 0, 128, 0, 0, 0, 0], + [0, 0, 0, 120, 8, 0, 0, 0], + [0, 0, 0, 112, 16, 0, 0, 0], + [0, 0, 0, 104, 24, 0, 0, 0], + [0, 0, 0, 96, 32, 0, 0, 0], + [0, 0, 0, 88, 40, 0, 0, 0], + [0, 0, 0, 80, 48, 0, 0, 0], + [0, 0, 0, 72, 56, 0, 0, 0], + [0, 0, 0, 64, 64, 0, 0, 0], + [0, 0, 0, 56, 72, 0, 0, 0], + [0, 0, 0, 48, 80, 0, 0, 0], + [0, 0, 0, 40, 88, 0, 0, 0], + [0, 0, 0, 32, 96, 0, 0, 0], + [0, 0, 0, 24, 104, 0, 0, 0], + [0, 0, 0, 16, 112, 0, 0, 0], + [0, 0, 0, 8, 120, 0, 0, 0], + ], + [ + [0, 0, 0, 128, 0, 0, 0, 0], + [0, 0, -4, 126, 8, -2, 0, 0], + [0, 0, -8, 122, 18, -4, 0, 0], + [0, 0, -10, 116, 28, -6, 0, 0], + [0, 0, -12, 110, 38, -8, 0, 0], + [0, 0, -12, 102, 48, -10, 0, 0], + [0, 0, -14, 94, 58, -10, 0, 0], + [0, 0, -12, 84, 66, -10, 0, 0], + [0, 0, -12, 76, 76, -12, 0, 0], + [0, 0, -10, 66, 84, -12, 0, 0], + [0, 0, -10, 58, 94, -14, 0, 0], + [0, 0, -10, 48, 102, -12, 0, 0], + [0, 0, -8, 38, 110, -12, 0, 0], + [0, 0, -6, 28, 116, -10, 0, 0], + [0, 0, -4, 18, 122, -8, 0, 0], + [0, 0, -2, 8, 126, -4, 0, 0], + ], + [ + [0, 0, 0, 128, 0, 0, 0, 0], + [0, 0, 30, 62, 34, 2, 0, 0], + [0, 0, 26, 62, 36, 4, 0, 0], + [0, 0, 22, 62, 40, 4, 0, 0], + [0, 0, 20, 60, 42, 6, 0, 0], + [0, 0, 18, 58, 44, 8, 0, 0], + [0, 0, 16, 56, 46, 10, 0, 0], + [0, 0, 14, 54, 48, 12, 0, 0], + [0, 0, 12, 52, 52, 12, 0, 0], + [0, 0, 12, 48, 54, 14, 0, 0], + [0, 0, 10, 46, 56, 16, 0, 0], + [0, 0, 8, 44, 58, 18, 0, 0], + [0, 0, 6, 42, 60, 20, 0, 0], + [0, 0, 4, 40, 62, 22, 0, 0], + [0, 0, 4, 36, 62, 26, 0, 0], + [0, 0, 2, 34, 62, 30, 0, 0], + ], +]; + +pub static SUBSAMPLED_SIZE: [[[u8; 2]; 2]; 22] = [ + [[0, 0], [0, 0]], + [[1, 0], [22, 0]], + [[2, 22], [0, 0]], + [[3, 2], [1, 0]], + [[4, 3], [22, 1]], + [[5, 22], [3, 2]], + [[6, 5], [4, 3]], + [[7, 6], [22, 4]], + [[8, 22], [6, 5]], + [[9, 8], [7, 6]], + [[10, 9], [22, 7]], + [[11, 22], [9, 8]], + [[12, 11], [10, 9]], + [[13, 12], [22, 10]], + [[14, 22], [12, 11]], + [[15, 14], [13, 12]], + [[16, 1], [22, 1]], + [[17, 22], [2, 2]], + [[18, 4], [22, 16]], + [[19, 22], [5, 17]], + [[20, 7], [22, 18]], + [[21, 22], [8, 19]], +]; + +pub static TRANSFORM_ROW_SHIFT: [u8; 19] = [0, 1, 2, 2, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2]; + +pub static TX_HEIGHT: [u8; 19] = [4, 8, 16, 32, 64, 8, 4, 16, 8, 32, 16, 64, 32, 16, 4, 32, 8, 64, 16]; + +pub static TX_HEIGHT_LOG2: [u8; 19] = [2, 3, 4, 5, 6, 3, 2, 4, 3, 5, 4, 6, 5, 4, 2, 5, 3, 6, 4]; + +pub static TX_SIZE_SQR: [u8; 19] = [0, 1, 2, 3, 4, 0, 0, 1, 1, 2, 2, 3, 3, 0, 0, 1, 1, 2, 2]; + +pub static TX_SIZE_SQR_UP: [u8; 19] = [0, 1, 2, 3, 4, 1, 1, 2, 2, 3, 3, 4, 4, 2, 2, 3, 3, 4, 4]; + +pub static TX_TYPE_IN_SET_INTER: [[u8; 16]; 4] = [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0], + [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0], +]; + +pub static TX_TYPE_IN_SET_INTRA: [[u8; 16]; 3] = [ + [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0], + [1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0], +]; + +pub static TX_TYPE_INTER_INV_SET3: [u8; 2] = [9, 0]; + +pub static TX_TYPE_INTRA_INV_SET2: [u8; 5] = [9, 0, 3, 1, 2]; + +pub static TX_WIDTH: [u8; 19] = [4, 8, 16, 32, 64, 4, 8, 8, 16, 16, 32, 32, 64, 4, 16, 8, 32, 16, 64]; + +pub static TX_WIDTH_LOG2: [u8; 19] = [2, 3, 4, 5, 6, 2, 3, 3, 4, 4, 5, 5, 6, 2, 4, 3, 5, 4, 6]; diff --git a/crates/av1enc/src/transform.rs b/crates/av1enc/src/transform.rs new file mode 100644 index 0000000000..9d81ead0bb --- /dev/null +++ b/crates/av1enc/src/transform.rs @@ -0,0 +1,575 @@ +//! Transforms. The inverse transforms are the normative ones of spec 7.13 (mirrored from +//! filmcraft-av1, so that the encoder reconstructs exactly like a decoder); the forward +//! transforms are floating-point matrices scaled to match them (not normative). + +use std::sync::OnceLock; + +use crate::tables::{ADST_ADST, ADST_DCT, COS128_LOOKUP, DCT_ADST, DCT_DCT, TRANSFORM_ROW_SHIFT, TX_WIDTH_LOG2}; + +#[inline(always)] +fn round2_64(x: i64, n: u32) -> i32 { + if n == 0 { x as i32 } else { ((x + (1i64 << (n - 1))) >> n) as i32 } +} + +#[inline(always)] +pub(crate) fn round2(x: i32, n: u32) -> i32 { + if n == 0 { x } else { (x + (1 << (n - 1))) >> n } +} + +#[inline(always)] +fn cos128(angle: i32) -> i64 { + let a = (angle & 255) as usize; + (match a { + 0..=64 => COS128_LOOKUP[a] as i32, + 65..=128 => -(COS128_LOOKUP[128 - a] as i32), + 129..=192 => -(COS128_LOOKUP[a - 128] as i32), + _ => COS128_LOOKUP[256 - a] as i32, + }) as i64 +} + +#[inline(always)] +fn sin128(angle: i32) -> i64 { + cos128(angle - 64) +} + +#[inline(always)] +fn brev(num_bits: u32, x: usize) -> usize { + let mut t = 0; + for i in 0..num_bits { + let bit = (x >> i) & 1; + t += bit << (num_bits - 1 - i); + } + t +} + +#[inline(always)] +fn bf(t: &mut [i32], a: usize, b: usize, angle: i32, flip: bool) { + let x = t[a] as i64 * cos128(angle) - t[b] as i64 * sin128(angle); + let y = t[a] as i64 * sin128(angle) + t[b] as i64 * cos128(angle); + t[a] = round2_64(x, 12); + t[b] = round2_64(y, 12); + if flip { + t.swap(a, b); + } +} + +#[inline(always)] +fn hd(t: &mut [i32], a: usize, b: usize, flip: bool, r: u32) { + let (a, b) = if flip { (b, a) } else { (a, b) }; + let x = t[a]; + let y = t[b]; + let lo = -(1i32 << (r - 1)); + let hi = (1i32 << (r - 1)) - 1; + t[a] = (x + y).clamp(lo, hi); + t[b] = (x - y).clamp(lo, hi); +} + +/// Inverse DCT of length 2^N, N = 2..=5 (7.13.2.3), in place. `N` is a const generic so that the +/// bit-reversal permutations and loop bounds fold at compile time. +#[inline(always)] +fn inverse_dct_n(t: &mut [i32], r: u32) { + let n = N; + let len = 1usize << n; + let mut copy = [0i32; 32]; + copy[..len].copy_from_slice(&t[..len]); + for i in 0..len { + t[i] = copy[brev(n, i)]; + } + inverse_dct_initial_stages(t, n, r); + inverse_dct_final_stages(t, n, r); +} + +/// Rotate the high-frequency branches before the four-point core. +#[inline(always)] +fn inverse_dct_initial_stages(t: &mut [i32], n: u32, r: u32) { + if n >= 5 { + for i in 0..8 { + bf(t, 16 + i, 31 - i, 6 + ((brev(3, 7 - i) as i32) << 3), false); + } + } + if n >= 4 { + for i in 0..4 { + bf(t, 8 + i, 15 - i, 12 + ((brev(2, 3 - i) as i32) << 4), false); + } + } + if n >= 5 { + for i in 0..8 { + hd(t, 16 + 2 * i, 17 + 2 * i, i & 1 == 1, r); + } + } + if n >= 3 { + for i in 0..2 { + bf(t, 4 + i, 7 - i, 56 - 32 * i as i32, false); + } + } + if n >= 4 { + for i in 0..4 { + hd(t, 8 + 2 * i, 9 + 2 * i, i & 1 == 1, r); + } + } + if n >= 5 { + for i in 0..2 { + for j in 0..2 { + bf(t, 30 - 4 * i - j, 17 + 4 * i + j, 24 + ((j as i32) << 6) + (((1 - i) as i32) << 5), true); + } + } + } +} + +/// Evaluate the four-point core and merge progressively larger branches. +#[inline(always)] +fn inverse_dct_final_stages(t: &mut [i32], n: u32, r: u32) { + for i in 0..2 { + bf(t, 2 * i, 2 * i + 1, 32 + 16 * i as i32, i == 0); + } + if n >= 3 { + for i in 0..2 { + hd(t, 4 + 2 * i, 5 + 2 * i, i == 1, r); + } + } + if n >= 4 { + for i in 0..2 { + bf(t, 14 - i, 9 + i, 48 + 64 * i as i32, true); + } + } + if n >= 5 { + for i in 0..4 { + for j in 0..2 { + hd(t, 16 + 4 * i + j, 19 + 4 * i - j, i & 1 == 1, r); + } + } + } + for i in 0..2 { + hd(t, i, 3 - i, false, r); + } + if n >= 3 { + bf(t, 6, 5, 32, true); + } + if n >= 4 { + for i in 0..2 { + for j in 0..2 { + hd(t, 8 + 4 * i + j, 11 + 4 * i - j, i == 1, r); + } + } + } + if n >= 5 { + for i in 0..4 { + bf(t, 29 - i, 18 + i, 48 + (i as i32 >> 1) * 64, true); + } + } + if n >= 3 { + for i in 0..4 { + hd(t, i, 7 - i, false, r); + } + } + if n >= 4 { + for i in 0..2 { + bf(t, 13 - i, 10 + i, 32, true); + } + } + if n >= 5 { + for i in 0..2 { + for j in 0..4 { + hd(t, 16 + i * 8 + j, 23 + i * 8 - j, i == 1, r); + } + } + } + if n >= 4 { + for i in 0..8 { + hd(t, i, 15 - i, false, r); + } + } + if n >= 5 { + for i in 0..4 { + bf(t, 27 - i, 20 + i, 32, true); + } + } + if n >= 5 { + for i in 0..16 { + hd(t, i, 31 - i, false, r); + } + } +} + +const SINPI_1_9: i64 = 1321; +const SINPI_2_9: i64 = 2482; +const SINPI_3_9: i64 = 3344; +const SINPI_4_9: i64 = 3803; + +#[inline(always)] +fn inverse_adst4(t: &mut [i32]) { + let (t0, t1, t2, t3) = (t[0] as i64, t[1] as i64, t[2] as i64, t[3] as i64); + let mut s = [SINPI_1_9 * t0, SINPI_2_9 * t0, SINPI_3_9 * t1, SINPI_4_9 * t2, SINPI_1_9 * t2, SINPI_2_9 * t3, SINPI_4_9 * t3]; + let a7 = t0 - t2; + let b7 = a7 + t3; + s[0] += s[3]; + s[1] -= s[4]; + s[3] = s[2]; + s[2] = SINPI_3_9 * b7; + s[0] += s[5]; + s[1] -= s[6]; + let x = [s[0] + s[3], s[1] + s[3], s[2], s[0] + s[1] - s[3]]; + for i in 0..4 { + t[i] = round2_64(x[i], 12); + } +} + +#[inline(always)] +fn adst_in_permute(t: &mut [i32]) { + let n = N; + let n0 = 1usize << n; + let mut copy = [0i32; 16]; + copy[..n0].copy_from_slice(&t[..n0]); + for i in 0..n0 { + let idx = if i & 1 == 1 { i - 1 } else { n0 - i - 1 }; + t[i] = copy[idx]; + } +} + +#[inline(always)] +fn adst_out_permute(t: &mut [i32]) { + let n = N; + let n0 = 1usize << n; + let mut copy = [0i32; 16]; + copy[..n0].copy_from_slice(&t[..n0]); + for i in 0..n0 { + let a = (i >> 3) & 1; + let b = ((i >> 2) & 1) ^ ((i >> 3) & 1); + let c = ((i >> 1) & 1) ^ ((i >> 2) & 1); + let d = (i & 1) ^ ((i >> 1) & 1); + let idx = ((d << 3) | (c << 2) | (b << 1) | a) >> (4 - n); + t[i] = if i & 1 == 1 { -copy[idx] } else { copy[idx] }; + } +} + +#[inline(always)] +fn inverse_adst8(t: &mut [i32], r: u32) { + adst_in_permute::<3>(t); + for i in 0..4 { + bf(t, 2 * i, 2 * i + 1, 60 - 16 * i as i32, true); + } + for i in 0..4 { + hd(t, i, 4 + i, false, r); + } + for i in 0..2 { + bf(t, 4 + 3 * i, 5 + i, 48 - 32 * i as i32, true); + } + for i in 0..2 { + for j in 0..2 { + hd(t, 4 * j + i, 2 + 4 * j + i, false, r); + } + } + for i in 0..2 { + bf(t, 2 + 4 * i, 3 + 4 * i, 32, true); + } + adst_out_permute::<3>(t); +} + +#[inline(always)] +fn inverse_adst16(t: &mut [i32], r: u32) { + adst_in_permute::<4>(t); + for i in 0..8 { + bf(t, 2 * i, 2 * i + 1, 62 - 8 * i as i32, true); + } + for i in 0..8 { + hd(t, i, 8 + i, false, r); + } + for i in 0..2 { + bf(t, 8 + 2 * i, 9 + 2 * i, 56 - 32 * i as i32, true); + bf(t, 13 + 2 * i, 12 + 2 * i, 8 + 32 * i as i32, true); + } + for i in 0..4 { + for j in 0..2 { + hd(t, 8 * j + i, 4 + 8 * j + i, false, r); + } + } + for i in 0..2 { + for j in 0..2 { + bf(t, 4 + 8 * j + 3 * i, 5 + 8 * j + i, 48 - 32 * i as i32, true); + } + } + for i in 0..2 { + for j in 0..4 { + hd(t, 4 * j + i, 2 + 4 * j + i, false, r); + } + } + for i in 0..4 { + bf(t, 2 + 4 * i, 3 + 4 * i, 32, true); + } + adst_out_permute::<4>(t); +} + +/// 1-D transform kind. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Kind { + Dct, + Adst, +} + +/// (vertical / column kind, horizontal / row kind) of a transform type. +pub(crate) fn kinds(tx_type: usize) -> (Kind, Kind) { + match tx_type { + DCT_DCT => (Kind::Dct, Kind::Dct), + ADST_DCT => (Kind::Adst, Kind::Dct), + DCT_ADST => (Kind::Dct, Kind::Adst), + ADST_ADST => (Kind::Adst, Kind::Adst), + // The encoder only chooses the four types above. + _ => { + debug_assert!(false, "unsupported transform type {tx_type}"); + (Kind::Dct, Kind::Dct) + } + } +} + +#[inline(always)] +fn inverse_dct(t: &mut [i32], n: u32, r: u32) { + match n { + 2 => inverse_dct_n::<2>(t, r), + 3 => inverse_dct_n::<3>(t, r), + 4 => inverse_dct_n::<4>(t, r), + _ => inverse_dct_n::<5>(t, r), + } +} + +#[inline(always)] +fn inverse_1d(t: &mut [i32], kind: Kind, n: u32, r: u32) { + match kind { + Kind::Dct => inverse_dct(t, n, r), + Kind::Adst => match n { + 2 => inverse_adst4(t), + 3 => inverse_adst8(t, r), + _ => inverse_adst16(t, r), + }, + } +} + +/// 2-D inverse transform (7.13.3) of a square `n x n` block (n = 4..=32) of dequantised +/// coefficients (row stride `n`) into `residual` (row stride `n`). +pub(crate) fn inverse_2d(dequant: &[i32], residual: &mut [i32], tx_sz: usize, tx_type: usize, bit_depth: u32) { + match TX_WIDTH_LOG2[tx_sz] { + 2 => inverse_2d_n::<2>(dequant, residual, tx_sz, tx_type, bit_depth), + 3 => inverse_2d_n::<3>(dequant, residual, tx_sz, tx_type, bit_depth), + 4 => inverse_2d_n::<4>(dequant, residual, tx_sz, tx_type, bit_depth), + _ => inverse_2d_n::<5>(dequant, residual, tx_sz, tx_type, bit_depth), + } +} + +/// `inverse_2d` for a block of width `1 << LOG2`. +fn inverse_2d_n(dequant: &[i32], residual: &mut [i32], tx_sz: usize, tx_type: usize, bit_depth: u32) { + let log2 = LOG2; + let n = 1usize << log2; + let row_shift = TRANSFORM_ROW_SHIFT[tx_sz] as u32; + let row_clamp = bit_depth + 8; + let col_clamp = (bit_depth + 6).max(16); + let (ck, rk) = kinds(tx_type); + let lo = -(1i32 << (col_clamp - 1)); + let hi = (1i32 << (col_clamp - 1)) - 1; + let mut t = [0i32; 32]; + for i in 0..n { + let row = &dequant[i * n..i * n + n]; + if row.iter().all(|&v| v == 0) { + residual[i * n..i * n + n].iter_mut().for_each(|v| *v = 0); + continue; + } + t[..n].copy_from_slice(row); + inverse_1d(&mut t, rk, log2, row_clamp); + for j in 0..n { + residual[i * n + j] = round2(t[j], row_shift).clamp(lo, hi); + } + } + for j in 0..n { + for i in 0..n { + t[i] = residual[i * n + j]; + } + inverse_1d(&mut t, ck, log2, col_clamp); + for i in 0..n { + residual[i * n + j] = round2(t[i], 4); + } + } +} + +/// Orthonormal forward basis matrices `[k][i]` (row k = basis function k), sizes 4..=32. +struct Bases { + dct: [Vec; 4], + adst: [Vec; 3], + /// The same matrices transposed (`[i][k]`). + dct_t: [Vec; 4], + adst_t: [Vec; 3], +} + +fn transpose(m: &[f32]) -> Vec { + let n = (m.len() as f64).sqrt() as usize; + let mut t = vec![0f32; n * n]; + for k in 0..n { + for i in 0..n { + t[i * n + k] = m[k * n + i]; + } + } + t +} + +fn bases() -> &'static Bases { + static B: OnceLock = OnceLock::new(); + B.get_or_init(|| { + let pi = std::f64::consts::PI; + let dct = std::array::from_fn(|l| { + let n = 4usize << l; + let mut m = vec![0f32; n * n]; + for k in 0..n { + let c = if k == 0 { std::f64::consts::FRAC_1_SQRT_2 } else { 1.0 }; + for i in 0..n { + m[k * n + i] = (c * (2.0 / n as f64).sqrt() * (pi * (2 * i + 1) as f64 * k as f64 / (2 * n) as f64).cos()) as f32; + } + } + m + }); + let adst = std::array::from_fn(|l| { + let n = 4usize << l; + let mut m = vec![0f32; n * n]; + for k in 0..n { + for i in 0..n { + m[k * n + i] = if n == 4 { + // the AV1 4-point ADST (sin(pi (i + 1)(2k + 1) / 9) basis) + (2.0 / 3.0 * (pi * ((i + 1) * (2 * k + 1)) as f64 / 9.0).sin()) as f32 + } else { + ((2.0 / n as f64).sqrt() * (pi * ((2 * i + 1) * (2 * k + 1)) as f64 / (4 * n) as f64).sin()) as f32 + }; + } + } + m + }); + let dct_t = std::array::from_fn(|l| transpose(&dct[l])); + let adst_t = std::array::from_fn(|l| transpose(&adst[l])); + Bases { dct, adst, dct_t, adst_t } + }) +} + +fn basis_t(kind: Kind, n: usize) -> &'static [f32] { + let l = n.trailing_zeros() as usize - 2; + match kind { + Kind::Dct => &bases().dct_t[l], + Kind::Adst => &bases().adst_t[l], + } +} + +fn basis(kind: Kind, n: usize) -> &'static [f32] { + let l = n.trailing_zeros() as usize - 2; + match kind { + Kind::Dct => &bases().dct[l], + Kind::Adst => &bases().adst[l], + } +} + +/// Forward 2-D transform of the `n x n` residual (row stride `n`) into orthonormal-domain +/// coefficients scaled by 8 (so that `level = out / q` with the spec's dequantiser). +pub(crate) fn forward_2d(residual: &[i32], out: &mut [f32], n: usize, tx_type: usize) { + match n { + 4 => forward_2d_n::<4, 16>(residual, out, tx_type), + 8 => forward_2d_n::<8, 64>(residual, out, tx_type), + 16 => forward_2d_n::<16, 256>(residual, out, tx_type), + _ => forward_2d_dyn(residual, out, n, tx_type), + } +} + +/// `forward_2d` for a fixed size, so `tmp` is exactly `NN = N * N` floats (no 4 KiB zeroing). +#[inline(always)] +fn forward_2d_n(residual: &[i32], out: &mut [f32], tx_type: usize) { + let n = N; + let (ck, rk) = kinds(tx_type); + let bc = &basis(ck, n)[..NN]; + let brt = &basis_t(rk, n)[..NN]; + let residual = &residual[..NN]; + let out = &mut out[..NN]; + let mut tmp = [0f32; NN]; + for i in 0..n { + let t = &mut tmp[i * n..i * n + n]; + for j in 0..n { + let r = residual[i * n + j]; + if r == 0 { + continue; + } + let r = r as f32; + for (o, &b) in t.iter_mut().zip(&brt[j * n..j * n + n]) { + *o += r * b; + } + } + } + out.iter_mut().for_each(|v| *v = 0.0); + for k in 0..n { + let o = &mut out[k * n..k * n + n]; + for i in 0..n { + let b = 8.0 * bc[k * n + i]; + for (v, &t) in o.iter_mut().zip(&tmp[i * n..i * n + n]) { + *v += b * t; + } + } + } +} + +/// 32x32 (and any other size): a const-generic 32 version measured slower. +fn forward_2d_dyn(residual: &[i32], out: &mut [f32], n: usize, tx_type: usize) { + let (ck, rk) = kinds(tx_type); + let bc = basis(ck, n); + let brt = basis_t(rk, n); + // tmp[i][k] = sum_j residual[i][j] * br[k][j] (horizontal; written as row updates so that + // the inner loops vectorise) + let mut tmp = [0f32; 32 * 32]; + for i in 0..n { + let t = &mut tmp[i * n..i * n + n]; + for j in 0..n { + let r = residual[i * n + j]; + if r == 0 { + continue; + } + let r = r as f32; + for (o, &b) in t.iter_mut().zip(&brt[j * n..j * n + n]) { + *o += r * b; + } + } + } + // out[k][j] = 8 * sum_i bc[k][i] * tmp[i][j] (vertical) + out[..n * n].iter_mut().for_each(|v| *v = 0.0); + for k in 0..n { + let o = &mut out[k * n..k * n + n]; + for i in 0..n { + let b = 8.0 * bc[k * n + i]; + for (v, &t) in o.iter_mut().zip(&tmp[i * n..i * n + n]) { + *v += b * t; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tables::{TX_4X4, TX_8X8, TX_16X16, TX_32X32}; + + /// forward (float) then the normative inverse reproduces the residual: the forward basis + /// and scale match the decoder's inverse transforms for every supported size and type. + #[test] + fn forward_matches_inverse() { + let mut seed = 99u32; + let mut rnd = || { + seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12345); + ((seed >> 16) % 201) as i32 - 100 + }; + for (tx, n) in [(TX_4X4, 4), (TX_8X8, 8), (TX_16X16, 16), (TX_32X32, 32)] { + let denom = if tx == TX_32X32 { 2.0 } else { 1.0 }; + for tx_type in [DCT_DCT, ADST_DCT, DCT_ADST, ADST_ADST] { + if n == 32 && tx_type != DCT_DCT { + continue; + } + let res: Vec = (0..n * n).map(|_| rnd()).collect(); + let mut coef = vec![0f32; n * n]; + forward_2d(&res, &mut coef, n, tx_type); + // q = 8 / denom keeps full precision: dequant = level * q / denom = coef + let deq: Vec = coef.iter().map(|&c| (c / denom).round() as i32).collect(); + let mut out = vec![0i32; n * n]; + inverse_2d(&deq, &mut out, tx, tx_type, 8); + let err: i32 = res.iter().zip(&out).map(|(a, b)| (a - b).abs()).max().unwrap(); + assert!(err <= 1, "tx {tx} type {tx_type}: max error {err}"); + } + } + } +} diff --git a/crates/av1enc/src/writer.rs b/crates/av1enc/src/writer.rs new file mode 100644 index 0000000000..d7842372e5 --- /dev/null +++ b/crates/av1enc/src/writer.rs @@ -0,0 +1,1040 @@ +//! Tile syntax writer (spec 5.11 with the 8.3 CDF selection). It keeps the same per-4x4 mode +//! info, coefficient contexts and adaptive CDFs as a decoder and writes the decisions taken by +//! `decide.rs` as symbols. The context derivations and the motion vector prediction process +//! (7.10.2) mirror the decoder exactly, since symbols and coded motion vector differences +//! depend on them. + +use crate::cdf::Cdfs; +use crate::decide::{FrameGeom, Leaf, Node, intra_uv_tx_type, scan_for, tx_in_set, tx_set, tx_size_for}; +use crate::loopfilter::LfInfo; +use crate::symbol::SymbolWriter; +use crate::tables::{ + ADST_ADST, ADST_DCT, ALTREF_FRAME, ALTREF2_FRAME, BLOCK_8X8, BLOCK_64X64, BR_CDF_SIZE, BWDREF_FRAME, CLASS0_SIZE, COEFF_BASE_CTX_OFFSET, + COEFF_BASE_RANGE, D67_PRED, DC_PRED, DCT_ADST, DCT_DCT, DEFAULT_INTRA_FRAME_Y_MODE_CDF, GLOBALMV, GOLDEN_FRAME, INTRA_FRAME, INTRA_MODE_CONTEXT, + LAST_FRAME, LAST2_FRAME, LAST3_FRAME, MAG_REF_OFFSET_WITH_TX_CLASS, MAX_ANGLE_DELTA, MAX_REF_MV_STACK_SIZE, MI_HEIGHT_LOG2, MI_WIDTH_LOG2, + MV_BORDER, MV_CLASS_0, NEARESTMV, NEARMV, NEWMV, NUM_4X4_BLOCKS_HIGH, NUM_4X4_BLOCKS_WIDE, NUM_BASE_LEVELS, PARTITION_HORZ, PARTITION_HORZ_4, + PARTITION_HORZ_A, PARTITION_HORZ_B, PARTITION_NONE, PARTITION_SPLIT, PARTITION_SUBSIZE, PARTITION_VERT, PARTITION_VERT_4, PARTITION_VERT_A, + PARTITION_VERT_B, REF_CAT_LEVEL, SIG_REF_DIFF_OFFSET, SIZE_GROUP, SUBSAMPLED_SIZE, TX_32X32, TX_CLASS_2D, TX_HEIGHT, TX_HEIGHT_LOG2, TX_SIZE_SQR, + TX_SIZE_SQR_UP, TX_TYPE_INTER_INV_SET3, TX_TYPE_INTRA_INV_SET2, TX_WIDTH, TX_WIDTH_LOG2, V_PRED, +}; + +type MvP = [i32; 2]; + +/// Mode info per 4x4 (MiSizes, YModes, RefFrames, Mvs, ...). +struct Mi { + cols: usize, + mi_size: Vec, + y_mode: Vec, + ref_frame: Vec<[i8; 2]>, + mv: Vec, + skip: Vec, + is_inter: Vec, + tx_type: Vec, + written: Vec, +} + +impl Mi { + #[inline(always)] + fn idx(&self, r: usize, c: usize) -> usize { + r * self.cols + c + } +} + +/// Output of find_mv_stack. +#[derive(Default)] +struct MvStack { + ref_stack_mv: [MvP; MAX_REF_MV_STACK_SIZE + 1], + weight_stack: [u32; MAX_REF_MV_STACK_SIZE + 1], + num_mv_found: usize, + new_mv_count: usize, + found_match: bool, + close_matches: usize, + total_matches: usize, + new_mv_context: usize, + ref_mv_context: usize, + zero_mv_context: usize, + drl_ctx_stack: [usize; MAX_REF_MV_STACK_SIZE + 1], +} + +pub(crate) struct TileWriter<'a> { + g: &'a FrameGeom, + intra_only: bool, + qidx: u8, + sw: SymbolWriter, + cdf: Box, + kf_y_mode: [[[u16; 14]; 5]; 5], + mi: Mi, + above_level: [Vec; 3], + above_dc: [Vec; 3], + left_level: [Vec; 3], + left_dc: [Vec; 3], + /// LoopfilterTxSizes per plane (row stride mi_cols + 32). + lf_tx: [Vec; 3], + /// Scratch for the levels coded so far in `coeffs`. + quant: Vec, + // current block + r: usize, + c: usize, + bsize: usize, + avail_u: bool, + avail_l: bool, + mvs: MvStack, +} + +fn block_size_px(bs: usize) -> usize { + 4 * NUM_4X4_BLOCKS_WIDE[bs] as usize +} + +fn is_directional(mode: usize) -> bool { + (V_PRED..=D67_PRED).contains(&mode) +} + +impl<'a> TileWriter<'a> { + /// `init`: the CDFs saved by the primary reference frame (load_cdfs), or None for the + /// defaults (init_non_coeff_cdfs / init_coeff_cdfs). + pub fn new(g: &'a FrameGeom, intra_only: bool, qidx: u8, init: Option<&Cdfs>) -> Self { + let n = g.mi_cols * g.mi_rows; + TileWriter { + g, + intra_only, + qidx, + sw: SymbolWriter::new(false), + cdf: match init { + Some(c) => { + let mut c = Box::new(c.clone()); + c.clear_counts(); + c + } + None => Cdfs::new(qidx as u32), + }, + kf_y_mode: DEFAULT_INTRA_FRAME_Y_MODE_CDF, + mi: Mi { + cols: g.mi_cols, + mi_size: vec![0; n], + y_mode: vec![0; n], + ref_frame: vec![[0, -1]; n], + mv: vec![[0, 0]; n], + skip: vec![false; n], + is_inter: vec![false; n], + tx_type: vec![0; n], + written: vec![false; n], + }, + above_level: std::array::from_fn(|_| vec![0; g.mi_cols + 64]), + above_dc: std::array::from_fn(|_| vec![0; g.mi_cols + 64]), + left_level: std::array::from_fn(|_| vec![0; g.mi_rows + 64]), + left_dc: std::array::from_fn(|_| vec![0; g.mi_rows + 64]), + lf_tx: std::array::from_fn(|_| vec![0; (g.mi_cols + 32) * (g.mi_rows + 32)]), + quant: Vec::new(), + r: 0, + c: 0, + bsize: 0, + avail_u: false, + avail_l: false, + mvs: MvStack::default(), + } + } + + /// The tile data and the final CDFs (saved for the frame-end CDF update). + pub fn finish(self) -> (Vec, Box) { + (self.sw.finish(), self.cdf) + } + + /// The mode info the loop filter needs. + pub fn lf_info(&self) -> LfInfo<'_> { + LfInfo { + mi_cols: self.g.mi_cols, + mi_rows: self.g.mi_rows, + width: self.g.width, + height: self.g.height, + mi_size: &self.mi.mi_size, + skip: &self.mi.skip, + is_inter: &self.mi.is_inter, + tx: [&self.lf_tx[0], &self.lf_tx[1], &self.lf_tx[2]], + } + } + + /// Start of a superblock row (clear_left_context). + pub fn start_sb_row(&mut self) { + for p in 0..3 { + self.left_level[p].iter_mut().for_each(|v| *v = 0); + self.left_dc[p].iter_mut().for_each(|v| *v = 0); + } + } + + pub fn superblock(&mut self, r: usize, c: usize, node: &Node) { + self.partition(r, c, BLOCK_64X64, node); + } + + fn partition(&mut self, r: usize, c: usize, bsize: usize, node: &Node) { + let (mi_rows, mi_cols) = (self.g.mi_rows, self.g.mi_cols); + if r >= mi_rows || c >= mi_cols { + return; + } + let avail_u = r > 0; + let avail_l = c > 0; + let n4 = NUM_4X4_BLOCKS_WIDE[bsize] as usize; + let half = n4 >> 1; + let has_rows = r + half < mi_rows; + let has_cols = c + half < mi_cols; + let split = matches!(node, Node::Split(_)); + let bsl = MI_WIDTH_LOG2[bsize] as usize; + let above = avail_u && (MI_WIDTH_LOG2[self.mi.mi_size[self.mi.idx(r - 1, c)] as usize] as usize) < bsl; + let left = avail_l && (MI_HEIGHT_LOG2[self.mi.mi_size[self.mi.idx(r, c - 1)] as usize] as usize) < bsl; + let ctx = left as usize * 2 + above as usize; + let partition = if split { PARTITION_SPLIT } else { PARTITION_NONE }; + if has_rows && has_cols { + let cdf: &mut [u16] = match bsl { + 1 => &mut self.cdf.partition_w8[ctx], + 2 => &mut self.cdf.partition_w16[ctx], + 3 => &mut self.cdf.partition_w32[ctx], + _ => &mut self.cdf.partition_w64[ctx], + }; + self.sw.symbol(partition, cdf); + } else if has_cols || has_rows { + assert!(split, "blocks crossing the frame edge are split"); + let pcdf: &[u16] = match bsl { + 2 => &self.cdf.partition_w16[ctx], + 3 => &self.cdf.partition_w32[ctx], + _ => &self.cdf.partition_w64[ctx], + }; + let pr = |k: usize| pcdf[k] as i32 - if k == 0 { 0 } else { pcdf[k - 1] as i32 }; + let psum = if has_cols { + // split_or_horz + pr(PARTITION_VERT) + pr(PARTITION_SPLIT) + pr(PARTITION_HORZ_A) + pr(PARTITION_VERT_A) + pr(PARTITION_VERT_B) + pr(PARTITION_VERT_4) + } else { + // split_or_vert + pr(PARTITION_HORZ) + pr(PARTITION_SPLIT) + pr(PARTITION_HORZ_A) + pr(PARTITION_HORZ_B) + pr(PARTITION_VERT_A) + pr(PARTITION_HORZ_4) + }; + let cdf = [((1i32 << 15) - psum) as u16, 1 << 15, 0]; + self.sw.symbol_fixed(1, &cdf); + } else { + assert!(split); + } + match node { + Node::Split(ch) => { + let sub = PARTITION_SUBSIZE[PARTITION_SPLIT][bsize] as usize; + self.partition(r, c, sub, &ch[0]); + self.partition(r, c + half, sub, &ch[1]); + self.partition(r + half, c, sub, &ch[2]); + self.partition(r + half, c + half, sub, &ch[3]); + } + Node::Leaf(leaf) => { + debug_assert_eq!(leaf.bsize, bsize); + self.block(r, c, leaf); + } + // Blocks inside the frame are always decided; nothing is coded outside it. + Node::Out => debug_assert!(false, "undecided block inside the frame"), + } + } + + fn block(&mut self, r: usize, c: usize, b: &Leaf) { + self.r = r; + self.c = c; + self.bsize = b.bsize; + self.avail_u = r > 0; + self.avail_l = c > 0; + let bsize = b.bsize; + let n4 = NUM_4X4_BLOCKS_WIDE[bsize] as usize; + // skip + let mut ctx = 0; + if self.avail_u { + ctx += self.mi.skip[self.mi.idx(r - 1, c)] as usize; + } + if self.avail_l { + ctx += self.mi.skip[self.mi.idx(r, c - 1)] as usize; + } + let mut y_mode = b.y_mode; + if self.intra_only { + self.sw.symbol(b.skip as usize, &mut self.cdf.skip[ctx]); + let above = if self.avail_u { self.mi.y_mode[self.mi.idx(r - 1, c)] as usize } else { DC_PRED }; + let left = if self.avail_l { self.mi.y_mode[self.mi.idx(r, c - 1)] as usize } else { DC_PRED }; + let actx = INTRA_MODE_CONTEXT[above] as usize; + let lctx = INTRA_MODE_CONTEXT[left] as usize; + self.sw.symbol(b.y_mode, &mut self.kf_y_mode[actx][lctx]); + self.intra_angle_and_uv(b); + } else { + self.sw.symbol(b.skip as usize, &mut self.cdf.skip[ctx]); + // is_inter + let left = if self.avail_l { self.mi.ref_frame[self.mi.idx(r, c - 1)] } else { [INTRA_FRAME as i8, -1] }; + let above = if self.avail_u { self.mi.ref_frame[self.mi.idx(r - 1, c)] } else { [INTRA_FRAME as i8, -1] }; + let left_intra = left[0] <= INTRA_FRAME as i8; + let above_intra = above[0] <= INTRA_FRAME as i8; + let ctx = if self.avail_u && self.avail_l { + if left_intra && above_intra { 3 } else { (left_intra || above_intra) as usize } + } else if self.avail_u || self.avail_l { + 2 * (if self.avail_u { above_intra } else { left_intra }) as usize + } else { + 0 + }; + self.sw.symbol(b.is_inter as usize, &mut self.cdf.is_inter[ctx]); + if b.is_inter { + y_mode = self.inter_block_mode_info(b, left, above); + } else { + let ctx = SIZE_GROUP[bsize] as usize; + self.sw.symbol(b.y_mode, &mut self.cdf.y_mode[ctx]); + self.intra_angle_and_uv(b); + } + } + // mode info stored before the residual + let (rows, cols) = (n4.min(self.g.mi_rows - r), n4.min(self.g.mi_cols - c)); + for y in 0..rows { + for x in 0..cols { + let i = self.mi.idx(r + y, c + x); + self.mi.y_mode[i] = y_mode as u8; + self.mi.ref_frame[i] = if b.is_inter { [LAST_FRAME as i8, -1] } else { [INTRA_FRAME as i8, -1] }; + self.mi.written[i] = true; + if b.is_inter { + self.mi.mv[i] = b.mv; + } + } + } + if b.skip { + // reset_block_context( ) + for plane in 0..3 { + let s = (plane > 0) as usize; + for i in (c >> s)..((c + n4) >> s) { + self.above_level[plane][i] = 0; + self.above_dc[plane][i] = 0; + } + for i in (r >> s)..((r + n4) >> s) { + self.left_level[plane][i] = 0; + self.left_dc[plane][i] = 0; + } + } + } else { + let log2 = (n4 * 4).trailing_zeros(); + assert!(log2 <= 5, "64x64 blocks are coded as skip blocks"); + for plane in 0..3 { + let s = (plane > 0) as u32; + self.coeffs(plane, (c * 4) >> s, (r * 4) >> s, tx_size_for(log2 - s), b, &b.coefs[plane]); + } + } + for y in 0..rows { + for x in 0..cols { + let i = self.mi.idx(r + y, c + x); + self.mi.is_inter[i] = b.is_inter; + self.mi.skip[i] = b.skip; + self.mi.mi_size[i] = bsize as u8; + } + } + // LoopfilterTxSizes: one transform block per plane + let log2 = (n4 * 4).trailing_zeros(); + let stride = self.g.mi_cols + 32; + for plane in 0..3 { + let s = (plane > 0) as usize; + let tx = tx_size_for(log2 - s as u32) as u8; + let (r0, c0, m) = (r >> s, c >> s, n4 >> s); + for y in r0..r0 + m { + self.lf_tx[plane][y * stride + c0..y * stride + c0 + m].iter_mut().for_each(|v| *v = tx); + } + } + } + + fn intra_angle_and_uv(&mut self, b: &Leaf) { + if self.bsize >= BLOCK_8X8 && is_directional(b.y_mode) { + self.sw.symbol(MAX_ANGLE_DELTA, &mut self.cdf.angle_delta[b.y_mode - V_PRED]); + } + let bs = block_size_px(self.bsize); + if bs <= 32 { + self.sw.symbol(b.uv_mode, &mut self.cdf.uv_mode_cfl_allowed[b.y_mode]); + } else { + self.sw.symbol(b.uv_mode, &mut self.cdf.uv_mode_cfl_not_allowed[b.y_mode]); + } + if self.bsize >= BLOCK_8X8 && is_directional(b.uv_mode) { + self.sw.symbol(MAX_ANGLE_DELTA, &mut self.cdf.angle_delta[b.uv_mode - V_PRED]); + } + } + + /// read_ref_frames (LAST_FRAME), find_mv_stack, the inter mode, DRL and motion vector. + /// Returns the inter mode (YMode). + fn inter_block_mode_info(&mut self, b: &Leaf, left: [i8; 2], above: [i8; 2]) -> usize { + let count = |ft: i8| -> usize { + let mut n = 0; + if self.avail_u { + n += (above[0] == ft) as usize + (above[1] == ft) as usize; + } + if self.avail_l { + n += (left[0] == ft) as usize + (left[1] == ft) as usize; + } + n + }; + let cmp = |a: usize, b: usize| match a.cmp(&b) { + std::cmp::Ordering::Less => 0, + std::cmp::Ordering::Equal => 1, + std::cmp::Ordering::Greater => 2, + }; + let (last, last2, last3, gold) = (count(LAST_FRAME as i8), count(LAST2_FRAME as i8), count(LAST3_FRAME as i8), count(GOLDEN_FRAME as i8)); + let (bwd, alt2, alt) = (count(BWDREF_FRAME as i8), count(ALTREF2_FRAME as i8), count(ALTREF_FRAME as i8)); + let ctx_p1 = cmp(last + last2 + last3 + gold, bwd + alt2 + alt); + let ctx_comp_ref = cmp(last + last2, last3 + gold); + let ctx_comp_ref_p1 = cmp(last, last2); + self.sw.symbol(0, &mut self.cdf.single_ref[ctx_p1][0]); + self.sw.symbol(0, &mut self.cdf.single_ref[ctx_comp_ref][2]); + self.sw.symbol(0, &mut self.cdf.single_ref[ctx_comp_ref_p1][3]); + self.find_mv_stack(); + let mv = b.mv; + let s = &self.mvs; + let mode = if mv == [0, 0] { + GLOBALMV + } else if mv == s.ref_stack_mv[0] { + NEARESTMV + } else if mv == s.ref_stack_mv[1] { + NEARMV + } else { + NEWMV + }; + let (nctx, zctx, rctx) = (s.new_mv_context, s.zero_mv_context, s.ref_mv_context); + self.sw.symbol((mode != NEWMV) as usize, &mut self.cdf.new_mv[nctx]); + if mode != NEWMV { + self.sw.symbol((mode != GLOBALMV) as usize, &mut self.cdf.zero_mv[zctx]); + if mode != GLOBALMV { + self.sw.symbol((mode == NEARMV) as usize, &mut self.cdf.ref_mv[rctx]); + } + } + // DRL: ref_mv_idx 0 for NEWMV, 1 for NEARMV + if mode == NEWMV { + if self.mvs.num_mv_found > 1 { + let ctx = self.mvs.drl_ctx_stack[0]; + self.sw.symbol(0, &mut self.cdf.drl_mode[ctx]); + } + } else if mode == NEARMV && self.mvs.num_mv_found > 2 { + let ctx = self.mvs.drl_ctx_stack[1]; + self.sw.symbol(0, &mut self.cdf.drl_mode[ctx]); + } + if mode == NEWMV { + let pred = self.mvs.ref_stack_mv[0]; + self.write_mv([mv[0] - pred[0], mv[1] - pred[1]]); + } + mode + } + + fn write_mv(&mut self, diff: MvP) { + let joint = ((diff[0] != 0) as usize) << 1 | (diff[1] != 0) as usize; + // MV_JOINT_ZERO, MV_JOINT_HNZVZ (col only), MV_JOINT_HZVNZ (row only), MV_JOINT_HNZVNZ + self.sw.symbol(joint, &mut self.cdf.mv_joint); + for (comp, &d) in diff.iter().enumerate() { + if d != 0 { + self.write_mv_component(comp, d); + } + } + } + + fn write_mv_component(&mut self, comp: usize, v: i32) { + debug_assert!(v % 2 == 0, "quarter-sample precision"); + self.sw.symbol((v < 0) as usize, &mut self.cdf.mv_sign[comp]); + let m = v.unsigned_abs() - 1; // odd: the implied hp bit is 1 + let fr = ((m >> 1) & 3) as usize; + let int = m >> 3; + if int < CLASS0_SIZE as u32 { + self.sw.symbol(MV_CLASS_0, &mut self.cdf.mv_class[comp]); + self.sw.symbol(int as usize, &mut self.cdf.mv_class0_bit[comp]); + self.sw.symbol(fr, &mut self.cdf.mv_class0_fr[comp][int as usize]); + } else { + let class = 31 - int.leading_zeros(); + let d = int - (1 << class); + self.sw.symbol(class as usize, &mut self.cdf.mv_class[comp]); + for i in 0..class as usize { + self.sw.symbol(((d >> i) & 1) as usize, &mut self.cdf.mv_bit[comp][i]); + } + self.sw.symbol(fr, &mut self.cdf.mv_fr[comp]); + } + } + + // ---- coefficients ---------------------------------------------------------------------- + + fn coeffs(&mut self, plane: usize, start_x: usize, start_y: usize, tx_sz: usize, b: &Leaf, levels: &[i32]) { + let x4 = start_x >> 2; + let y4 = start_y >> 2; + let w4 = TX_WIDTH[tx_sz] as usize >> 2; + let h4 = TX_HEIGHT[tx_sz] as usize >> 2; + let tx_sz_ctx = (TX_SIZE_SQR[tx_sz] as usize + TX_SIZE_SQR_UP[tx_sz] as usize + 1) >> 1; + let ptype = (plane > 0) as usize; + let scan = scan_for(tx_sz); + let mut eob = 0; + for (i, &pos) in scan.iter().enumerate() { + if levels[pos as usize] != 0 { + eob = i + 1; + } + } + let ctx = self.all_zero_ctx(plane, tx_sz, x4, y4, w4, h4); + self.sw.symbol((eob == 0) as usize, &mut self.cdf.txb_skip[tx_sz_ctx][ctx]); + let mut cul_level = 0u32; + let mut dc_category = 0u8; + if eob == 0 { + if plane == 0 { + self.set_tx_types(x4, y4, tx_sz, DCT_DCT); + } + } else { + if plane == 0 { + self.write_transform_type(x4, y4, tx_sz, b); + } + let plane_tx_type = self.compute_tx_type(plane, tx_sz, x4, y4, b); + debug_assert!(plane_tx_type == DCT_DCT || matches!(plane_tx_type, ADST_DCT | DCT_ADST | ADST_ADST)); + let eob_multisize = (TX_WIDTH_LOG2[tx_sz] as usize).min(5) + (TX_HEIGHT_LOG2[tx_sz] as usize).min(5) - 4; + let eob_pt = if eob < 2 { eob } else { (32 - ((eob - 1) as u32).leading_zeros()) as usize + 1 }; + let ectx = 0; // TX_CLASS_2D + let s = eob_pt - 1; + match eob_multisize { + 0 => self.sw.symbol(s, &mut self.cdf.eob_pt_16[ptype][ectx]), + 1 => self.sw.symbol(s, &mut self.cdf.eob_pt_32[ptype][ectx]), + 2 => self.sw.symbol(s, &mut self.cdf.eob_pt_64[ptype][ectx]), + 3 => self.sw.symbol(s, &mut self.cdf.eob_pt_128[ptype][ectx]), + 4 => self.sw.symbol(s, &mut self.cdf.eob_pt_256[ptype][ectx]), + 5 => self.sw.symbol(s, &mut self.cdf.eob_pt_512[ptype]), + _ => self.sw.symbol(s, &mut self.cdf.eob_pt_1024[ptype]), + } + if eob_pt >= 3 { + let eob_shift = eob_pt - 3; + let offset = eob - ((1 << (eob_pt - 2)) + 1); + self.sw.symbol((offset >> eob_shift) & 1, &mut self.cdf.eob_extra[tx_sz_ctx][ptype][eob_pt - 3]); + for i in 1..eob_pt - 2 { + let sh = eob_pt - 2 - 1 - i; + self.sw.bool((offset >> sh) & 1 == 1); + } + } + let bwl = TX_WIDTH_LOG2[tx_sz] as usize; + let height = TX_HEIGHT[tx_sz] as usize; + let width = 1usize << bwl; + // Quant[] as the decoder sees it while parsing: the levels coded so far (capped) + let mut quant = std::mem::take(&mut self.quant); + quant.clear(); + quant.resize(width * height, 0); + for c in (0..eob).rev() { + let pos = scan[c] as usize; + let a = levels[pos].unsigned_abs().min(15) as i32; + if c == eob - 1 { + let ctx = coeff_base_eob_ctx(c, bwl, height); + self.sw.symbol((a.min(3) - 1) as usize, &mut self.cdf.coeff_base_eob[tx_sz_ctx][ptype][ctx]); + } else { + let ctx = coeff_base_ctx(&quant, tx_sz, bwl, width, height, pos); + self.sw.symbol(a.min(3) as usize, &mut self.cdf.coeff_base[tx_sz_ctx][ptype][ctx]); + } + if a > NUM_BASE_LEVELS as i32 { + let ctx = coeff_br_ctx(&quant, bwl, height, pos); + let mut rem = a - 3; + for _ in 0..(COEFF_BASE_RANGE / (BR_CDF_SIZE - 1)) { + let br = rem.min(3); + self.sw.symbol(br as usize, &mut self.cdf.coeff_br[tx_sz_ctx.min(TX_32X32)][ptype][ctx]); + rem -= br; + if br < 3 { + break; + } + } + } + quant[pos] = a; + } + self.quant = quant; + for c in 0..eob { + let pos = scan[c] as usize; + let v = levels[pos]; + if v != 0 { + if c == 0 { + let ctx = self.dc_sign_ctx(plane, x4, y4, w4, h4); + self.sw.symbol((v < 0) as usize, &mut self.cdf.dc_sign[ptype][ctx]); + } else { + self.sw.bool(v < 0); + } + } + let a = v.unsigned_abs(); + if a > (NUM_BASE_LEVELS + COEFF_BASE_RANGE) as u32 { + // Golomb code of a - 15 + let x = a - 15 + 1; + let length = 32 - x.leading_zeros(); + for _ in 0..length - 1 { + self.sw.bool(false); + } + self.sw.bool(true); + for i in (0..length - 1).rev() { + self.sw.bool((x >> i) & 1 == 1); + } + } + if pos == 0 && a > 0 { + dc_category = if v < 0 { 1 } else { 2 }; + } + cul_level += a & 0xFFFFF; + } + cul_level = cul_level.min(63); + } + for i in 0..w4 { + if x4 + i < self.above_level[plane].len() { + self.above_level[plane][x4 + i] = cul_level as u8; + self.above_dc[plane][x4 + i] = dc_category; + } + } + for i in 0..h4 { + if y4 + i < self.left_level[plane].len() { + self.left_level[plane][y4 + i] = cul_level as u8; + self.left_dc[plane][y4 + i] = dc_category; + } + } + } + + fn write_transform_type(&mut self, x4: usize, y4: usize, tx_sz: usize, b: &Leaf) { + // transform_type( ) + let set = tx_set(tx_sz, b.is_inter); + let t = b.tx_type_y; + if set > 0 && self.qidx > 0 { + let sqr = TX_SIZE_SQR[tx_sz] as usize; + if b.is_inter { + // The decision only picks types of this set (DCT_DCT otherwise). + let s = TX_TYPE_INTER_INV_SET3.iter().position(|&v| v as usize == t).unwrap_or_default(); + self.sw.symbol(s, &mut self.cdf.inter_tx_type_set3[sqr]); + } else { + let s = TX_TYPE_INTRA_INV_SET2.iter().position(|&v| v as usize == t).unwrap_or_default(); + self.sw.symbol(s, &mut self.cdf.intra_tx_type_set2[sqr][b.y_mode]); + } + } else { + assert_eq!(t, DCT_DCT); + } + self.set_tx_types(x4, y4, tx_sz, t); + } + + fn set_tx_types(&mut self, x4: usize, y4: usize, tx_sz: usize, t: usize) { + for i in 0..(TX_WIDTH[tx_sz] as usize >> 2) { + for j in 0..(TX_HEIGHT[tx_sz] as usize >> 2) { + if y4 + j < self.g.mi_rows && x4 + i < self.g.mi_cols { + let k = self.mi.idx(y4 + j, x4 + i); + self.mi.tx_type[k] = t as u8; + } + } + } + } + + fn compute_tx_type(&self, plane: usize, tx_sz: usize, x4: usize, y4: usize, b: &Leaf) -> usize { + if TX_SIZE_SQR_UP[tx_sz] as usize > TX_32X32 { + return DCT_DCT; + } + if plane == 0 { + return self.mi.tx_type[self.mi.idx(y4, x4)] as usize; + } + if b.is_inter { + let set = tx_set(tx_sz, true); + let xx = self.c.max(x4 << 1); + let yy = self.r.max(y4 << 1); + let t = self.mi.tx_type[self.mi.idx(yy, xx)] as usize; + return if tx_in_set(set, true, t) { t } else { DCT_DCT }; + } + intra_uv_tx_type(b.uv_mode, tx_sz) + } + + fn all_zero_ctx(&self, plane: usize, tx_sz: usize, x4: usize, y4: usize, w4: usize, h4: usize) -> usize { + let s = (plane > 0) as usize; + let max_x4 = self.g.mi_cols >> s; + let max_y4 = self.g.mi_rows >> s; + let w = TX_WIDTH[tx_sz] as usize; + let h = TX_HEIGHT[tx_sz] as usize; + let bsize = SUBSAMPLED_SIZE[self.bsize][s][s] as usize; + let bw = block_size_px(bsize); + let bh = 4 * NUM_4X4_BLOCKS_HIGH[bsize] as usize; + let al = &self.above_level[plane]; + let ll = &self.left_level[plane]; + if plane == 0 { + let mut top = 0u32; + let mut left = 0u32; + for k in 0..w4 { + if x4 + k < max_x4 { + top = top.max(al[x4 + k] as u32); + } + } + for k in 0..h4 { + if y4 + k < max_y4 { + left = left.max(ll[y4 + k] as u32); + } + } + top = top.min(255); + left = left.min(255); + if bw == w && bh == h { + 0 + } else if top == 0 && left == 0 { + 1 + } else if top == 0 || left == 0 { + 2 + (top.max(left) > 3) as usize + } else if top.max(left) <= 3 { + 4 + } else if top.min(left) <= 3 { + 5 + } else { + 6 + } + } else { + let ad = &self.above_dc[plane]; + let ld = &self.left_dc[plane]; + let mut above = 0u8; + let mut left = 0u8; + for i in 0..w4 { + if x4 + i < max_x4 { + above |= al[x4 + i] | ad[x4 + i]; + } + } + for i in 0..h4 { + if y4 + i < max_y4 { + left |= ll[y4 + i] | ld[y4 + i]; + } + } + let mut ctx = (above != 0) as usize + (left != 0) as usize + 7; + if bw * bh > w * h { + ctx += 3; + } + ctx + } + } + + fn dc_sign_ctx(&self, plane: usize, x4: usize, y4: usize, w4: usize, h4: usize) -> usize { + let s = (plane > 0) as usize; + let max_x4 = self.g.mi_cols >> s; + let max_y4 = self.g.mi_rows >> s; + let mut dc_sign = 0i32; + for k in 0..w4 { + if x4 + k < max_x4 { + match self.above_dc[plane][x4 + k] { + 1 => dc_sign -= 1, + 2 => dc_sign += 1, + _ => {} + } + } + } + for k in 0..h4 { + if y4 + k < max_y4 { + match self.left_dc[plane][y4 + k] { + 1 => dc_sign -= 1, + 2 => dc_sign += 1, + _ => {} + } + } + } + match dc_sign.signum() { + -1 => 1, + 1 => 2, + _ => 0, + } + } + + // ---- motion vector prediction (7.10.2), single reference LAST_FRAME -------------------- + + fn inside(&self, r: isize, c: isize) -> bool { + c >= 0 && c < self.g.mi_cols as isize && r >= 0 && r < self.g.mi_rows as isize + } + + fn find_mv_stack(&mut self) { + let n4 = NUM_4X4_BLOCKS_WIDE[self.bsize] as isize; + let (bw4, bh4) = (n4, n4); + self.mvs = MvStack::default(); + self.scan_row(-1); + let mut found_above = self.mvs.found_match; + self.mvs.found_match = false; + self.scan_col(-1); + let mut found_left = self.mvs.found_match; + self.mvs.found_match = false; + if bw4.max(bh4) <= 16 { + self.scan_point(-1, bw4); + } + if self.mvs.found_match { + found_above = true; + } + self.mvs.close_matches = found_above as usize + found_left as usize; + let num_nearest = self.mvs.num_mv_found; + let num_new = self.mvs.new_mv_count; + for idx in 0..num_nearest { + self.mvs.weight_stack[idx] += REF_CAT_LEVEL as u32; + } + self.mvs.zero_mv_context = 0; + // use_ref_frame_mvs = 0: no temporal candidates + self.scan_point(-1, -1); + if self.mvs.found_match { + found_above = true; + } + self.mvs.found_match = false; + self.scan_row(-3); + if self.mvs.found_match { + found_above = true; + } + self.mvs.found_match = false; + self.scan_col(-3); + if self.mvs.found_match { + found_left = true; + } + self.mvs.found_match = false; + if bh4 > 1 { + self.scan_row(-5); + } + if self.mvs.found_match { + found_above = true; + } + self.mvs.found_match = false; + if bw4 > 1 { + self.scan_col(-5); + } + if self.mvs.found_match { + found_left = true; + } + self.mvs.total_matches = found_above as usize + found_left as usize; + let n = self.mvs.num_mv_found; + self.sort_stack(0, num_nearest); + self.sort_stack(num_nearest, n); + if self.mvs.num_mv_found < 2 { + self.extra_search(); + } + self.context_and_clamping(num_new); + } + + fn scan_row(&mut self, delta_row: isize) { + let bw4 = NUM_4X4_BLOCKS_WIDE[self.bsize] as isize; + let (mi_row, mi_col) = (self.r as isize, self.c as isize); + let end4 = bw4.min(self.g.mi_cols as isize - mi_col).min(16); + let mut delta_col = 0; + let use_step16 = bw4 >= 16; + let mut delta_row = delta_row; + if delta_row.abs() > 1 { + delta_row += mi_row & 1; + delta_col = 1 - (mi_col & 1); + } + let mut i = 0; + while i < end4 { + let mv_row = mi_row + delta_row; + let mv_col = mi_col + delta_col + i; + if !self.inside(mv_row, mv_col) { + break; + } + let k = self.mi.idx(mv_row as usize, mv_col as usize); + let mut len = bw4.min(NUM_4X4_BLOCKS_WIDE[self.mi.mi_size[k] as usize] as isize); + if delta_row.abs() > 1 { + len = len.max(2); + } + if use_step16 { + len = len.max(4); + } + self.add_ref_mv_candidate(mv_row as usize, mv_col as usize, len as u32 * 2); + i += len; + } + } + + fn scan_col(&mut self, delta_col: isize) { + let bh4 = NUM_4X4_BLOCKS_HIGH[self.bsize] as isize; + let (mi_row, mi_col) = (self.r as isize, self.c as isize); + let end4 = bh4.min(self.g.mi_rows as isize - mi_row).min(16); + let mut delta_row = 0; + let use_step16 = bh4 >= 16; + let mut delta_col = delta_col; + if delta_col.abs() > 1 { + delta_row = 1 - (mi_row & 1); + delta_col += mi_col & 1; + } + let mut i = 0; + while i < end4 { + let mv_row = mi_row + delta_row + i; + let mv_col = mi_col + delta_col; + if !self.inside(mv_row, mv_col) { + break; + } + let k = self.mi.idx(mv_row as usize, mv_col as usize); + let mut len = bh4.min(NUM_4X4_BLOCKS_HIGH[self.mi.mi_size[k] as usize] as isize); + if delta_col.abs() > 1 { + len = len.max(2); + } + if use_step16 { + len = len.max(4); + } + self.add_ref_mv_candidate(mv_row as usize, mv_col as usize, len as u32 * 2); + i += len; + } + } + + fn scan_point(&mut self, delta_row: isize, delta_col: isize) { + let mv_row = self.r as isize + delta_row; + let mv_col = self.c as isize + delta_col; + if self.inside(mv_row, mv_col) && self.mi.written[self.mi.idx(mv_row as usize, mv_col as usize)] { + self.add_ref_mv_candidate(mv_row as usize, mv_col as usize, 4); + } + } + + fn add_ref_mv_candidate(&mut self, mv_row: usize, mv_col: usize, weight: u32) { + let i = self.mi.idx(mv_row, mv_col); + if !self.mi.is_inter[i] { + return; + } + let rf = self.mi.ref_frame[i]; + for cand_list in 0..2 { + if rf[cand_list] == LAST_FRAME as i8 { + self.search_stack(i, cand_list, weight); + } + } + } + + fn search_stack(&mut self, i: usize, cand_list: usize, weight: u32) { + debug_assert_eq!(cand_list, 0); + let cand_mode = self.mi.y_mode[i] as usize; + // gm_type is IDENTITY: the stored motion vector is used + let mut cand = self.mi.mv[i]; + lower_mv_precision(&mut cand); + if cand_mode == NEWMV { + self.mvs.new_mv_count += 1; + } + self.mvs.found_match = true; + let n = self.mvs.num_mv_found; + if let Some(idx) = (0..n).find(|&k| self.mvs.ref_stack_mv[k] == cand) { + self.mvs.weight_stack[idx] += weight; + } else if n < MAX_REF_MV_STACK_SIZE { + self.mvs.ref_stack_mv[n] = cand; + self.mvs.weight_stack[n] = weight; + self.mvs.num_mv_found += 1; + } + } + + fn sort_stack(&mut self, start: usize, end: usize) { + let mut end = end; + let s = &mut self.mvs; + while end > start { + let mut new_end = start; + for idx in start + 1..end { + if s.weight_stack[idx - 1] < s.weight_stack[idx] { + s.weight_stack.swap(idx - 1, idx); + s.ref_stack_mv.swap(idx - 1, idx); + new_end = idx; + } + } + end = new_end; + } + } + + fn extra_search(&mut self) { + let (mi_row, mi_col) = (self.r as isize, self.c as isize); + let n4 = NUM_4X4_BLOCKS_WIDE[self.bsize] as isize; + let w4 = n4.min(16).min(self.g.mi_cols as isize - mi_col); + let h4 = n4.min(16).min(self.g.mi_rows as isize - mi_row); + let num4x4 = w4.min(h4); + for pass in 0..2 { + let mut idx = 0; + while idx < num4x4 && self.mvs.num_mv_found < 2 { + let (mv_row, mv_col) = if pass == 0 { (mi_row - 1, mi_col + idx) } else { (mi_row + idx, mi_col - 1) }; + if !self.inside(mv_row, mv_col) { + break; + } + let k = self.mi.idx(mv_row as usize, mv_col as usize); + // add_extra_mv_candidate( ) (no sign bias: a single forward reference) + let rfs = self.mi.ref_frame[k]; + for &cand_ref in rfs.iter().take(2) { + if cand_ref > INTRA_FRAME as i8 { + let cand = self.mi.mv[k]; + let n = self.mvs.num_mv_found; + if !(0..n).any(|j| self.mvs.ref_stack_mv[j] == cand) { + self.mvs.ref_stack_mv[n] = cand; + self.mvs.weight_stack[n] = 2; + self.mvs.num_mv_found += 1; + } + } + } + let sz = self.mi.mi_size[k] as usize; + idx += if pass == 0 { NUM_4X4_BLOCKS_WIDE[sz] } else { NUM_4X4_BLOCKS_HIGH[sz] } as isize; + } + } + for idx in self.mvs.num_mv_found..2 { + self.mvs.ref_stack_mv[idx] = [0, 0]; // GlobalMvs[ 0 ] + } + } + + fn context_and_clamping(&mut self, num_new: usize) { + let bs = block_size_px(self.bsize) as i32; + let n = self.mvs.num_mv_found; + for idx in 0..n { + let mut z = 0; + if idx + 1 < n { + let w0 = self.mvs.weight_stack[idx]; + let w1 = self.mvs.weight_stack[idx + 1]; + if w0 >= REF_CAT_LEVEL as u32 { + if w1 < REF_CAT_LEVEL as u32 { + z = 1; + } + } else { + z = 2; + } + } + self.mvs.drl_ctx_stack[idx] = z; + } + let n4 = NUM_4X4_BLOCKS_WIDE[self.bsize] as i32; + for idx in 0..n { + let mut mv = self.mvs.ref_stack_mv[idx]; + let border_r = MV_BORDER as i32 + bs * 8; + let to_top = -((self.r as i32 * 4) * 8); + let to_bottom = ((self.g.mi_rows as i32 - n4 - self.r as i32) * 4) * 8; + mv[0] = mv[0].clamp(to_top - border_r, to_bottom + border_r); + let to_left = -((self.c as i32 * 4) * 8); + let to_right = ((self.g.mi_cols as i32 - n4 - self.c as i32) * 4) * 8; + mv[1] = mv[1].clamp(to_left - border_r, to_right + border_r); + self.mvs.ref_stack_mv[idx] = mv; + } + let (close, total) = (self.mvs.close_matches, self.mvs.total_matches); + if close == 0 { + self.mvs.new_mv_context = total.min(1); + self.mvs.ref_mv_context = total; + } else if close == 1 { + self.mvs.new_mv_context = 3 - num_new.min(1); + self.mvs.ref_mv_context = 2 + total; + } else { + self.mvs.new_mv_context = 5 - num_new.min(1); + self.mvs.ref_mv_context = 5; + } + } +} + +/// lower_mv_precision( ) with allow_high_precision_mv = 0 and force_integer_mv = 0. +fn lower_mv_precision(mv: &mut MvP) { + for c in mv.iter_mut() { + if *c & 1 != 0 { + *c += if *c > 0 { -1 } else { 1 }; + } + } +} + +fn coeff_base_eob_ctx(c: usize, bwl: usize, height: usize) -> usize { + if c == 0 { + 0 + } else if c <= (height << bwl) / 8 { + 1 + } else if c <= (height << bwl) / 4 { + 2 + } else { + 3 + } +} + +/// Context of coeff_base for TX_CLASS_2D (8.3.2). +fn coeff_base_ctx(quant: &[i32], tx_sz: usize, bwl: usize, width: usize, height: usize, pos: usize) -> usize { + let row = pos >> bwl; + let col = pos - (row << bwl); + let mut mag = 0i32; + for off in SIG_REF_DIFF_OFFSET[TX_CLASS_2D].iter() { + let ref_row = row + off[0] as usize; + let ref_col = col + off[1] as usize; + if ref_row < height && ref_col < width { + mag += quant[(ref_row << bwl) + ref_col].abs().min(3); + } + } + let ctx = ((mag + 1) >> 1).min(4) as usize; + if row == 0 && col == 0 { + return 0; + } + ctx + COEFF_BASE_CTX_OFFSET[tx_sz][row.min(4)][col.min(4)] as usize +} + +/// Context of coeff_br for TX_CLASS_2D. +fn coeff_br_ctx(quant: &[i32], bwl: usize, txh: usize, pos: usize) -> usize { + let txw = 1usize << bwl; + let row = pos >> bwl; + let col = pos - (row << bwl); + let mut mag = 0i32; + for off in MAG_REF_OFFSET_WITH_TX_CLASS[TX_CLASS_2D].iter() { + let ref_row = row + off[0] as usize; + let ref_col = col + off[1] as usize; + if ref_row < txh && ref_col < txw { + mag += quant[ref_row * txw + ref_col].min((COEFF_BASE_RANGE + NUM_BASE_LEVELS + 1) as i32); + } + } + let mag = ((mag + 1) >> 1).min(6) as usize; + if pos == 0 { + mag + } else if row < 2 && col < 2 { + mag + 7 + } else { + mag + 14 + } +} diff --git a/crates/bitstream/Cargo.toml b/crates/bitstream/Cargo.toml new file mode 100644 index 0000000000..2b29efc829 --- /dev/null +++ b/crates/bitstream/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "craft-bitstream" +description = "Bit-level I/O and Exp-Golomb bitstream primitives for the Craft ecosystem" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-media = { workspace = true, features = ["bitstream"] } diff --git a/crates/bitstream/src/lib.rs b/crates/bitstream/src/lib.rs new file mode 100644 index 0000000000..728440b6de --- /dev/null +++ b/crates/bitstream/src/lib.rs @@ -0,0 +1,5 @@ +//! Bit-level reading and writing primitives for the Craft ecosystem. + +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +pub use craft_media::bitstream::*; diff --git a/crates/codecs/Cargo.toml b/crates/codecs/Cargo.toml new file mode 100644 index 0000000000..4bb75af496 --- /dev/null +++ b/crates/codecs/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "craft-codecs" +description = "Umbrella crate re-exporting pure-Rust media codecs (H.264, ProRes, HEVC, AV1, VP9, AAC, Audio-IO)" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[features] +default = ["all"] +all = ["h264", "prores", "hevc", "av1", "vp9", "aac", "audio-io"] +h264 = ["dep:craft-h264enc"] +prores = ["dep:craft-prores"] +hevc = ["dep:craft-hevcenc"] +av1 = ["dep:craft-av1enc"] +vp9 = ["dep:craft-vp9enc"] +aac = ["dep:craft-aac"] +audio-io = ["dep:craft-audio-io"] + +[dependencies] +craft-h264enc = { workspace = true, optional = true } +craft-prores = { workspace = true, optional = true } +craft-hevcenc = { workspace = true, optional = true } +craft-av1enc = { workspace = true, optional = true } +craft-vp9enc = { workspace = true, optional = true } +craft-aac = { workspace = true, optional = true } +craft-audio-io = { workspace = true, optional = true } diff --git a/crates/codecs/src/lib.rs b/crates/codecs/src/lib.rs new file mode 100644 index 0000000000..a8377a4b8e --- /dev/null +++ b/crates/codecs/src/lib.rs @@ -0,0 +1,33 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +//! Storytold ArtCraft shared media codecs library. +//! +//! Centralizes high-performance pure-Rust media codecs across video and audio: +//! - H.264 / AVC video encoding (`craft-h264enc`) +//! - Apple ProRes video encoding and decoding (`craft-prores`) +//! - HEVC / H.265 video encoding (`craft-hevcenc`) +//! - AV1 video encoding (`craft-av1enc`) +//! - VP9 video encoding (`craft-vp9enc`) +//! - AAC-LC audio encoding and decoding (`craft-aac`) +//! - Universal audio decoding & FLAC/AIFF encoding (`craft-audio-io`) + +#[cfg(feature = "h264")] +pub use craft_h264enc as h264; + +#[cfg(feature = "prores")] +pub use craft_prores as prores; + +#[cfg(feature = "hevc")] +pub use craft_hevcenc as hevc; + +#[cfg(feature = "av1")] +pub use craft_av1enc as av1; + +#[cfg(feature = "vp9")] +pub use craft_vp9enc as vp9; + +#[cfg(feature = "aac")] +pub use craft_aac as aac; + +#[cfg(feature = "audio-io")] +pub use craft_audio_io as audio_io; diff --git a/crates/h264enc/Cargo.toml b/crates/h264enc/Cargo.toml new file mode 100644 index 0000000000..ce5184c1f0 --- /dev/null +++ b/crates/h264enc/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "craft-h264enc" +description = "Clean-room pure-Rust H.264/AVC encoder for the Craft ecosystem" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[features] +default = ["threads"] +threads = ["dep:rayon"] + +[dependencies] +filmcraft-bitstream = { package = "craft-bitstream", path = "../bitstream" } +thiserror = { workspace = true } +rayon = { workspace = true, optional = true } diff --git a/crates/h264enc/README.md b/crates/h264enc/README.md new file mode 100644 index 0000000000..27496214d3 --- /dev/null +++ b/crates/h264enc/README.md @@ -0,0 +1,89 @@ +# filmcraft-h264enc + +A clean-room, pure-Rust H.264/AVC encoder for FilmCraft's export path, written from the public +ITU-T H.264 specification (04/2013 edition) and general encoder literature. Layer L0: it depends only on +`filmcraft-bitstream`, `thiserror` and (optionally, feature `threads`, on by default) `rayon`. No `unsafe`, +builds for `wasm32-unknown-unknown` with or without `threads`. + +## Usage + +```rust +use filmcraft_h264enc::{Encoder, EncoderConfig, PacketFormat, Preset, Profile, RateControl, YuvFrame}; + +let mut cfg = EncoderConfig::new(1920, 1080, 30000, 1001); +cfg.profile = Profile::High; +cfg.preset = Preset::Balanced; // Speed / Balanced / Quality +cfg.rate = RateControl::Crf(20.0); // Crf(f32) | Cbr { kbps } | Vbr { target_kbps, max_kbps } | Qp(u8) +cfg.bframes = 2; // 0..=3 +cfg.keyint = 250; +cfg.format = PacketFormat::LengthPrefixed; // MP4 samples; or AnnexB (default) for .h264 +let mut enc = Encoder::new(cfg)?; +let avcc = enc.avcc(); // AVCDecoderConfigurationRecord for the MP4 `avcC` box +for (i, pic) in pictures.iter().enumerate() { + let frame = YuvFrame { y: &pic.y, u: &pic.u, v: &pic.v, y_stride: w, uv_stride: w / 2 }; + for p in enc.encode(&frame, i as i64) { /* p.data, p.pts, p.dts, p.keyframe, p.frame_type */ } +} +for p in enc.flush() { /* ... */ } +``` + +- Input: planar 4:2:0 8-bit (`YuvFrame` with strides); any even size, non-multiple-of-16 sizes are + cropped via the SPS frame cropping. `try_encode` returns an error instead of panicking on short planes. +- Output: one `Packet` per picture in decoding order with `pts` (as given), `dts` (reorder-delayed, always + `<= pts` and strictly increasing), `keyframe` (IDR) and the frame type/QP. Annex-B packets carry an AUD, + SPS/PPS on every IDR and an identification SEI on the first picture. +- Two-pass: encode once with `cfg.pass = Pass::First`, take `enc.pass_stats()` (serialisable with + `PassStats::to_text/from_text`), then encode again with `Pass::Second(stats)`. +- `set_recon_capture(true)` + `take_recon()` expose the encoder's reconstructed pictures (used by the tests to + prove encoder/decoder match). + +## Features + +| Area | Implemented | +|---|---| +| Profiles | Constrained Baseline (CAVLC, I/P), Main (CABAC, I/P/B), High (CABAC, 8x8 transform, I/P/B); 4:2:0 8-bit, frame coding | +| Parameter sets | SPS (level auto-selected from size/rate/DPB/bitrate, cropping, VUI: SAR, colour description + range, timing, bitstream restriction/reorder), PPS, AUD, user-data SEI, avcC | +| Intra | 16x16 (4 modes, SATD), 4x4 (9 modes, SATD + mode-cost, sequential reconstruction), 8x8 (9 modes with reference filtering, High), chroma (4 modes) | +| Inter P | hexagon integer search from predictors, half- then quarter-sample refinement (SATD) on precomputed 6-tap half-sample planes; 16x16/16x8/8x16/8x8 partitions (Balanced/Quality); P_Skip detection with the normative skip MV | +| Inter B | 1–3 non-reference B-frames, spatial direct / B_Skip, L0/L1/Bi 16x16 and (Balanced/Quality) 16x8/8x16 with a per-partition direction; all B-frames of a mini-GOP are encoded concurrently | +| Residual | 4x4 / 8x8 integer transforms, Intra16x16 and chroma DC Hadamards, deadzone quantisation (intra 1/3, inter 1/6), adaptive 4x4/8x8 inter transform choice, coefficient decimation for inter blocks | +| Entropy | CABAC (all I and cabac_init_idc 0 context tables, verified against the spec text) and CAVLC (all coeff_token/total_zeros/run_before tables, level escapes) | +| Loop filter | normative deblocking (bS 0–4 incl. bi-predicted rules, 8x8-transform edges, chroma QP mapping) in the reconstruction loop | +| Mode decision | SATD + λ·bits for Speed/Balanced; Quality adds RD-lite (SSD + λ·CABAC-estimated bits) between the best intra/inter/skip candidates | +| Rate control | `Qp`, `Crf` (constant quality with I/P/B offsets), 1-pass ABR with complexity model + VBV (`Cbr`, 1-pass `Vbr`), closed-loop 2-pass VBR; variance adaptive quantisation (`aq_strength`), lookahead scene-cut IDR insertion | +| Threading | slice-parallel encoding (MB-row slices, `disable_deblocking_filter_idc = 2` with >1 slice so each slice is also deblocked in parallel), parallel half-sample interpolation and lookahead | + +## Verification + +`cargo test --release -p filmcraft-h264enc` (ffmpeg/ffprobe used only as external oracles; tests skip when +absent). Every stream is decoded with `ffmpeg -v warning -err_detect +crccheck+bitstream+buffer+explode -f rawvideo` +and must print nothing (not even a `corrupt decoded frame` warning), produce exactly the number of frames encoded, +and be **bit-identical to the encoder's reconstruction** — this covers Baseline/Main/High, all presets, 0–3 +B-frames, 1–4 slices and automatic slicing at 1080p, odd sizes, `testsrc2` and `mandelbrot` sources, and +length-prefixed output re-wrapped with the avcC parameter sets. B-frame display order is checked with ffprobe. A +negative control drops and truncates slices and checks that the oracle reports them. + +The oracle leaves ffmpeg's error concealment on. With `-ec 0`, ffmpeg 7.1 and 8.1 print `corrupt decoded frame` +for every picture that has more than one slice, libx264's multi-slice streams included, while the decoded pixels +are unchanged (#74). The bit-exact comparison is what rules out concealment. + +Results on the synthetic content (moving gradient, grain noise, text-like edges, sub-pixel panning): + +| Test | Result | +|---|---| +| 1080p, High, Speed, CRF 20, 2 B | mean luma PSNR 43.0 dB (IDR 39.1, P/B 43.2–44.5) | +| 640x360 CBR 1500 kbit/s | 1518 kbit/s (+1.2%) | +| 640x360 1-pass VBR 1000 kbit/s | 981 kbit/s (−1.9%) | +| 640x360 2-pass VBR 800 kbit/s | 840 kbit/s (+5.0%) | + +Development tool: `cargo run --release -p filmcraft-h264enc --example h264enc_synth -- W H FRAMES +[profile] [preset] [bframes] [qpN|crfN|cbrN|vbrN] [slices] [synth|testsrc2|mandelbrot]` encodes, decodes with +ffmpeg and reports speed, bitrate, PSNR and any encoder/decoder mismatch (`MAP=1` prints a per-MB mismatch map, +`PERF=k` cycles k source pictures for speed runs). + +## Limitations / not yet implemented + +- Single reference frame per list; no B-pyramid, no weighted prediction, no interlaced/MBAFF, no 4:2:2/4:4:4 or + high bit depth, no custom scaling matrices, no sub-8x8 partitions, no B_8x8. +- CAVLC is used only for Baseline (High/Main always use CABAC); 8x8 transform is therefore CABAC-only. +- No trellis quantisation, no psy-RD; no HRD/buffering-period SEI (VBV is enforced internally only). +- Threading is slice-based only (no frame-level pipelining); with many slices, slice edges are not deblocked. diff --git a/crates/h264enc/src/cabac.rs b/crates/h264enc/src/cabac.rs new file mode 100644 index 0000000000..11a3fd76ec --- /dev/null +++ b/crates/h264enc/src/cabac.rs @@ -0,0 +1,344 @@ +//! CABAC arithmetic encoder (ITU-T H.264 §9.3.4.2) and a matching bit-cost estimator. + +use crate::tables::{NUM_CTX, RANGE_LPS, TRANS_LPS}; + +/// A sink for CABAC bins. Implemented by the real arithmetic coder and by the rate estimator, so the same +/// syntax-element code drives both bitstream writing and RD cost estimation. +pub trait BinSink { + fn decision(&mut self, ctx: usize, bin: u32); + fn bypass(&mut self, bin: u32); + fn terminate(&mut self, bin: u32); +} + +/// Context states: `pStateIdx << 1 | valMPS`. +#[derive(Clone)] +pub struct Contexts(pub [u8; NUM_CTX]); + +impl Contexts { + pub fn init(table: &[(i8, i8); NUM_CTX], slice_qp: i32) -> Self { + let qp = slice_qp.clamp(0, 51); + let mut s = [0u8; NUM_CTX]; + for (i, &(m, n)) in table.iter().enumerate() { + let pre = (((m as i32) * qp) >> 4) + n as i32; + let pre = pre.clamp(1, 126); + s[i] = if pre <= 63 { ((63 - pre) as u8) << 1 } else { (((pre - 64) as u8) << 1) | 1 }; + } + Contexts(s) + } +} + +/// The arithmetic encoder writing into a byte buffer. +/// +/// `low` holds the 10-bit coding window plus `queue + 8` pending bits above it; whole bytes are emitted +/// as soon as they are complete. A pending byte of 0xff may still receive a carry, so runs of them are +/// counted in `outstanding` until a byte that is not 0xff resolves the carry. +#[derive(Clone)] +pub struct CabacEncoder { + pub ctx: Contexts, + low: u32, + range: u32, + queue: i32, + outstanding: usize, + buf: Vec, +} + +impl CabacEncoder { + /// Start encoding after a slice header; `header` must be byte aligned (cabac_alignment_one_bits written). + pub fn new(ctx: Contexts, header: Vec) -> Self { + // queue = -9: the first bit out of the window is the spec's discarded first bit, which is always 0. + CabacEncoder { ctx, low: 0, range: 510, queue: -9, outstanding: 0, buf: header } + } + + #[inline] + fn put_byte(&mut self) { + if self.queue >= 0 { + let out = self.low >> (self.queue + 10); + self.low &= (0x400 << self.queue) - 1; + self.queue -= 8; + if out & 0xff == 0xff { + self.outstanding += 1; + } else { + let carry = out >> 8; + if let Some(last) = self.buf.last_mut() { + *last = last.wrapping_add(carry as u8); + } + let fill = (carry as u8).wrapping_sub(1); + self.buf.extend(std::iter::repeat_n(fill, self.outstanding)); + self.buf.push(out as u8); + self.outstanding = 0; + } + } + } + + #[inline] + fn renorm(&mut self) { + let shift = self.range.leading_zeros() - 23; + self.range <<= shift; + self.low <<= shift; + self.queue += shift as i32; + self.put_byte(); + } + + /// Terminate with end_of_slice_flag = 1 and flush; returns the RBSP bytes (with rbsp trailing bits). + pub fn finish(mut self) -> Vec { + self.terminate(1); + // EncodeFlush: the window's top two bits, then the stop bit, then zeros to byte alignment. + let bits = self.queue + 11; + let pad = (8 - bits % 8) % 8; + self.low = ((self.low >> 7) | 1) << (pad + 10); + self.queue = bits + pad - 8; + while self.queue >= 0 { + self.put_byte(); + } + self.buf.extend(std::iter::repeat_n(0xff, self.outstanding)); + self.buf + } +} + +impl BinSink for CabacEncoder { + #[inline] + fn decision(&mut self, ctx: usize, bin: u32) { + let s = self.ctx.0[ctx]; + let state = (s >> 1) as usize; + let mps = (s & 1) as u32; + let lps = RANGE_LPS[state][((self.range >> 6) & 3) as usize] as u32; + self.range -= lps; + if bin != mps { + self.low += self.range; + self.range = lps; + let new_mps = if state == 0 { 1 - mps } else { mps }; + self.ctx.0[ctx] = (TRANS_LPS[state] << 1) | new_mps as u8; + } else if state < 62 { + self.ctx.0[ctx] = (((state + 1) as u8) << 1) | mps as u8; + } + self.renorm(); + } + + #[inline] + fn bypass(&mut self, bin: u32) { + self.low <<= 1; + if bin != 0 { + self.low += self.range; + } + self.queue += 1; + self.put_byte(); + } + + fn terminate(&mut self, bin: u32) { + self.range -= 2; + if bin != 0 { + self.low += self.range; + self.range = 2; + } + self.renorm(); + } +} + +/// Fractional-bit cost table: `COST[state][is_mps]` in 1/256 bit units. +pub struct CostTable { + pub cost: [[u32; 2]; 64], +} + +impl CostTable { + pub fn new() -> Self { + let mut cost = [[0u32; 2]; 64]; + // Probability of the LPS for each state, estimated from rangeTabLPS relative to a mid-range interval. + for (s, c) in cost.iter_mut().enumerate() { + let avg: f64 = RANGE_LPS[s].iter().map(|&v| v as f64).sum::() / 4.0; + let p_lps = (avg / 384.0).clamp(1e-4, 0.5); + c[0] = (-(p_lps.log2()) * 256.0).round() as u32; + c[1] = (-((1.0 - p_lps).log2()) * 256.0).round() as u32; + } + CostTable { cost } + } +} + +impl Default for CostTable { + fn default() -> Self { + Self::new() + } +} + +/// Rate estimator: tracks context states like the encoder and accumulates cost in 1/256 bits. +#[derive(Clone)] +pub struct CabacEstimator<'a> { + pub ctx: Contexts, + pub cost: u64, + table: &'a CostTable, +} + +impl<'a> CabacEstimator<'a> { + pub fn new(ctx: Contexts, table: &'a CostTable) -> Self { + CabacEstimator { ctx, cost: 0, table } + } +} + +impl BinSink for CabacEstimator<'_> { + #[inline] + fn decision(&mut self, ctx: usize, bin: u32) { + let s = self.ctx.0[ctx]; + let state = (s >> 1) as usize; + let mps = (s & 1) as u32; + if bin != mps { + self.cost += self.table.cost[state][0] as u64; + let new_mps = if state == 0 { 1 - mps } else { mps }; + self.ctx.0[ctx] = (TRANS_LPS[state] << 1) | new_mps as u8; + } else { + self.cost += self.table.cost[state][1] as u64; + if state < 62 { + self.ctx.0[ctx] = (((state + 1) as u8) << 1) | mps as u8; + } + } + } + #[inline] + fn bypass(&mut self, _bin: u32) { + self.cost += 256; + } + fn terminate(&mut self, bin: u32) { + self.cost += if bin != 0 { 7 * 256 } else { 2 }; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A reference CABAC decoder (§9.3.1.2 / 9.3.3.2) used only to round-trip-test the encoder. + struct Dec<'a> { + data: &'a [u8], + pos: usize, + range: u32, + offset: u32, + ctx: Contexts, + } + impl<'a> Dec<'a> { + fn bit(&mut self) -> u32 { + let b = if self.pos / 8 < self.data.len() { (self.data[self.pos / 8] >> (7 - self.pos % 8)) & 1 } else { 0 }; + self.pos += 1; + b as u32 + } + fn new(data: &'a [u8], ctx: Contexts) -> Self { + let mut d = Dec { data, pos: 0, range: 510, offset: 0, ctx }; + for _ in 0..9 { + d.offset = (d.offset << 1) | d.bit(); + } + d + } + fn decision(&mut self, c: usize) -> u32 { + let s = self.ctx.0[c]; + let state = (s >> 1) as usize; + let mps = (s & 1) as u32; + let lps = RANGE_LPS[state][((self.range >> 6) & 3) as usize] as u32; + self.range -= lps; + let bin; + if self.offset >= self.range { + bin = 1 - mps; + self.offset -= self.range; + self.range = lps; + let new_mps = if state == 0 { 1 - mps } else { mps }; + self.ctx.0[c] = (TRANS_LPS[state] << 1) | new_mps as u8; + } else { + bin = mps; + if state < 62 { + self.ctx.0[c] = (((state + 1) as u8) << 1) | mps as u8; + } + } + while self.range < 256 { + self.range <<= 1; + self.offset = (self.offset << 1) | self.bit(); + } + bin + } + fn bypass(&mut self) -> u32 { + self.offset = (self.offset << 1) | self.bit(); + if self.offset >= self.range { + self.offset -= self.range; + 1 + } else { + 0 + } + } + fn terminate(&mut self) -> u32 { + self.range -= 2; + if self.offset >= self.range { + 1 + } else { + while self.range < 256 { + self.range <<= 1; + self.offset = (self.offset << 1) | self.bit(); + } + 0 + } + } + } + + fn roundtrip(n: usize, seed: u32) { + let table = crate::tables::cabac_init_pb0(); + let ctx = Contexts::init(&table, 26); + let mut enc = CabacEncoder::new(ctx.clone(), vec![0xa5]); + let mut seed = seed; + let mut rnd = || { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + seed >> 8 + }; + let mut ops = Vec::new(); + for _ in 0..n { + let kind = rnd() % 10; + let c = (rnd() % 400) as usize; + // skewed bins so contexts adapt + let bin = if rnd() % 100 < 80 { (c % 2) as u32 } else { 1 - (c % 2) as u32 }; + if kind < 7 { + enc.decision(c, bin); + ops.push((0, c, bin)); + } else if kind < 9 { + enc.bypass(bin); + ops.push((1, 0, bin)); + } else { + enc.terminate(0); + ops.push((2, 0, 0)); + } + } + let data = enc.finish(); + assert_eq!(data[0], 0xa5, "header byte changed"); + let mut dec = Dec::new(&data[1..], ctx); + for (i, &(k, c, bin)) in ops.iter().enumerate() { + let got = match k { + 0 => dec.decision(c), + 1 => dec.bypass(), + _ => dec.terminate(), + }; + assert_eq!(got, bin, "n {n} op {i}"); + } + assert_eq!(dec.terminate(), 1); + assert!(dec.pos.div_ceil(8) < data.len(), "n {n}: decoder read past the end"); + } + + #[test] + fn roundtrip_random_bins() { + roundtrip(20000, 12345); + } + + #[test] + fn roundtrip_short_streams() { + for n in 0..300 { + roundtrip(n, n as u32 * 7919 + 1); + } + } + + #[test] + fn long_bypass_runs_and_carries() { + let ctx = Contexts::init(&crate::tables::cabac_init_pb0(), 26); + for bin in 0..2 { + let mut enc = CabacEncoder::new(ctx.clone(), Vec::new()); + for _ in 0..5000 { + enc.bypass(bin); + } + let data = enc.finish(); + let mut dec = Dec::new(&data, ctx.clone()); + for i in 0..5000 { + assert_eq!(dec.bypass(), bin, "bin {bin} op {i}"); + } + assert_eq!(dec.terminate(), 1); + } + } +} diff --git a/crates/h264enc/src/cabac_mb.rs b/crates/h264enc/src/cabac_mb.rs new file mode 100644 index 0000000000..a4995573c1 --- /dev/null +++ b/crates/h264enc/src/cabac_mb.rs @@ -0,0 +1,446 @@ +//! CABAC binarisation and context selection for macroblock-layer syntax elements (§9.3.2, §9.3.3.1). + +use crate::cabac::BinSink; +use crate::mbinfo::{MbInfo, MbKind, Part}; +use crate::nal::SliceType; +use crate::syntax::{MbCode, NbCtx}; +use crate::tables::{BLK4_XY, LAST8X8, SIG8X8_FRAME}; + +pub fn write_skip_flag(s: &mut S, nb: &NbCtx, skip: bool) { + let cond = |m: Option<&MbInfo>| m.is_some_and(|m| !m.kind.is_skip()) as usize; + let base = if nb.slice_type == SliceType::B { 24 } else { 11 }; + s.decision(base + cond(nb.left) + cond(nb.top), skip as u32); +} + +fn eg_bypass(s: &mut S, mut v: u32, mut k: u32) { + loop { + if v >= (1 << k) { + s.bypass(1); + v -= 1 << k; + k += 1; + } else { + s.bypass(0); + while k > 0 { + k -= 1; + s.bypass((v >> k) & 1); + } + break; + } + } +} + +/// I16x16 / I_NxN mb_type bins. `off` = ctxIdxOffset of the (suffix) table; `i_slice` selects I-slice contexts. +fn write_intra_type(s: &mut S, nb: &NbCtx, c: &MbCode) { + let i_slice = nb.slice_type == SliceType::I; + let (off, b0ctx) = match nb.slice_type { + SliceType::I => { + let cond = |m: Option<&MbInfo>| m.is_some_and(|m| !m.kind.is_inxn()) as usize; + (3, 3 + cond(nb.left) + cond(nb.top)) + } + SliceType::P => { + s.decision(14, 1); // prefix: intra + (17, 17) + } + SliceType::B => { + // prefix 111101 + write_b_bins(s, nb, &[1, 1, 1, 1, 0, 1]); + (32, 32) + } + }; + if c.kind != MbKind::I16x16 { + s.decision(b0ctx, 0); + return; + } + s.decision(b0ctx, 1); + s.terminate(0); + let luma = (c.cbp & 15 != 0) as u32; + let chroma = (c.cbp >> 4) as u32; + let pred = c.i16_mode as u32; + if i_slice { + s.decision(off + 3, luma); + s.decision(off + 4, (chroma != 0) as u32); + if chroma != 0 { + s.decision(off + 5, (chroma == 2) as u32); + s.decision(off + 6, pred >> 1); + s.decision(off + 7, pred & 1); + } else { + s.decision(off + 6, pred >> 1); + s.decision(off + 7, pred & 1); + } + } else { + s.decision(off + 1, luma); + s.decision(off + 2, (chroma != 0) as u32); + if chroma != 0 { + s.decision(off + 2, (chroma == 2) as u32); + } + s.decision(off + 3, pred >> 1); + s.decision(off + 3, pred & 1); + } +} + +fn write_b_bins(s: &mut S, nb: &NbCtx, bins: &[u32]) { + for (i, &b) in bins.iter().enumerate() { + let ctx = match i { + 0 => { + let cond = |m: Option<&MbInfo>| m.is_some_and(|m| !matches!(m.kind, MbKind::BSkip | MbKind::BDirect)) as usize; + 27 + cond(nb.left) + cond(nb.top) + } + 1 => 27 + 3, + 2 => 27 + if bins[1] != 0 { 4 } else { 5 }, + _ => 27 + 5, + }; + s.decision(ctx, b); + } +} + +const B_BINS: [&[u32]; 23] = [ + &[0], + &[1, 0, 0], + &[1, 0, 1], + &[1, 1, 0, 0, 0, 0], + &[1, 1, 0, 0, 0, 1], + &[1, 1, 0, 0, 1, 0], + &[1, 1, 0, 0, 1, 1], + &[1, 1, 0, 1, 0, 0], + &[1, 1, 0, 1, 0, 1], + &[1, 1, 0, 1, 1, 0], + &[1, 1, 0, 1, 1, 1], + &[1, 1, 1, 1, 1, 0], + &[1, 1, 1, 0, 0, 0, 0], + &[1, 1, 1, 0, 0, 0, 1], + &[1, 1, 1, 0, 0, 1, 0], + &[1, 1, 1, 0, 0, 1, 1], + &[1, 1, 1, 0, 1, 0, 0], + &[1, 1, 1, 0, 1, 0, 1], + &[1, 1, 1, 0, 1, 1, 0], + &[1, 1, 1, 0, 1, 1, 1], + &[1, 1, 1, 1, 0, 0, 0], + &[1, 1, 1, 1, 0, 0, 1], + &[1, 1, 1, 1, 1, 1], +]; + +fn write_mvd(s: &mut S, comp: usize, v: i32, sum: u32) { + let base = if comp == 0 { 40 } else { 47 }; + let a = v.unsigned_abs(); + let inc = if sum < 3 { + 0 + } else if sum > 32 { + 2 + } else { + 1 + }; + let u = a.min(9); + s.decision(base + inc, (u > 0) as u32); + if u > 0 { + for k in 1..9u32 { + let ctx = base + [3, 4, 5, 6, 6, 6, 6, 6][(k - 1) as usize]; + if k < u { + s.decision(ctx, 1); + } else { + s.decision(ctx, 0); + break; + } + } + if a >= 9 { + eg_bypass(s, a - 9, 3); + } + s.bypass((v < 0) as u32); + } +} + +/// Sum of |mvd| of neighbours A and B for the 4x4 block at raster `r` (current MB info already holds earlier partitions). +fn mvd_sum(nb: &NbCtx, r: usize, list: usize, comp: usize) -> u32 { + let (x, y) = (r & 3, r >> 2); + let a = if x > 0 { nb.cur.mvd[list][r - 1][comp] } else { nb.left.map_or(0, |m| m.mvd[list][y * 4 + 3][comp]) }; + let b = if y > 0 { nb.cur.mvd[list][r - 4][comp] } else { nb.top.map_or(0, |m| m.mvd[list][12 + x][comp]) }; + a as u32 + b as u32 +} + +fn write_cbp(s: &mut S, nb: &NbCtx, cbp: u8) { + let luma_bit = |m: &MbInfo, b8: usize| (m.cbp >> b8) & 1; + for b8 in 0..4usize { + // (A, B) neighbour 8x8 blocks: (Some(mb), idx) where None = current MB + let cond = |m: Option<&MbInfo>, idx: usize, in_cur: bool| -> usize { + if in_cur { + ((cbp >> idx) & 1 == 0) as usize + } else { + match m { + None => 0, + Some(m) => (luma_bit(m, idx) == 0) as usize, + } + } + }; + let (ca, cb) = match b8 { + 0 => (cond(nb.left, 1, false), cond(nb.top, 2, false)), + 1 => (cond(None, 0, true), cond(nb.top, 3, false)), + 2 => (cond(nb.left, 3, false), cond(None, 0, true)), + _ => (cond(None, 2, true), cond(None, 1, true)), + }; + s.decision(73 + ca + 2 * cb, ((cbp >> b8) & 1) as u32); + } + let chroma = cbp >> 4; + let cc = |m: Option<&MbInfo>, two: bool| -> usize { + m.is_some_and(|m| { + let c = m.cbp >> 4; + !m.kind.is_skip() && if two { c == 2 } else { c != 0 } + }) as usize + }; + s.decision(77 + cc(nb.left, false) + 2 * cc(nb.top, false), (chroma != 0) as u32); + if chroma != 0 { + s.decision(77 + 4 + cc(nb.left, true) + 2 * cc(nb.top, true), (chroma == 2) as u32); + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Cat { + LumaDc = 0, + LumaAc = 1, + Luma4x4 = 2, + ChromaDc = 3, + ChromaAc = 4, + Luma8x8 = 5, +} + +/// Encode one residual block; `coeffs` are the block's coefficients in scan order (length = maxNumCoeff). +pub fn write_residual(s: &mut S, cat: Cat, coeffs: &[i16], cbf_inc: u32) { + const CBF_OFF: [usize; 5] = [0, 4, 8, 12, 16]; + const SIG_OFF: [usize; 5] = [0, 15, 29, 44, 47]; + const ABS_OFF: [usize; 5] = [0, 10, 20, 30, 39]; + let n = coeffs.len(); + let last = coeffs.iter().rposition(|&c| c != 0); + let ci = cat as usize; + if cat != Cat::Luma8x8 { + s.decision(85 + CBF_OFF[ci] + cbf_inc as usize, last.is_some() as u32); + } + let Some(last) = last else { return }; + let (sig_base, last_base, abs_base) = + if cat == Cat::Luma8x8 { (402, 417, 426) } else { (105 + SIG_OFF[ci], 166 + SIG_OFF[ci], 227 + ABS_OFF[ci]) }; + for i in 0..n - 1 { + let (si, li) = match cat { + Cat::ChromaDc => (i.min(2), i.min(2)), + Cat::Luma8x8 => (SIG8X8_FRAME[i] as usize, LAST8X8[i] as usize), + _ => (i, i), + }; + let sig = coeffs[i] != 0; + s.decision(sig_base + si, sig as u32); + if sig { + let is_last = i == last; + s.decision(last_base + li, is_last as u32); + if is_last { + break; + } + } + } + let mut gt1 = 0usize; + let mut eq1 = 0usize; + let max_gt1 = if cat == Cat::ChromaDc { 3 } else { 4 }; + for i in (0..=last).rev() { + let c = coeffs[i]; + if c == 0 { + continue; + } + let a = c.unsigned_abs() as u32; + let v = a - 1; + let inc0 = if gt1 != 0 { 0 } else { (1 + eq1).min(4) }; + s.decision(abs_base + inc0, (v > 0) as u32); + if v > 0 { + let ctx = abs_base + 5 + gt1.min(max_gt1); + let u = v.min(14); + for _ in 1..u { + s.decision(ctx, 1); + } + if u < 14 { + s.decision(ctx, 0); + } else { + eg_bypass(s, v - 14, 0); + } + } + s.bypass((c < 0) as u32); + if a == 1 { + eq1 += 1; + } else { + gt1 += 1; + } + } +} + +fn cbf_luma(nb: &NbCtx, raster: usize) -> u32 { + let f = |v: Option| v.map_or(nb.unavail_cbf(), |n| (n != 0) as u32); + f(nb.luma_nnz(raster, true)) + 2 * f(nb.luma_nnz(raster, false)) +} + +fn cbf_chroma_ac(nb: &NbCtx, c: usize, b: usize) -> u32 { + let f = |v: Option| v.map_or(nb.unavail_cbf(), |n| (n != 0) as u32); + f(nb.chroma_nnz(c, b, true)) + 2 * f(nb.chroma_nnz(c, b, false)) +} + +fn cbf_dc(nb: &NbCtx, bit: u8) -> u32 { + let f = |m: Option<&MbInfo>| m.map_or(nb.unavail_cbf(), |m| ((m.dc_cbf >> bit) & 1) as u32); + f(nb.left) + 2 * f(nb.top) +} + +/// mb_type (and the P_L0_8x8 sub_mb_types of a P_8x8 macroblock). +fn write_mb_type(s: &mut S, nb: &NbCtx, c: &MbCode) { + match c.kind { + MbKind::I4x4 | MbKind::I8x8 | MbKind::I16x16 => write_intra_type(s, nb, c), + MbKind::PInter => { + let bins: [u32; 3] = match c.part { + Part::P16x16 => [0, 0, 0], + Part::P16x8 => [0, 1, 1], + Part::P8x16 => [0, 1, 0], + Part::P8x8 => [0, 0, 1], + }; + s.decision(14, bins[0]); + s.decision(15, bins[1]); + s.decision(if bins[1] != 1 { 16 } else { 17 }, bins[2]); + } + MbKind::BDirect | MbKind::BInter => write_b_bins(s, nb, B_BINS[c.b_type_number() as usize]), + // Skip / none macroblocks are not written with write_mb. + _ => debug_assert!(false, "write_mb on a skipped macroblock"), + } + if c.kind == MbKind::PInter && c.part == Part::P8x8 { + for _ in 0..4 { + s.decision(21, 1); // P_L0_8x8 + } + } +} + +/// transform_size_8x8_flag. +fn write_transform_size_flag(s: &mut S, nb: &NbCtx, c: &MbCode) { + let cond = |m: Option<&MbInfo>| m.is_some_and(|m| m.t8x8) as usize; + s.decision(399 + cond(nb.left) + cond(nb.top), c.t8x8 as u32); +} + +/// prev_intra_NxN_pred_mode flags / rem_intra_NxN_pred_mode values. +fn write_intra_pred_modes(s: &mut S, c: &MbCode) { + let n = if c.t8x8 { 4 } else { 16 }; + for i in 0..n { + let p = c.ipred[i]; + if p < 0 { + s.decision(68, 1); + } else { + s.decision(68, 0); + s.decision(69, (p & 1) as u32); + s.decision(69, ((p >> 1) & 1) as u32); + s.decision(69, ((p >> 2) & 1) as u32); + } + } +} + +/// intra_chroma_pred_mode. +fn write_chroma_pred_mode(s: &mut S, nb: &NbCtx, c: &MbCode) { + let cond = |m: Option<&MbInfo>| m.is_some_and(|m| m.kind.is_intra() && m.chroma_mode != 0) as usize; + let m = c.chroma_mode as u32; + s.decision(64 + cond(nb.left) + cond(nb.top), (m > 0) as u32); + if m > 0 { + s.decision(64 + 3, (m > 1) as u32); + if m > 1 { + s.decision(64 + 3, (m > 2) as u32); + } + } +} + +/// mvd_lX components of every partition that uses each list. +fn write_mvds(s: &mut S, nb: &NbCtx, c: &MbCode) { + for list in 0..2 { + for p in 0..c.part.count() { + if !c.uses_list(p, list) { + continue; + } + let (px, py, _, _) = c.part.rect(p); + let r = (py / 4) * 4 + px / 4; + let mvd = c.mvd[list][p]; + write_mvd(s, 0, mvd.x as i32, mvd_sum(nb, r, list, 0)); + write_mvd(s, 1, mvd.y as i32, mvd_sum(nb, r, list, 1)); + } + } +} + +/// mb_qp_delta. +fn write_qp_delta(s: &mut S, nb: &NbCtx, v: i32) { + let m = if v > 0 { 2 * v as u32 - 1 } else { (-2 * v) as u32 }; + s.decision(60 + nb.prev_dqp_nz as usize, (m > 0) as u32); + if m > 0 { + for k in 1..=m { + s.decision(if k == 1 { 62 } else { 63 }, (k < m) as u32); + } + } +} + +/// Luma residual blocks of the macroblock. +fn write_luma_residual(s: &mut S, nb: &NbCtx, c: &MbCode) { + if c.kind == MbKind::I16x16 { + write_residual(s, Cat::LumaDc, &c.luma_dc, cbf_dc(nb, 0)); + if c.cbp & 15 != 0 { + for blk in 0..16 { + let (x, y) = BLK4_XY[blk]; + write_residual(s, Cat::LumaAc, &c.luma4[blk][1..16], cbf_luma(nb, y * 4 + x)); + } + } + return; + } + for b8 in 0..4 { + if (c.cbp >> b8) & 1 == 0 { + continue; + } + if c.t8x8 { + write_residual(s, Cat::Luma8x8, &c.luma8[b8], 0); + } else { + for i in 0..4 { + let blk = b8 * 4 + i; + let (x, y) = BLK4_XY[blk]; + write_residual(s, Cat::Luma4x4, &c.luma4[blk], cbf_luma(nb, y * 4 + x)); + } + } + } +} + +/// Chroma DC / AC residual blocks of the macroblock. +fn write_chroma_residual(s: &mut S, nb: &NbCtx, c: &MbCode) { + let chroma = c.cbp >> 4; + if chroma != 0 { + for comp in 0..2 { + write_residual(s, Cat::ChromaDc, &c.cdc[comp], cbf_dc(nb, 1 + comp as u8)); + } + } + if chroma == 2 { + for comp in 0..2 { + for b in 0..4 { + write_residual(s, Cat::ChromaAc, &c.cac[comp * 4 + b][1..16], cbf_chroma_ac(nb, comp, b)); + } + } + } +} + +/// Write the whole macroblock layer (after mb_skip_flag = 0 in P/B slices). +pub fn write_mb(s: &mut S, nb: &NbCtx, c: &MbCode) { + write_mb_type(s, nb, c); + let is_inxn = c.kind.is_inxn(); + // ---- mb_pred + if is_inxn { + if nb.t8x8_mode { + write_transform_size_flag(s, nb, c); + } + write_intra_pred_modes(s, c); + } + if c.kind.is_intra() { + write_chroma_pred_mode(s, nb, c); + } + if matches!(c.kind, MbKind::PInter | MbKind::BInter) { + write_mvds(s, nb, c); + } + // ---- cbp, transform size, qp delta + if c.kind != MbKind::I16x16 { + write_cbp(s, nb, c.cbp); + if c.cbp & 15 != 0 && nb.t8x8_mode && !is_inxn { + write_transform_size_flag(s, nb, c); + } + } + if c.cbp == 0 && c.kind != MbKind::I16x16 { + return; + } + write_qp_delta(s, nb, c.qp_delta); + // ---- residual + write_luma_residual(s, nb, c); + write_chroma_residual(s, nb, c); +} diff --git a/crates/h264enc/src/cavlc.rs b/crates/h264enc/src/cavlc.rs new file mode 100644 index 0000000000..a40fc417d2 --- /dev/null +++ b/crates/h264enc/src/cavlc.rs @@ -0,0 +1,288 @@ +//! CAVLC residual coding (§9.2) and macroblock-layer syntax for Baseline streams. + +use filmcraft_bitstream::BitWriter; + +use crate::mbinfo::{MbKind, Part}; +use crate::nal::SliceType; +use crate::syntax::{MbCode, NbCtx}; +use crate::tables::{ + BLK4_XY, CBP_INTER_TO_CODE, CBP_INTRA_TO_CODE, COEFF_TOKEN_CDC_CODE, COEFF_TOKEN_CDC_LEN, COEFF_TOKEN_CODE, COEFF_TOKEN_LEN, RUN_BEFORE, + TOTAL_ZEROS, TOTAL_ZEROS_CDC, +}; + +/// Largest level magnitude representable with level_prefix <= 15 in every suffixLength state. +pub const MAX_LEVEL: i32 = 2047; + +/// Write one CAVLC residual block. `nc` = -1 for chroma DC. Returns TotalCoeff. +pub fn write_residual(w: &mut BitWriter, coeffs: &[i16], nc: i32) -> u8 { + let max = coeffs.len(); + // levels from highest frequency to lowest + let mut levels = [0i32; 16]; + let mut pos = [0usize; 16]; + let mut total = 0usize; + for i in (0..max).rev() { + if coeffs[i] != 0 { + levels[total] = coeffs[i] as i32; + pos[total] = i; + total += 1; + } + } + let mut t1 = 0usize; + while t1 < total && t1 < 3 && levels[t1].abs() == 1 { + t1 += 1; + } + // coeff_token + if nc == -1 { + w.write_bits(COEFF_TOKEN_CDC_CODE[total][t1] as u32, COEFF_TOKEN_CDC_LEN[total][t1] as u32); + } else if nc >= 8 { + if total == 0 { + w.write_bits(3, 6); + } else { + w.write_bits((((total - 1) << 2) | t1) as u32, 6); + } + } else { + let t = if nc < 2 { + 0 + } else if nc < 4 { + 1 + } else { + 2 + }; + w.write_bits(COEFF_TOKEN_CODE[t][total][t1] as u32, COEFF_TOKEN_LEN[t][total][t1] as u32); + } + if total == 0 { + return 0; + } + let mut suffix_len: u32 = if total > 10 && t1 < 3 { 1 } else { 0 }; + for i in 0..total { + let l = levels[i]; + if i < t1 { + w.write_bit(l < 0); + continue; + } + let mut code = if l > 0 { 2 * l - 2 } else { -2 * l - 1 }; + if i == t1 && t1 < 3 { + code -= 2; + } + let code = code as u32; + if suffix_len == 0 { + if code < 14 { + w.write_bits(1, code + 1); + } else if code < 30 { + w.write_bits(1, 15); + w.write_bits(code - 14, 4); + } else { + write_escape(w, code - 30); + } + } else if code < (15 << suffix_len) { + w.write_bits(1, (code >> suffix_len) + 1); + w.write_bits(code & ((1 << suffix_len) - 1), suffix_len); + } else { + write_escape(w, code - (15 << suffix_len)); + } + if suffix_len == 0 { + suffix_len = 1; + } + if l.unsigned_abs() > (3 << (suffix_len - 1)) && suffix_len < 6 { + suffix_len += 1; + } + } + let total_zeros = pos[0] + 1 - total; + if total < max { + let (len, code) = if nc == -1 { TOTAL_ZEROS_CDC[total - 1][total_zeros] } else { TOTAL_ZEROS[total - 1][total_zeros] }; + w.write_bits(code as u32, len as u32); + } + let mut zeros_left = total_zeros; + for i in 0..total - 1 { + if zeros_left == 0 { + break; + } + let run = pos[i] - pos[i + 1] - 1; + let (len, code) = RUN_BEFORE[zeros_left.min(7) - 1][run]; + w.write_bits(code as u32, len as u32); + zeros_left -= run; + } + total as u8 +} + +/// level_prefix >= 15 escape; `rem` = levelCode minus the prefix-15 base. +fn write_escape(w: &mut BitWriter, rem: u32) { + if rem < 4096 { + w.write_bits(1, 16); + w.write_bits(rem, 12); + } else { + // level_prefix p >= 16 (High profiles): suffix of p-3 bits, value rem + 4096 - 2^(p-3) + let mut p = 16; + while rem + 4096 >= (1 << (p - 3)) * 2 { + p += 1; + } + w.write_bits(0, p); + w.write_bit(true); + w.write_bits(rem + 4096 - (1 << (p - 3)), p - 3); + } +} + +fn nc_of(a: Option, b: Option) -> i32 { + match (a, b) { + (Some(a), Some(b)) => (a as i32 + b as i32 + 1) >> 1, + (Some(a), None) => a as i32, + (None, Some(b)) => b as i32, + _ => 0, + } +} + +/// mb_type (and the P_8x8 sub_mb_types). +fn write_mb_type(w: &mut BitWriter, nb: &NbCtx, c: &MbCode) { + let intra_base = match nb.slice_type { + SliceType::I => 0, + SliceType::P => 5, + SliceType::B => 23, + }; + match c.kind { + MbKind::I4x4 | MbKind::I8x8 => w.write_ue(intra_base), + MbKind::I16x16 => { + let t = 1 + c.i16_mode as u32 + 4 * (c.cbp >> 4) as u32 + if c.cbp & 15 != 0 { 12 } else { 0 }; + w.write_ue(intra_base + t); + } + MbKind::PInter => w.write_ue(match c.part { + Part::P16x16 => 0, + Part::P16x8 => 1, + Part::P8x16 => 2, + Part::P8x8 => 3, + }), + MbKind::BDirect | MbKind::BInter => w.write_ue(c.b_type_number()), + // Skip / none macroblocks are not written with write_mb. + _ => debug_assert!(false, "write_mb on a skipped macroblock"), + } + if c.kind == MbKind::PInter && c.part == Part::P8x8 { + for _ in 0..4 { + w.write_ue(0); + } + } +} + +/// prev_intra_NxN_pred_mode flags / rem_intra_NxN_pred_mode values. +fn write_intra_pred_modes(w: &mut BitWriter, c: &MbCode) { + let n = if c.t8x8 { 4 } else { 16 }; + for i in 0..n { + let p = c.ipred[i]; + if p < 0 { + w.write_bit(true); + } else { + w.write_bit(false); + w.write_bits(p as u32, 3); + } + } +} + +/// coded_block_pattern. +fn write_cbp(w: &mut BitWriter, c: &MbCode) { + let code = if c.kind.is_intra() { CBP_INTRA_TO_CODE[c.cbp as usize] } else { CBP_INTER_TO_CODE[c.cbp as usize] }; + let cbp_idx = (c.cbp & 15) | ((c.cbp >> 4) << 4); + debug_assert_eq!(cbp_idx, c.cbp); + w.write_ue(code as u32); +} + +/// Luma residual blocks of the macroblock. +fn write_luma_residual(w: &mut BitWriter, nb: &NbCtx, c: &MbCode) { + let luma_nc = |r: usize| nc_of(nb.luma_nnz(r, true), nb.luma_nnz(r, false)); + if c.kind == MbKind::I16x16 { + write_residual(w, &c.luma_dc, luma_nc(0)); + if c.cbp & 15 != 0 { + for blk in 0..16 { + let (x, y) = BLK4_XY[blk]; + write_residual(w, &c.luma4[blk][1..16], luma_nc(y * 4 + x)); + } + } + return; + } + for b8 in 0..4 { + if (c.cbp >> b8) & 1 == 0 { + continue; + } + for i in 0..4 { + let blk = b8 * 4 + i; + let (x, y) = BLK4_XY[blk]; + if c.t8x8 { + let mut sub = [0i16; 16]; + for k in 0..16 { + sub[k] = c.luma8[b8][4 * k + i]; + } + write_residual(w, &sub, luma_nc(y * 4 + x)); + } else { + write_residual(w, &c.luma4[blk], luma_nc(y * 4 + x)); + } + } + } +} + +/// Chroma DC / AC residual blocks of the macroblock. +fn write_chroma_residual(w: &mut BitWriter, nb: &NbCtx, c: &MbCode) { + let chroma = c.cbp >> 4; + if chroma != 0 { + for comp in 0..2 { + write_residual(w, &c.cdc[comp], -1); + } + } + if chroma == 2 { + for comp in 0..2 { + for b in 0..4 { + let ncv = nc_of(nb.chroma_nnz(comp, b, true), nb.chroma_nnz(comp, b, false)); + write_residual(w, &c.cac[comp * 4 + b][1..16], ncv); + } + } + } +} + +/// Write the macroblock layer in CAVLC. For P/B slices the caller writes mb_skip_run first. +pub fn write_mb(w: &mut BitWriter, nb: &NbCtx, c: &MbCode) { + write_mb_type(w, nb, c); + if c.kind.is_inxn() { + if nb.t8x8_mode { + w.write_bit(c.t8x8); + } + write_intra_pred_modes(w, c); + } + if c.kind.is_intra() { + w.write_ue(c.chroma_mode as u32); + } + if matches!(c.kind, MbKind::PInter | MbKind::BInter) { + for list in 0..2 { + for p in 0..c.part.count() { + if c.uses_list(p, list) { + w.write_se(c.mvd[list][p].x as i32); + w.write_se(c.mvd[list][p].y as i32); + } + } + } + } + if c.kind != MbKind::I16x16 { + write_cbp(w, c); + if c.cbp & 15 != 0 && nb.t8x8_mode && !c.kind.is_inxn() { + w.write_bit(c.t8x8); + } + } + if c.cbp == 0 && c.kind != MbKind::I16x16 { + return; + } + w.write_se(c.qp_delta); + write_luma_residual(w, nb, c); + write_chroma_residual(w, nb, c); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn simple_block_matches_spec_example() { + // scan: 0,3,-1,0,0,-1,1,0,1 -> TotalCoeff 5, T1s 3. + let coeffs: [i16; 16] = [0, 3, -1, 0, 0, -1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0]; + let mut w = BitWriter::new(); + let t = write_residual(&mut w, &coeffs, 0); + assert_eq!(t, 5); + // coeff_token 0000100 | signs 001 | level -1: 01 | level 3: 001 0 | total_zeros(4) 110 | runs 10 11 01 1 + let bits: String = w.finish().iter().map(|b| format!("{b:08b}")).collect(); + assert!(bits.starts_with("00001000010100101101011011"), "{bits}"); + } +} diff --git a/crates/h264enc/src/deblock.rs b/crates/h264enc/src/deblock.rs new file mode 100644 index 0000000000..2e18995844 --- /dev/null +++ b/crates/h264enc/src/deblock.rs @@ -0,0 +1,263 @@ +//! In-loop deblocking filter (§8.7) for frame-coded 4:2:0 pictures without MBAFF. +//! +//! Operates on a band of macroblock rows that begins at a slice boundary; the top edge of the band is +//! never filtered (slices are MB-row aligned and either there is one slice, or +//! `disable_deblocking_filter_idc = 2` is used). + +use crate::mbinfo::{MbInfo, Mv}; +use crate::tables::{ALPHA, BETA, TC0, chroma_qp}; + +pub struct DeblockParams { + pub alpha_offset: i32, + pub beta_offset: i32, + pub chroma_qp_offset: i32, + /// Picture identity for (list, ref_idx 0). + pub ref_ids: [i32; 2], +} + +/// A mutable band of rows of a padded plane: sample (x, y) with y relative to the band top. +pub struct BandPlane<'a> { + pub data: &'a mut [u8], + pub stride: usize, + pub pad: usize, +} + +impl BandPlane<'_> { + #[inline] + fn idx(&self, x: usize, y: usize) -> usize { + y * self.stride + self.pad + x + } +} + +#[inline] +fn nz8(m: &MbInfo, blk: usize) -> bool { + if m.t8x8 { + let (x, y) = ((blk & 3) >> 1, (blk >> 2) >> 1); + let b = y * 8 + x * 2; + m.nnz[b] | m.nnz[b + 1] | m.nnz[b + 4] | m.nnz[b + 5] != 0 + } else { + m.nnz[blk] != 0 + } +} + +#[inline] +fn mvdiff(a: Mv, b: Mv) -> bool { + (a.x as i32 - b.x as i32).abs() >= 4 || (a.y as i32 - b.y as i32).abs() >= 4 +} + +fn bs_inter(p: &MbInfo, ip: usize, q: &MbInfo, iq: usize, ids: &[i32; 2]) -> u8 { + if nz8(p, ip) || nz8(q, iq) { + return 2; + } + let rp = [p.ref_at(0, ip), p.ref_at(1, ip)]; + let rq = [q.ref_at(0, iq), q.ref_at(1, iq)]; + // Only ref_idx 0 is ever used per list, so the list index identifies the picture. + let pic = |r: i8, l: usize| if r >= 0 { ids[l] } else { -1 }; + let pp = [pic(rp[0], 0), pic(rp[1], 1)]; + let pq = [pic(rq[0], 0), pic(rq[1], 1)]; + let np = (pp[0] >= 0) as u8 + (pp[1] >= 0) as u8; + let nq = (pq[0] >= 0) as u8 + (pq[1] >= 0) as u8; + if np != nq { + return 1; + } + if np == 1 { + let (a, ma) = if pp[0] >= 0 { (pp[0], p.mv[0][ip]) } else { (pp[1], p.mv[1][ip]) }; + let (b, mb) = if pq[0] >= 0 { (pq[0], q.mv[0][iq]) } else { (pq[1], q.mv[1][iq]) }; + if a != b { + return 1; + } + return mvdiff(ma, mb) as u8; + } + if np == 0 { + return 0; + } + // two motion vectors each + let (mp0, mp1, mq0, mq1) = (p.mv[0][ip], p.mv[1][ip], q.mv[0][iq], q.mv[1][iq]); + let same_set = (pp[0] == pq[0] && pp[1] == pq[1]) || (pp[0] == pq[1] && pp[1] == pq[0]); + if !same_set { + return 1; + } + if pp[0] != pp[1] { + // two different pictures: compare mvs referring to the same picture + if pp[0] == pq[0] { (mvdiff(mp0, mq0) || mvdiff(mp1, mq1)) as u8 } else { (mvdiff(mp0, mq1) || mvdiff(mp1, mq0)) as u8 } + } else { + ((mvdiff(mp0, mq0) || mvdiff(mp1, mq1)) && (mvdiff(mp0, mq1) || mvdiff(mp1, mq0))) as u8 + } +} + +#[inline] +fn bs_of(p: &MbInfo, ip: usize, q: &MbInfo, iq: usize, mb_edge: bool, ids: &[i32; 2]) -> u8 { + if p.kind.is_intra() || q.kind.is_intra() { + return if mb_edge { 4 } else { 3 }; + } + bs_inter(p, ip, q, iq, ids) +} + +#[inline] +fn filter_luma(d: &mut [u8], q0i: usize, step: usize, bs: u8, alpha: i32, beta: i32, tc0: i32) { + let p0 = d[q0i - step] as i32; + let q0 = d[q0i] as i32; + let p1 = d[q0i - 2 * step] as i32; + let q1 = d[q0i + step] as i32; + if (p0 - q0).abs() >= alpha || (p1 - p0).abs() >= beta || (q1 - q0).abs() >= beta { + return; + } + let p2 = d[q0i - 3 * step] as i32; + let q2 = d[q0i + 2 * step] as i32; + let ap = (p2 - p0).abs(); + let aq = (q2 - q0).abs(); + if bs < 4 { + let tc = tc0 + (ap < beta) as i32 + (aq < beta) as i32; + let delta = ((((q0 - p0) << 2) + (p1 - q1) + 4) >> 3).clamp(-tc, tc); + d[q0i - step] = (p0 + delta).clamp(0, 255) as u8; + d[q0i] = (q0 - delta).clamp(0, 255) as u8; + if ap < beta { + d[q0i - 2 * step] = (p1 + ((p2 + ((p0 + q0 + 1) >> 1) - (p1 << 1)) >> 1).clamp(-tc0, tc0)) as u8; + } + if aq < beta { + d[q0i + step] = (q1 + ((q2 + ((p0 + q0 + 1) >> 1) - (q1 << 1)) >> 1).clamp(-tc0, tc0)) as u8; + } + } else { + let p3 = d[q0i - 4 * step] as i32; + let q3 = d[q0i + 3 * step] as i32; + let strong = (p0 - q0).abs() < ((alpha >> 2) + 2); + if ap < beta && strong { + d[q0i - step] = ((p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3) as u8; + d[q0i - 2 * step] = ((p2 + p1 + p0 + q0 + 2) >> 2) as u8; + d[q0i - 3 * step] = ((2 * p3 + 3 * p2 + p1 + p0 + q0 + 4) >> 3) as u8; + } else { + d[q0i - step] = ((2 * p1 + p0 + q1 + 2) >> 2) as u8; + } + if aq < beta && strong { + d[q0i] = ((p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + 4) >> 3) as u8; + d[q0i + step] = ((p0 + q0 + q1 + q2 + 2) >> 2) as u8; + d[q0i + 2 * step] = ((2 * q3 + 3 * q2 + q1 + q0 + p0 + 4) >> 3) as u8; + } else { + d[q0i] = ((2 * q1 + q0 + p1 + 2) >> 2) as u8; + } + } +} + +#[inline] +fn filter_chroma(d: &mut [u8], q0i: usize, step: usize, bs: u8, alpha: i32, beta: i32, tc0: i32) { + let p0 = d[q0i - step] as i32; + let q0 = d[q0i] as i32; + let p1 = d[q0i - 2 * step] as i32; + let q1 = d[q0i + step] as i32; + if (p0 - q0).abs() >= alpha || (p1 - p0).abs() >= beta || (q1 - q0).abs() >= beta { + return; + } + if bs < 4 { + let tc = tc0 + 1; + let delta = ((((q0 - p0) << 2) + (p1 - q1) + 4) >> 3).clamp(-tc, tc); + d[q0i - step] = (p0 + delta).clamp(0, 255) as u8; + d[q0i] = (q0 - delta).clamp(0, 255) as u8; + } else { + d[q0i - step] = ((2 * p1 + p0 + q1 + 2) >> 2) as u8; + d[q0i] = ((2 * q1 + q0 + p1 + 2) >> 2) as u8; + } +} + +struct EdgeQ { + alpha: i32, + beta: i32, + idx_a: usize, +} + +fn edge_q(qp_p: i32, qp_q: i32, dp: &DeblockParams) -> EdgeQ { + let qav = (qp_p + qp_q + 1) >> 1; + let ia = (qav + dp.alpha_offset).clamp(0, 51) as usize; + let ib = (qav + dp.beta_offset).clamp(0, 51) as usize; + EdgeQ { alpha: ALPHA[ia] as i32, beta: BETA[ib] as i32, idx_a: ia } +} + +/// Deblock `rows` macroblock rows of a band. `mbs` holds the band's MbInfo (row-major, `mbw` wide). +pub fn deblock_band<'a>( + y: &mut BandPlane<'a>, + u: &mut BandPlane<'a>, + v: &mut BandPlane<'a>, + mbs: &[MbInfo], + mbw: usize, + rows: usize, + dp: &DeblockParams, +) { + let ids = &dp.ref_ids; + for my in 0..rows { + for mx in 0..mbw { + let q = &mbs[my * mbw + mx]; + let cqp = |m: &MbInfo| chroma_qp(m.qp as i32 + dp.chroma_qp_offset) as i32; + // ---- vertical edges (dir 0) then horizontal (dir 1) + for dir in 0..2 { + let mut bs_all = [[0u8; 4]; 4]; + for e in 0..4 { + if e == 0 && ((dir == 0 && mx == 0) || (dir == 1 && my == 0)) { + continue; + } + if e & 1 == 1 && q.t8x8 { + continue; + } + let p_mb = if e == 0 { if dir == 0 { &mbs[my * mbw + mx - 1] } else { &mbs[(my - 1) * mbw + mx] } } else { q }; + let mut bs = [0u8; 4]; + for s in 0..4 { + let (iq, ip) = if dir == 0 { + let iq = s * 4 + e; + (iq, if e == 0 { s * 4 + 3 } else { iq - 1 }) + } else { + let iq = e * 4 + s; + (iq, if e == 0 { 12 + s } else { iq - 4 }) + }; + bs[s] = bs_of(p_mb, ip, q, iq, e == 0, ids); + } + bs_all[e] = bs; + if bs == [0; 4] { + continue; + } + let eq = edge_q(p_mb.qp as i32, q.qp as i32, dp); + if eq.alpha == 0 || eq.beta == 0 { + continue; + } + for k in 0..16 { + let b = bs[k >> 2]; + if b == 0 { + continue; + } + let tc0 = if b < 4 { TC0[eq.idx_a][b as usize - 1] as i32 } else { 0 }; + let (qi, step) = + if dir == 0 { (y.idx(mx * 16 + e * 4, my * 16 + k), 1) } else { (y.idx(mx * 16 + k, my * 16 + e * 4), y.stride) }; + filter_luma(y.data, qi, step, b, eq.alpha, eq.beta, tc0); + } + } + // chroma edges 0 and 2 (chroma sample offsets 0 and 4) + for e in [0usize, 2] { + if e == 0 && ((dir == 0 && mx == 0) || (dir == 1 && my == 0)) { + continue; + } + let bs = bs_all[e]; + if bs == [0; 4] { + continue; + } + let p_mb = if e == 0 { if dir == 0 { &mbs[my * mbw + mx - 1] } else { &mbs[(my - 1) * mbw + mx] } } else { q }; + let eq = edge_q(cqp(p_mb), cqp(q), dp); + if eq.alpha == 0 || eq.beta == 0 { + continue; + } + for plane in [&mut *u, &mut *v] { + for k in 0..8 { + let b = bs[k >> 1]; + if b == 0 { + continue; + } + let tc0 = if b < 4 { TC0[eq.idx_a][b as usize - 1] as i32 } else { 0 }; + let (qi, step) = if dir == 0 { + (plane.idx(mx * 8 + e * 2, my * 8 + k), 1) + } else { + (plane.idx(mx * 8 + k, my * 8 + e * 2), plane.stride) + }; + filter_chroma(plane.data, qi, step, b, eq.alpha, eq.beta, tc0); + } + } + } + } + } + } +} diff --git a/crates/h264enc/src/dsp.rs b/crates/h264enc/src/dsp.rs new file mode 100644 index 0000000000..3baecc91dd --- /dev/null +++ b/crates/h264enc/src/dsp.rs @@ -0,0 +1,255 @@ +//! Distortion metrics: SAD, SATD (4x4 Hadamard), SSD. + +#[inline] +pub fn sad(a: &[u8], sa: usize, b: &[u8], sb: usize, w: usize, h: usize) -> u32 { + let mut s = 0u32; + for r in 0..h { + let ra = &a[r * sa..r * sa + w]; + let rb = &b[r * sb..r * sb + w]; + s += ra.iter().zip(rb).map(|(&x, &y)| x.abs_diff(y) as u32).sum::(); + } + s +} + +/// SAD of a 16-wide block (fast path). +#[inline] +pub fn sad16(a: &[u8], sa: usize, b: &[u8], sb: usize, h: usize) -> u32 { + sad_n::<16>(a, sa, b, sb, h) +} + +/// SAD of an 8-wide block (fast path). +#[inline] +pub fn sad8(a: &[u8], sa: usize, b: &[u8], sb: usize, h: usize) -> u32 { + sad_n::<8>(a, sa, b, sb, h) +} + +#[inline(always)] +fn sad_n(a: &[u8], sa: usize, b: &[u8], sb: usize, h: usize) -> u32 { + let mut s = 0u32; + for r in 0..h { + let (Some(ra), Some(rb)) = (a.get(r * sa..).and_then(|x| x.first_chunk::()), b.get(r * sb..).and_then(|x| x.first_chunk::())) else { + break; + }; + let mut acc = 0u16; + for i in 0..N { + acc += ra[i].abs_diff(rb[i]) as u16; + } + s += acc as u32; + } + s +} + +#[cfg(test)] +fn hadamard4x4_sum(d: &[i32; 16]) -> u32 { + let mut t = [0i32; 16]; + for i in 0..4 { + let a0 = d[i * 4] + d[i * 4 + 1]; + let a1 = d[i * 4] - d[i * 4 + 1]; + let a2 = d[i * 4 + 2] + d[i * 4 + 3]; + let a3 = d[i * 4 + 2] - d[i * 4 + 3]; + t[i * 4] = a0 + a2; + t[i * 4 + 1] = a1 + a3; + t[i * 4 + 2] = a0 - a2; + t[i * 4 + 3] = a1 - a3; + } + let mut s = 0u32; + for i in 0..4 { + let a0 = t[i] + t[4 + i]; + let a1 = t[i] - t[4 + i]; + let a2 = t[8 + i] + t[12 + i]; + let a3 = t[8 + i] - t[12 + i]; + s += (a0 + a2).unsigned_abs() + (a1 + a3).unsigned_abs() + (a0 - a2).unsigned_abs() + (a1 - a3).unsigned_abs(); + } + s +} + +/// SATD of a strip of `N` columns (N/4 side-by-side 4x4 blocks) and 4 rows, computed lane-wise so the compiler +/// can vectorise it. Uses |p+q| + |p-q| = 2 max(|p|, |q|) for the last butterfly, which also absorbs the /2. +#[inline(always)] +fn satd_strip(a: &[u8], sa: usize, b: &[u8], sb: usize) -> u32 { + let mut d = [[0i16; N]; 4]; + for r in 0..4 { + let ra = &a[r * sa..r * sa + N]; + let rb = &b[r * sb..r * sb + N]; + for i in 0..N { + d[r][i] = ra[i] as i16 - rb[i] as i16; + } + } + let mut v = [[0i16; N]; 4]; + for i in 0..N { + let s01 = d[0][i] + d[1][i]; + let d01 = d[0][i] - d[1][i]; + let s23 = d[2][i] + d[3][i]; + let d23 = d[2][i] - d[3][i]; + v[0][i] = s01 + s23; + v[1][i] = s01 - s23; + v[2][i] = d01 + d23; + v[3][i] = d01 - d23; + } + let mut total = 0u32; + for row in &v { + let mut hs = [0i16; H]; + let mut hd = [0i16; H]; + for k in 0..H { + hs[k] = row[2 * k] + row[2 * k + 1]; + hd[k] = row[2 * k] - row[2 * k + 1]; + } + for k in 0..H / 2 { + total += hs[2 * k].unsigned_abs().max(hs[2 * k + 1].unsigned_abs()) as u32; + total += hd[2 * k].unsigned_abs().max(hd[2 * k + 1].unsigned_abs()) as u32; + } + } + total +} + +/// [`satd_strip`] for 8 and 4 columns, where its pairwise sums across neighbouring lanes do not +/// vectorise: the same sums are formed lane by lane against copies of the rows shifted by one and +/// two lanes, and only every fourth lane (the first column of a 4x4 block) is counted. +#[inline(always)] +fn satd_strip_narrow(a: &[u8], sa: usize, b: &[u8], sb: usize) -> u32 { + let mut d = [[0i16; N]; 4]; + for r in 0..4 { + let ra = &a[r * sa..r * sa + N]; + let rb = &b[r * sb..r * sb + N]; + for i in 0..N { + d[r][i] = ra[i] as i16 - rb[i] as i16; + } + } + let mut v = [[0i16; N]; 4]; + for i in 0..N { + let s01 = d[0][i] + d[1][i]; + let d01 = d[0][i] - d[1][i]; + let s23 = d[2][i] + d[3][i]; + let d23 = d[2][i] - d[3][i]; + v[0][i] = s01 + s23; + v[1][i] = s01 - s23; + v[2][i] = d01 + d23; + v[3][i] = d01 - d23; + } + // per lane at most 4 rows x 2 x 2040: fits u16 + let mut acc = [0u16; N]; + for row in &v { + let mut p = [0i16; N]; + let mut m = [0i16; N]; + p[..N - 1].copy_from_slice(&row[1..]); + m[..N - 1].copy_from_slice(&row[1..]); + for i in 0..N { + p[i] += row[i]; + m[i] = row[i] - m[i]; + } + let mut p2 = [0i16; N]; + let mut m2 = [0i16; N]; + p2[..N - 2].copy_from_slice(&p[2..]); + m2[..N - 2].copy_from_slice(&m[2..]); + for i in 0..N { + let t = p[i].unsigned_abs().max(p2[i].unsigned_abs()) + m[i].unsigned_abs().max(m2[i].unsigned_abs()); + acc[i] += if i % 4 == 0 { t } else { 0 }; + } + } + acc.iter().map(|&x| x as u32).sum() +} + +/// Sum of absolute 4x4 Hadamard-transformed differences (halved), for w, h multiples of 4. +pub fn satd(a: &[u8], sa: usize, b: &[u8], sb: usize, w: usize, h: usize) -> u32 { + let mut total = 0; + for by in (0..h).step_by(4) { + let (ao, bo) = (by * sa, by * sb); + match w { + 16 => total += satd_strip::<16, 8>(&a[ao..], sa, &b[bo..], sb), + 8 => total += satd_strip_narrow::<8>(&a[ao..], sa, &b[bo..], sb), + 4 => total += satd_strip_narrow::<4>(&a[ao..], sa, &b[bo..], sb), + _ => { + for bx in (0..w).step_by(4) { + total += satd_strip_narrow::<4>(&a[ao + bx..], sa, &b[bo + bx..], sb); + } + } + } + } + total +} + +pub fn ssd(a: &[u8], sa: usize, b: &[u8], sb: usize, w: usize, h: usize) -> u64 { + let mut s = 0u64; + for r in 0..h { + let ra = &a[r * sa..r * sa + w]; + let rb = &b[r * sb..r * sb + w]; + s += ra.iter().zip(rb).map(|(&x, &y)| (x as i32 - y as i32).pow(2) as u32).sum::() as u64; + } + s +} + +/// Variance*N (sum of squares minus square of sum / N) of a block. +pub fn block_var(a: &[u8], sa: usize, w: usize, h: usize) -> u32 { + let mut s = 0u32; + let mut ss = 0u32; + for r in 0..h { + for &v in &a[r * sa..r * sa + w] { + s += v as u32; + ss += (v as u32) * (v as u32); + } + } + ss - ((s as u64 * s as u64) / (w * h) as u64) as u32 +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn satd_of_dc_difference() { + let a = [10u8; 16]; + let b = [0u8; 16]; + // DC-only difference of 10: hadamard DC = 160, halved = 80. + assert_eq!(satd(&a, 4, &b, 4, 4, 4), 80); + assert_eq!(sad(&a, 4, &b, 4, 4, 4), 160); + } + + #[test] + fn satd_matches_reference() { + let mut seed = 7u32; + let mut rnd = || { + seed = seed.wrapping_mul(1103515245).wrapping_add(12345); + (seed >> 16) as u8 + }; + let a: Vec = (0..16 * 16).map(|_| rnd()).collect(); + let b: Vec = (0..16 * 16).map(|_| rnd()).collect(); + for (w, h) in [(16, 16), (8, 8), (4, 4), (16, 8), (8, 16), (12, 4)] { + let mut want = 0; + for by in (0..h).step_by(4) { + for bx in (0..w).step_by(4) { + let mut d = [0i32; 16]; + for r in 0..4 { + for c in 0..4 { + d[r * 4 + c] = a[(by + r) * 16 + bx + c] as i32 - b[(by + r) * 16 + bx + c] as i32; + } + } + want += hadamard4x4_sum(&d); + } + } + assert_eq!(satd(&a, 16, &b, 16, w, h), want / 2, "{w}x{h}"); + } + } + + #[test] + fn satd_matches_reference_at_the_extremes() { + // the largest differences (every lane at +-255) are where 16-bit sums could overflow + let (zero, full) = ([0u8; 16 * 16], [255u8; 16 * 16]); + let stripes: Vec = (0..16 * 16).map(|i| if (i / 16 + i % 16) % 2 == 0 { 255 } else { 0 }).collect(); + for (a, b) in [(&zero[..], &full[..]), (&full[..], &zero[..]), (&stripes[..], &zero[..]), (&zero[..], &stripes[..])] { + for (w, h) in [(16, 16), (8, 8), (4, 4), (8, 16), (12, 4)] { + let mut want = 0; + for by in (0..h).step_by(4) { + for bx in (0..w).step_by(4) { + let mut d = [0i32; 16]; + for r in 0..4 { + for c in 0..4 { + d[r * 4 + c] = a[(by + r) * 16 + bx + c] as i32 - b[(by + r) * 16 + bx + c] as i32; + } + } + want += hadamard4x4_sum(&d); + } + } + assert_eq!(satd(a, 16, b, 16, w, h), want / 2, "{w}x{h}"); + } + } + } +} diff --git a/crates/h264enc/src/intra.rs b/crates/h264enc/src/intra.rs new file mode 100644 index 0000000000..cae6fe8102 --- /dev/null +++ b/crates/h264enc/src/intra.rs @@ -0,0 +1,303 @@ +//! Intra prediction (§8.3): 4x4, 8x8 (with reference filtering), 16x16 and chroma. + +/// Neighbouring samples of a block. `top` holds p[x,-1] for x = 0..2N (top then top-right), `left` p[-1,y]. +#[derive(Clone, Copy)] +pub struct Edge { + pub top: [u8; N2], + pub left: [u8; N], + pub tl: u8, + pub has_top: bool, + pub has_left: bool, + pub has_tl: bool, +} + +pub type Edge4 = Edge<8, 4>; +pub type Edge8 = Edge<16, 8>; + +/// Which modes are usable given neighbour availability. +pub fn mode_available(mode: u8, has_top: bool, has_left: bool, has_tl: bool) -> bool { + match mode { + 0 | 3 | 7 => has_top, + 1 | 8 => has_left, + 2 => true, + _ => has_top && has_left && has_tl, + } +} + +/// Generic directional prediction for an NxN block given (already filtered, for 8x8) neighbours. +/// `t(x)` = p[x,-1] for x in -1..2N, `l(y)` = p[-1,y] for y in -1..N. +fn pred_dir(mode: u8, t: &dyn Fn(i32) -> i32, l: &dyn Fn(i32) -> i32, dc: i32, out: &mut [u8]) { + let n = N as i32; + for y in 0..n { + for x in 0..n { + let v = match mode { + 0 => t(x), + 1 => l(y), + 2 => dc, + 3 => { + if x == n - 1 && y == n - 1 { + (t(2 * n - 2) + 3 * t(2 * n - 1) + 2) >> 2 + } else { + (t(x + y) + 2 * t(x + y + 1) + t(x + y + 2) + 2) >> 2 + } + } + 4 => { + if x > y { + (t(x - y - 2) + 2 * t(x - y - 1) + t(x - y) + 2) >> 2 + } else if x < y { + (l(y - x - 2) + 2 * l(y - x - 1) + l(y - x) + 2) >> 2 + } else { + (t(0) + 2 * t(-1) + l(0) + 2) >> 2 + } + } + 5 => { + let z = 2 * x - y; + if z >= 0 && z & 1 == 0 { + (t(x - (y >> 1) - 1) + t(x - (y >> 1)) + 1) >> 1 + } else if z >= 0 { + (t(x - (y >> 1) - 2) + 2 * t(x - (y >> 1) - 1) + t(x - (y >> 1)) + 2) >> 2 + } else if z == -1 { + (l(0) + 2 * l(-1) + t(0) + 2) >> 2 + } else { + (l(y - 2 * x - 1) + 2 * l(y - 2 * x - 2) + l(y - 2 * x - 3) + 2) >> 2 + } + } + 6 => { + let z = 2 * y - x; + if z >= 0 && z & 1 == 0 { + (l(y - (x >> 1) - 1) + l(y - (x >> 1)) + 1) >> 1 + } else if z >= 0 { + (l(y - (x >> 1) - 2) + 2 * l(y - (x >> 1) - 1) + l(y - (x >> 1)) + 2) >> 2 + } else if z == -1 { + (l(0) + 2 * l(-1) + t(0) + 2) >> 2 + } else { + (t(x - 2 * y - 1) + 2 * t(x - 2 * y - 2) + t(x - 2 * y - 3) + 2) >> 2 + } + } + 7 => { + if y & 1 == 0 { + (t(x + (y >> 1)) + t(x + (y >> 1) + 1) + 1) >> 1 + } else { + (t(x + (y >> 1)) + 2 * t(x + (y >> 1) + 1) + t(x + (y >> 1) + 2) + 2) >> 2 + } + } + _ => { + let z = x + 2 * y; + let lim = 2 * n - 3; + if z < lim && z & 1 == 0 { + (l(y + (x >> 1)) + l(y + (x >> 1) + 1) + 1) >> 1 + } else if z < lim { + (l(y + (x >> 1)) + 2 * l(y + (x >> 1) + 1) + l(y + (x >> 1) + 2) + 2) >> 2 + } else if z == lim { + (l(n - 2) + 3 * l(n - 1) + 2) >> 2 + } else { + l(n - 1) + } + } + }; + out[(y * n + x) as usize] = v as u8; + } + } +} + +/// Intra 4x4 prediction. `e.top[4..8]` must already be substituted with top[3] when top-right is unavailable. +pub fn pred4x4(mode: u8, e: &Edge4, out: &mut [u8; 16]) { + let dc = match (e.has_top, e.has_left) { + (true, true) => (e.top[..4].iter().map(|&v| v as i32).sum::() + e.left.iter().map(|&v| v as i32).sum::() + 4) >> 3, + (true, false) => (e.top[..4].iter().map(|&v| v as i32).sum::() + 2) >> 2, + (false, true) => (e.left.iter().map(|&v| v as i32).sum::() + 2) >> 2, + _ => 128, + }; + if mode == 2 { + out.fill(dc as u8); + return; + } + let t = |x: i32| if x < 0 { e.tl as i32 } else { e.top[x as usize] as i32 }; + let l = |y: i32| if y < 0 { e.tl as i32 } else { e.left[y as usize] as i32 }; + pred_dir::<4>(mode, &t, &l, dc, out); +} + +/// Reference sample filtering for Intra 8x8 (§8.3.2.2.1). `e.top[8..16]` must be substituted if unavailable. +pub fn filter8(e: &Edge8) -> Edge8 { + let mut f = *e; + if e.has_top { + let p = |x: usize| e.top[x] as u32; + f.top[0] = if e.has_tl { ((e.tl as u32 + 2 * p(0) + p(1) + 2) >> 2) as u8 } else { ((3 * p(0) + p(1) + 2) >> 2) as u8 }; + for x in 1..15 { + f.top[x] = ((p(x - 1) + 2 * p(x) + p(x + 1) + 2) >> 2) as u8; + } + f.top[15] = ((p(14) + 3 * p(15) + 2) >> 2) as u8; + } + if e.has_tl { + let tl = e.tl as u32; + f.tl = if !e.has_top || !e.has_left { + if e.has_top { + ((3 * tl + e.top[0] as u32 + 2) >> 2) as u8 + } else if e.has_left { + ((3 * tl + e.left[0] as u32 + 2) >> 2) as u8 + } else { + e.tl + } + } else { + ((e.top[0] as u32 + 2 * tl + e.left[0] as u32 + 2) >> 2) as u8 + }; + } + if e.has_left { + let p = |y: usize| e.left[y] as u32; + f.left[0] = if e.has_tl { ((e.tl as u32 + 2 * p(0) + p(1) + 2) >> 2) as u8 } else { ((3 * p(0) + p(1) + 2) >> 2) as u8 }; + for y in 1..7 { + f.left[y] = ((p(y - 1) + 2 * p(y) + p(y + 1) + 2) >> 2) as u8; + } + f.left[7] = ((p(6) + 3 * p(7) + 2) >> 2) as u8; + } + f +} + +/// Intra 8x8 prediction from *filtered* neighbours. +pub fn pred8x8(mode: u8, f: &Edge8, out: &mut [u8; 64]) { + let dc = match (f.has_top, f.has_left) { + (true, true) => (f.top[..8].iter().map(|&v| v as i32).sum::() + f.left.iter().map(|&v| v as i32).sum::() + 8) >> 4, + (true, false) => (f.top[..8].iter().map(|&v| v as i32).sum::() + 4) >> 3, + (false, true) => (f.left.iter().map(|&v| v as i32).sum::() + 4) >> 3, + _ => 128, + }; + if mode == 2 { + out.fill(dc as u8); + return; + } + let t = |x: i32| if x < 0 { f.tl as i32 } else { f.top[x as usize] as i32 }; + let l = |y: i32| if y < 0 { f.tl as i32 } else { f.left[y as usize] as i32 }; + pred_dir::<8>(mode, &t, &l, dc, out); +} + +/// Intra 16x16 prediction. Modes: 0 V, 1 H, 2 DC, 3 plane. +pub fn pred16x16(mode: u8, top: &[u8; 16], left: &[u8; 16], tl: u8, has_top: bool, has_left: bool, out: &mut [u8; 256]) { + match mode { + 0 => { + for y in 0..16 { + out[y * 16..y * 16 + 16].copy_from_slice(top); + } + } + 1 => { + for y in 0..16 { + out[y * 16..y * 16 + 16].fill(left[y]); + } + } + 2 => { + let st: u32 = top.iter().map(|&v| v as u32).sum(); + let sl: u32 = left.iter().map(|&v| v as u32).sum(); + let dc = match (has_top, has_left) { + (true, true) => (st + sl + 16) >> 5, + (true, false) => (st + 8) >> 4, + (false, true) => (sl + 8) >> 4, + _ => 128, + }; + out.fill(dc as u8); + } + _ => { + let p_t = |x: i32| if x < 0 { tl as i32 } else { top[x as usize] as i32 }; + let p_l = |y: i32| if y < 0 { tl as i32 } else { left[y as usize] as i32 }; + let mut h = 0; + let mut v = 0; + for i in 0..8 { + h += (i + 1) * (p_t(8 + i) - p_t(6 - i)); + v += (i + 1) * (p_l(8 + i) - p_l(6 - i)); + } + let a = 16 * (p_l(15) + p_t(15)); + let b = (5 * h + 32) >> 6; + let c = (5 * v + 32) >> 6; + for y in 0..16 { + for x in 0..16 { + out[y * 16 + x] = ((a + b * (x as i32 - 7) + c * (y as i32 - 7) + 16) >> 5).clamp(0, 255) as u8; + } + } + } + } +} + +/// Chroma 8x8 prediction (4:2:0). Modes: 0 DC, 1 H, 2 V, 3 plane. +pub fn pred_chroma(mode: u8, top: &[u8; 8], left: &[u8; 8], tl: u8, has_top: bool, has_left: bool, out: &mut [u8; 64]) { + match mode { + 0 => { + for by in 0..2 { + for bx in 0..2 { + let st: u32 = top[bx * 4..bx * 4 + 4].iter().map(|&v| v as u32).sum(); + let sl: u32 = left[by * 4..by * 4 + 4].iter().map(|&v| v as u32).sum(); + let dc = if (bx == 0 && by == 0) || (bx == 1 && by == 1) { + match (has_top, has_left) { + (true, true) => (st + sl + 4) >> 3, + (true, false) => (st + 2) >> 2, + (false, true) => (sl + 2) >> 2, + _ => 128, + } + } else if bx == 1 && by == 0 { + if has_top { + (st + 2) >> 2 + } else if has_left { + (sl + 2) >> 2 + } else { + 128 + } + } else if has_left { + (sl + 2) >> 2 + } else if has_top { + (st + 2) >> 2 + } else { + 128 + }; + for y in 0..4 { + for x in 0..4 { + out[(by * 4 + y) * 8 + bx * 4 + x] = dc as u8; + } + } + } + } + } + 1 => { + for y in 0..8 { + out[y * 8..y * 8 + 8].fill(left[y]); + } + } + 2 => { + for y in 0..8 { + out[y * 8..y * 8 + 8].copy_from_slice(top); + } + } + _ => { + let p_t = |x: i32| if x < 0 { tl as i32 } else { top[x as usize] as i32 }; + let p_l = |y: i32| if y < 0 { tl as i32 } else { left[y as usize] as i32 }; + let mut h = 0; + let mut v = 0; + for i in 0..4 { + h += (i + 1) * (p_t(4 + i) - p_t(2 - i)); + v += (i + 1) * (p_l(4 + i) - p_l(2 - i)); + } + let a = 16 * (p_l(7) + p_t(7)); + let b = (34 * h + 32) >> 6; + let c = (34 * v + 32) >> 6; + for y in 0..8 { + for x in 0..8 { + out[y * 8 + x] = ((a + b * (x as i32 - 3) + c * (y as i32 - 3) + 16) >> 5).clamp(0, 255) as u8; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dc_and_vertical() { + let e = Edge4 { top: [10, 20, 30, 40, 40, 40, 40, 40], left: [1, 2, 3, 4], tl: 5, has_top: true, has_left: true, has_tl: true }; + let mut o = [0u8; 16]; + pred4x4(0, &e, &mut o); + assert_eq!(&o[12..16], &[10, 20, 30, 40]); + pred4x4(2, &e, &mut o); + assert_eq!(o[0], ((100 + 10 + 4) >> 3) as u8); + // horizontal up saturates at bottom-left + pred4x4(8, &e, &mut o); + assert_eq!(o[15], 4); + } +} diff --git a/crates/h264enc/src/lib.rs b/crates/h264enc/src/lib.rs new file mode 100644 index 0000000000..0790acea0a --- /dev/null +++ b/crates/h264enc/src/lib.rs @@ -0,0 +1,1020 @@ +//! Clean-room, pure-Rust H.264/AVC encoder (ITU-T H.264), written from the public specification. +//! +//! - Profiles: Constrained Baseline (CAVLC, I/P), Main (CABAC, I/P/B) and High (CABAC, 8x8 transform, I/P/B), 4:2:0 8-bit. +//! - Intra 16x16 / 4x4 / 8x8 mode decision, hexagon motion search with quarter-sample refinement, +//! 16x16/16x8/8x16/8x8 partitions, P_Skip, B-frames with spatial direct/B_Skip, in-loop deblocking. +//! - Rate control: constant QP, constant quality (CRF-like), 1-pass ABR with VBV, 2-pass VBR. +//! - Slice-parallel encoding with rayon (feature `threads`, on by default). +//! +//! ```no_run +//! use craft_h264enc::{Encoder, EncoderConfig, RateControl, YuvFrame}; +//! let mut cfg = EncoderConfig::new(1280, 720, 30, 1); +//! cfg.rate = RateControl::Crf(20.0); +//! let mut enc = Encoder::new(cfg).unwrap(); +//! let (y, u, v) = (vec![128u8; 1280 * 720], vec![128u8; 640 * 360], vec![128u8; 640 * 360]); +//! let frame = YuvFrame { y: &y, u: &u, v: &v, y_stride: 1280, uv_stride: 640 }; +//! let mut packets = enc.encode(&frame, 0).unwrap(); +//! packets.extend(enc.flush()); +//! ``` + +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +mod cabac; +mod cabac_mb; +mod cavlc; +mod deblock; +mod dsp; +mod intra; +mod lookahead; +mod mbinfo; +mod nal; +mod picture; +mod ratecontrol; +mod slice; +mod syntax; +mod tables; +mod transform; + +use std::collections::VecDeque; + +pub use ratecontrol::{FrameStat, PassStats}; + +use cabac::CostTable; +use deblock::{BandPlane, DeblockParams, deblock_band}; +use lookahead::{Complexity, LowRes}; +use mbinfo::MbInfo; +use nal::{NAL_AUD, NAL_IDR, NAL_PPS, NAL_SEI, NAL_SLICE, NAL_SPS, Pps, SliceHeader, SliceType, Sps, Vui}; +use picture::{CHROMA_PAD, Frame, LUMA_PAD, RefPic, build_hpel}; +use ratecontrol::RcKind; +use slice::{Band, EncParams, FrameEnc, Lambdas, SliceEnc, SliceOut}; +use tables::NUM_CTX; +use transform::QuantTables; + +/// Encoder errors. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid dimensions {0}x{1} (must be even and non-zero)")] + InvalidDimensions(u32, u32), + #[error("invalid frame rate {0}/{1}")] + InvalidFrameRate(u32, u32), + #[error("invalid configuration: {0}")] + InvalidConfig(String), + #[error("input frame planes too small for the configured size")] + BadFrame, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum Profile { + /// Constrained Baseline: CAVLC, no B-frames. + Baseline, + /// Main: CABAC, B-frames. + Main, + /// High: CABAC, B-frames, 8x8 transform. + #[default] + High, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum Preset { + Speed, + #[default] + Balanced, + Quality, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum RateControl { + /// Constant quality; roughly the QP of P frames (0–51, lower is better). + Crf(f32), + /// Constant bitrate target (1-pass ABR with a 1-second VBV at `kbps`). + Cbr { kbps: u32 }, + /// Variable bitrate: average `target_kbps`, VBV-limited to `max_kbps`. Use [`Pass`] for 2-pass. + Vbr { target_kbps: u32, max_kbps: u32 }, + /// Constant QP (B frames use QP + 2). + Qp(u8), +} + +/// Multi-pass mode for bitrate-targeted rate control. +#[derive(Clone, Debug, Default, PartialEq)] +pub enum Pass { + #[default] + Single, + /// First pass: encode and collect [`PassStats`] via [`Encoder::pass_stats`]. + First, + /// Second pass using the statistics of the first. + Second(PassStats), +} + +/// Colour description written to the VUI (ITU-T H.273 code points). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ColorConfig { + pub primaries: u8, + pub transfer: u8, + pub matrix: u8, + pub full_range: bool, +} + +impl Default for ColorConfig { + /// BT.709, limited range. + fn default() -> Self { + ColorConfig { primaries: 1, transfer: 1, matrix: 1, full_range: false } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum PacketFormat { + /// Start-code prefixed NAL units (`00 00 00 01`), with AUD and in-band SPS/PPS on keyframes. + #[default] + AnnexB, + /// 4-byte big-endian length-prefixed NAL units (MP4 samples); SPS/PPS live in [`Encoder::avcc`]. + LengthPrefixed, +} + +#[derive(Clone, Debug)] +pub struct EncoderConfig { + pub width: u32, + pub height: u32, + pub fps_num: u32, + pub fps_den: u32, + pub profile: Profile, + pub preset: Preset, + pub rate: RateControl, + /// Maximum distance between IDR frames. + pub keyint: u32, + /// Consecutive B-frames (0–3). Ignored for Baseline. + pub bframes: u8, + pub color: ColorConfig, + /// Worker threads (0 = all available cores). Also bounds the number of slices. + pub threads: usize, + pub pass: Pass, + pub format: PacketFormat, + /// Variance adaptive quantisation strength (0 = off). + pub aq_strength: f32, + /// Sample aspect ratio (1:1 by default). + pub sar: (u16, u16), + /// Insert IDR frames at detected scene cuts. + pub scenecut: bool, + /// Number of slices per picture (0 = derived from `threads`). + pub slices: usize, + /// Access unit delimiters in Annex-B output. + pub aud: bool, + /// HDR static metadata sent in an SEI with the first access unit: the 24-byte SMPTE ST 2086 + /// mastering display payload (payloadType 137) … + pub mastering_display: Option<[u8; 24]>, + /// … and (MaxCLL, MaxFALL) content light level (payloadType 144). + pub content_light: Option<(u16, u16)>, + /// Requested level_idc (41 = level 4.1); None = the lowest level that fits. A level too low + /// for the stream is raised to the one it needs. + pub level: Option, +} + +impl EncoderConfig { + pub fn new(width: u32, height: u32, fps_num: u32, fps_den: u32) -> Self { + EncoderConfig { + width, + height, + fps_num, + fps_den, + profile: Profile::High, + preset: Preset::Balanced, + rate: RateControl::Crf(23.0), + keyint: 250, + bframes: 2, + color: ColorConfig::default(), + threads: 0, + pass: Pass::Single, + format: PacketFormat::AnnexB, + aq_strength: 0.6, + sar: (1, 1), + scenecut: true, + slices: 0, + mastering_display: None, + content_light: None, + aud: true, + level: None, + } + } +} + +/// A planar 4:2:0 8-bit input picture of the configured size. +#[derive(Clone, Copy, Debug)] +pub struct YuvFrame<'a> { + pub y: &'a [u8], + pub u: &'a [u8], + pub v: &'a [u8], + pub y_stride: usize, + pub uv_stride: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameType { + Idr, + I, + P, + B, +} + +/// One coded picture (access unit). +#[derive(Clone, Debug)] +pub struct Packet { + pub data: Vec, + pub pts: i64, + pub dts: i64, + pub keyframe: bool, + pub frame_type: FrameType, + /// QP used for the picture (slice QP). + pub qp: u8, +} + +/// A reconstructed (decoded) picture, available when [`Encoder::set_recon_capture`] is on. +#[derive(Clone, Debug)] +pub struct ReconFrame { + pub pts: i64, + pub y: Vec, + pub u: Vec, + pub v: Vec, +} + +struct Input { + frame: Frame, + pts: i64, + display: u64, + cplx: Complexity, + aq: Vec, +} + +struct Pool { + #[cfg(feature = "threads")] + pool: Option, +} + +impl Pool { + fn install(&self, f: impl FnOnce() -> R + Send) -> R { + #[cfg(feature = "threads")] + if let Some(p) = &self.pool { + return p.install(f); + } + f() + } +} + +pub struct Encoder { + cfg: EncoderConfig, + params: EncParams, + sps: Sps, + sps_nal: Vec, + pps_nal: Vec, + qt: QuantTables, + cost_tab: CostTable, + lam: Lambdas, + cabac_i: Box<[(i8, i8); NUM_CTX]>, + cabac_pb: Box<[(i8, i8); NUM_CTX]>, + rc: ratecontrol::RateControl, + record_pass1: bool, + pending: VecDeque, + prev_lowres: Option, + display_count: u64, + last_idr: u64, + idr_count: u32, + next_frame_num: u32, + dpb: Vec, + coded_count: u64, + display_pts: VecDeque, + dts_offset: Option, + /// First two presentation times (to derive the frame interval). + all_pts: Vec, + held: Vec, + frame_pool: Vec, + hpel_pool: Vec<[picture::Plane; 3]>, + sei_sent: bool, + slices: usize, + pool: Pool, + recon: Option>, + ref_uid: u32, + max_refs: usize, +} + +const LOG2_MAX_FRAME_NUM: u32 = 8; +const LOG2_MAX_POC_LSB: u32 = 8; + +/// The bitrate a level must allow for constant quality `crf`, which has no bitrate cap. It +/// assumes the worst footage: about 3 bits per pixel at CRF 0 (FilmCraft's CRF 0 exports of real +/// camera footage measured 1.7 on average and 2.4 in their busiest second), halving every 6 steps +/// (one doubling of the quantiser step). The default CRF 23 keeps 1080p30 at level 4. +fn crf_level_kbps(crf: f32, mbw: usize, mbh: usize, fps: f64) -> u32 { + let crf = if crf.is_finite() { f64::from(crf.clamp(0.0, 51.0)) } else { 23.0 }; + let bpp = 3.0 * (-crf / 6.0).exp2(); + let pixels = (mbw as f64) * (mbh as f64) * 256.0; + // a float-to-int `as` saturates, and NaN (a hostile fps) becomes 0 + (pixels * fps * bpp / 1000.0).ceil() as u32 +} + +impl Encoder { + /// The settings that shaped the stream, x264-style (`key=value` separated by spaces), for the + /// user_data_unregistered SEI: a file says how it was made (`strings`, `ffprobe -show_data`). + fn options(&self) -> String { + let c = &self.cfg; + let rate = match c.rate { + RateControl::Qp(q) => format!("rc=cqp qp={q}"), + RateControl::Crf(f) => format!("rc=crf crf={:.1}", f.clamp(0.0, 51.0)), + RateControl::Cbr { kbps } => format!("rc=cbr bitrate={kbps}"), + RateControl::Vbr { target_kbps, max_kbps } => format!("rc=vbr bitrate={target_kbps} vbv_maxrate={max_kbps}"), + }; + let pass = match c.pass { + Pass::Single => "", + Pass::First => " pass=1", + Pass::Second(_) => " pass=2", + }; + let profile = match c.profile { + Profile::Baseline => "baseline", + Profile::Main => "main", + Profile::High => "high", + }; + let preset = match c.preset { + Preset::Speed => "speed", + Preset::Balanced => "balanced", + Preset::Quality => "quality", + }; + let level = self.sps.level_idc; + format!( + "{rate}{pass} profile={profile} level={}.{} preset={preset} keyint={} bframes={} refs={} cabac={} 8x8dct={} slices={} aq={:.2} scenecut={}", + level / 10, + level % 10, + c.keyint, + c.bframes, + self.max_refs, + u8::from(c.profile != Profile::Baseline), + u8::from(c.profile == Profile::High), + self.slices, + c.aq_strength, + u8::from(c.scenecut), + ) + } + + pub fn new(mut cfg: EncoderConfig) -> Result { + if cfg.width == 0 + || cfg.height == 0 + || !cfg.width.is_multiple_of(2) + || !cfg.height.is_multiple_of(2) + || cfg.width > 16384 + || cfg.height > 16384 + { + return Err(Error::InvalidDimensions(cfg.width, cfg.height)); + } + if cfg.fps_num == 0 || cfg.fps_den == 0 { + return Err(Error::InvalidFrameRate(cfg.fps_num, cfg.fps_den)); + } + if cfg.bframes > 3 { + return Err(Error::InvalidConfig("bframes must be 0..=3".into())); + } + if cfg.profile == Profile::Baseline { + cfg.bframes = 0; + } + cfg.keyint = cfg.keyint.max(1); + let mbw = cfg.width.div_ceil(16) as usize; + let mbh = cfg.height.div_ceil(16) as usize; + let fps = cfg.fps_num as f64 / cfg.fps_den as f64; + let cabac = cfg.profile != Profile::Baseline; + let high = cfg.profile == Profile::High; + let params = match cfg.preset { + Preset::Speed => EncParams { + mbw, + mbh, + cabac, + t8x8: high, + i4x4: true, + i8x8: false, + partitions: false, + me_range: 8, + subpel: 2, + subpel_rounds: 0, + rd: false, + chroma_qp_offset: 0, + decimate: true, + adaptive_t8: false, + always_intra: false, + cavlc_clamp: !cabac, + }, + Preset::Balanced => EncParams { + mbw, + mbh, + cabac, + t8x8: high, + i4x4: true, + i8x8: high, + partitions: true, + me_range: 16, + subpel: 2, + subpel_rounds: 1, + rd: false, + chroma_qp_offset: 0, + decimate: true, + adaptive_t8: high, + always_intra: false, + cavlc_clamp: !cabac, + }, + Preset::Quality => EncParams { + mbw, + mbh, + cabac, + t8x8: high, + i4x4: true, + i8x8: high, + partitions: true, + me_range: 24, + subpel: 2, + subpel_rounds: 2, + rd: true, + chroma_qp_offset: 0, + decimate: true, + adaptive_t8: high, + always_intra: true, + cavlc_clamp: !cabac, + }, + }; + let max_refs = if cfg.bframes > 0 { 2 } else { 1 }; + let (kbps_for_level, rc_kind, vbv) = match (&cfg.rate, &cfg.pass) { + (RateControl::Qp(q), _) => (None, RcKind::Qp((*q).min(51)), (None, None)), + (RateControl::Crf(c), _) => (Some(crf_level_kbps(*c, mbw, mbh, fps)), RcKind::Crf(c.clamp(0.0, 51.0)), (None, None)), + (RateControl::Cbr { kbps }, Pass::Second(st)) => { + (Some(*kbps), ratecontrol::RateControl::plan_two_pass(st, *kbps), (Some(*kbps), Some(*kbps))) + } + (RateControl::Cbr { kbps }, Pass::First) => (Some(*kbps), RcKind::Abr { kbps: *kbps }, (None, None)), + (RateControl::Cbr { kbps }, Pass::Single) => (Some(*kbps), RcKind::Abr { kbps: *kbps }, (Some(*kbps), Some(*kbps))), + (RateControl::Vbr { target_kbps, max_kbps }, Pass::Second(st)) => { + let buffer = + max_kbps.checked_mul(2).ok_or_else(|| Error::InvalidConfig("VBR buffer rate exceeds the supported integer range".into()))?; + (Some(*max_kbps), ratecontrol::RateControl::plan_two_pass(st, *target_kbps), (Some(*max_kbps), Some(buffer))) + } + (RateControl::Vbr { target_kbps, max_kbps }, Pass::First) => (Some(*max_kbps), RcKind::Abr { kbps: *target_kbps }, (None, None)), + (RateControl::Vbr { target_kbps, max_kbps }, Pass::Single) => { + let buffer = + max_kbps.checked_mul(2).ok_or_else(|| Error::InvalidConfig("VBR buffer rate exceeds the supported integer range".into()))?; + (Some(*max_kbps), RcKind::Abr { kbps: *target_kbps }, (Some(*max_kbps), Some(buffer))) + } + }; + if matches!(cfg.rate, RateControl::Cbr { kbps: 0 } | RateControl::Vbr { target_kbps: 0, .. }) { + return Err(Error::InvalidConfig("bitrate must be non-zero".into())); + } + let auto = nal::pick_level(mbw as u32, mbh as u32, fps, max_refs as u32, kbps_for_level, high); + let level = cfg.level.map_or(auto, |l| nal::level_at_least(l, auto)); + let (profile_idc, constraint) = match cfg.profile { + Profile::Baseline => (66u8, 0xC0u8), // constraint_set0 + set1 => Constrained Baseline + Profile::Main => (77, 0x40), + Profile::High => (100, 0x00), + }; + let reorder = if cfg.bframes > 0 { 1 } else { 0 }; + let sps = Sps { + profile_idc, + constraint_flags: constraint, + level_idc: level, + width_mbs: mbw as u32, + height_mbs: mbh as u32, + crop_right: mbw as u32 * 16 - cfg.width, + crop_bottom: mbh as u32 * 16 - cfg.height, + log2_max_frame_num: LOG2_MAX_FRAME_NUM, + log2_max_poc_lsb: LOG2_MAX_POC_LSB, + max_num_ref_frames: max_refs as u32, + vui: Vui { + sar: cfg.sar, + video_full_range: cfg.color.full_range, + colour_primaries: cfg.color.primaries, + transfer: cfg.color.transfer, + matrix: cfg.color.matrix, + num_units_in_tick: cfg.fps_den, + time_scale: cfg.fps_num.saturating_mul(2), + max_num_reorder_frames: reorder, + max_dec_frame_buffering: max_refs as u32, + }, + }; + let pps = Pps { cabac, pic_init_qp: 26, chroma_qp_offset: params.chroma_qp_offset, transform_8x8: params.t8x8, high }; + let sps_nal = nal::nal(3, NAL_SPS, &sps.rbsp()); + let pps_nal = nal::nal(3, NAL_PPS, &pps.rbsp()); + let threads = if cfg.threads == 0 { std::thread::available_parallelism().map_or(1, |n| n.get()) } else { cfg.threads }; + let slices = if cfg.slices > 0 { cfg.slices.min(mbh) } else { threads.min(mbh.div_ceil(4)).max(1) }; + #[cfg(feature = "threads")] + let pool = Pool { pool: if threads > 1 { rayon::ThreadPoolBuilder::new().num_threads(threads).build().ok() } else { None } }; + #[cfg(not(feature = "threads"))] + let pool = Pool {}; + let record_pass1 = cfg.pass == Pass::First; + let rc = ratecontrol::RateControl::new(rc_kind, fps, vbv.0, vbv.1); + Ok(Encoder { + params, + sps, + sps_nal, + pps_nal, + qt: QuantTables::new(), + cost_tab: CostTable::new(), + lam: Lambdas::new(), + cabac_i: Box::new(tables::cabac_init_i()), + cabac_pb: Box::new(tables::cabac_init_pb0()), + rc, + record_pass1, + pending: VecDeque::new(), + prev_lowres: None, + display_count: 0, + last_idr: 0, + idr_count: 0, + next_frame_num: 0, + dpb: Vec::new(), + coded_count: 0, + display_pts: VecDeque::new(), + dts_offset: None, + all_pts: Vec::new(), + held: Vec::new(), + frame_pool: Vec::new(), + hpel_pool: Vec::new(), + sei_sent: false, + slices, + pool, + recon: None, + ref_uid: 0, + max_refs, + cfg, + }) + } + + pub fn config(&self) -> &EncoderConfig { + &self.cfg + } + + /// Run `f` on the encoder's worker threads (the calling thread without the `threads` + /// feature). The pool is idle between pictures, so input preparation such as colour + /// conversion that runs here does not queue behind work on the global rayon pool. + pub fn install(&self, f: impl FnOnce() -> R + Send) -> R { + self.pool.install(f) + } + + /// The SPS and PPS NAL units (without start codes). + pub fn sps_pps(&self) -> (Vec, Vec) { + (self.sps_nal.clone(), self.pps_nal.clone()) + } + + /// AVCDecoderConfigurationRecord (`avcC` box payload) for MP4 muxing (4-byte NAL lengths). + pub fn avcc(&self) -> Vec { + nal::avcc(&self.sps_nal, &self.pps_nal, self.sps.profile_idc, self.sps.constraint_flags, self.sps.level_idc) + } + + /// Number of frames the output is delayed by (B-frame reordering). + pub fn delay(&self) -> usize { + self.cfg.bframes as usize + } + + /// Enable capture of reconstructed pictures (for verification). + pub fn set_recon_capture(&mut self, on: bool) { + self.recon = if on { Some(Vec::new()) } else { None }; + } + + /// Take the reconstructed pictures captured so far (coding order). + pub fn take_recon(&mut self) -> Vec { + self.recon.as_mut().map(std::mem::take).unwrap_or_default() + } + + /// First-pass statistics (only when configured with [`Pass::First`]). + pub fn pass_stats(&self) -> Option { + if !self.record_pass1 { + return None; + } + Some(PassStats { frames: self.rc.pass1.clone(), fps: self.cfg.fps_num as f64 / self.cfg.fps_den as f64 }) + } + + /// Encode one picture. Returns zero or more packets in decoding order, or + /// [`Error::BadFrame`] if the planes are smaller than the configured size (same as + /// [`Encoder::try_encode`]). + pub fn encode(&mut self, frame: &YuvFrame, pts: i64) -> Result, Error> { + self.try_encode(frame, pts) + } + + pub fn try_encode(&mut self, frame: &YuvFrame, pts: i64) -> Result, Error> { + let src = self.import(frame)?; + let use_aq = self.cfg.aq_strength > 0.0 && !matches!(self.cfg.rate, RateControl::Qp(_)); + let (mbw, mbh, strength) = (self.params.mbw, self.params.mbh, self.cfg.aq_strength); + let prev = self.prev_lowres.take(); + let analyse = || { + let lowres = LowRes::new(&src); + let cplx = lookahead::complexity(&lowres, prev.as_ref()); + (lowres, cplx) + }; + let aq_of = || if use_aq { lookahead::aq_offsets(&src, mbw, mbh, strength) } else { Vec::new() }; + // the two analyses are independent: side by side, the pool's threads are not left idle + // at the end of each + #[cfg(feature = "threads")] + let ((lowres, cplx), aq) = self.pool.install(|| rayon::join(analyse, aq_of)); + #[cfg(not(feature = "threads"))] + let ((lowres, cplx), aq) = (analyse(), aq_of()); + self.prev_lowres = Some(lowres); + let display = self.display_count; + self.display_count += 1; + self.display_pts.push_back(pts); + if self.all_pts.len() < 2 { + self.all_pts.push(pts); + } + let input = Input { frame: src, pts, display, cplx, aq }; + let since = display - self.last_idr; + let fps = self.cfg.fps_num as f64 / self.cfg.fps_den as f64; + let min_gap = (fps / 2.0).max(1.0) as u64; + let scenecut = self.cfg.scenecut && display > 0 && since >= min_gap && cplx.inter as f64 > 0.75 * cplx.intra as f64; + let idr = display == 0 || since >= self.cfg.keyint as u64 || scenecut; + let mut out = Vec::new(); + if idr { + self.flush_minigop(&mut out); + self.last_idr = display; + self.encode_batch(vec![(input, SliceType::I, true)], &mut out); + } else { + self.pending.push_back(input); + if self.pending.len() > self.cfg.bframes as usize { + self.flush_minigop(&mut out); + } + } + Ok(self.release(out, false)) + } + + /// Encode all buffered pictures. + pub fn flush(&mut self) -> Vec { + let mut out = Vec::new(); + self.flush_minigop(&mut out); + self.release(out, true) + } + + /// Assign decoding timestamps. The k-th coded picture gets the k-th smallest presentation time minus the + /// reordering delay (one frame interval when B-frames are enabled); packets are held back until that interval is known. + fn release(&mut self, out: Vec, flushing: bool) -> Vec { + self.held.extend(out); + if self.dts_offset.is_none() { + if self.cfg.bframes == 0 { + self.dts_offset = Some(0); + } else if self.all_pts.len() >= 2 { + let mut v = self.all_pts.clone(); + v.sort_unstable(); + self.dts_offset = Some(v[1] - v[0]); + } else if flushing { + self.dts_offset = Some(0); + } else { + return Vec::new(); + } + } + let off = self.dts_offset.unwrap_or(0); + let mut res = std::mem::take(&mut self.held); + for p in res.iter_mut() { + p.dts -= off; + } + res + } + + fn flush_minigop(&mut self, out: &mut Vec) { + let Some(p) = self.pending.pop_back() else { return }; + let bs: Vec<(Input, SliceType, bool)> = self.pending.drain(..).map(|b| (b, SliceType::B, false)).collect(); + self.encode_batch(vec![(p, SliceType::P, false)], out); + if !bs.is_empty() { + // B-frames are non-reference pictures: encode them all concurrently. + self.encode_batch(bs, out); + } + } + + fn import(&mut self, f: &YuvFrame) -> Result { + let (w, h) = (self.cfg.width as usize, self.cfg.height as usize); + let (cw, ch) = (w / 2, h / 2); + if f.y_stride < w + || f.uv_stride < cw + || f.y.len() < f.y_stride * (h - 1) + w + || f.u.len() < f.uv_stride * (ch - 1) + cw + || f.v.len() < f.uv_stride * (ch - 1) + cw + { + return Err(Error::BadFrame); + } + let mut fr = self.new_frame(); + for (pl, src, stride, pw, ph) in + [(&mut fr.y, f.y, f.y_stride, w, h), (&mut fr.u, f.u, f.uv_stride, cw, ch), (&mut fr.v, f.v, f.uv_stride, cw, ch)] + { + let aw = pl.w; + for y in 0..pl.h { + let sy = y.min(ph - 1); + let o = pl.idx(0, y as isize); + pl.data[o..o + pw].copy_from_slice(&src[sy * stride..sy * stride + pw]); + let last = pl.data[o + pw - 1]; + pl.data[o + pw..o + aw].fill(last); + } + } + Ok(fr) + } + + fn new_frame(&mut self) -> Frame { + self.frame_pool.pop().unwrap_or_else(|| Frame::new(self.params.mbw * 16, self.params.mbh * 16)) + } + + fn recycle(&mut self, f: Frame) { + if self.frame_pool.len() < 8 { + self.frame_pool.push(f); + } + } + + /// Encode a batch of pictures that do not reference each other (a single I/P picture, or the B-frames of a + /// mini-GOP). Slices of all pictures in the batch are encoded in parallel. + fn encode_batch(&mut self, pics: Vec<(Input, SliceType, bool)>, out: &mut Vec) { + let mut jobs: Vec = Vec::with_capacity(pics.len()); + for (input, st, idr) in pics { + let job = self.plan_job(input, st, idr, jobs.len()); + jobs.push(job); + } + self.encode_slices(&mut jobs); + // ---- finalise each picture in coding order + for job in jobs { + self.finalise_job(job, out); + } + } + + /// Choose the final slice type and references of one picture, derive its quantiser and slice header, + /// and allocate its reconstruction buffers. `queued` is the number of pictures already planned in the batch. + fn plan_job(&mut self, input: Input, mut st: SliceType, idr: bool, queued: usize) -> Job { + if idr { + let old: Vec = std::mem::take(&mut self.dpb); + for r in old { + self.recycle_ref(r); + } + self.next_frame_num = 0; + } + let poc = 2 * (input.display as i64 - self.last_idr as i64) as i32; + let by_id = self.dpb.iter().enumerate().max_by_key(|(_, r)| r.id).map(|(i, _)| i); + let (mut l0, mut l1) = match st { + SliceType::I => (None, None), + SliceType::P => (by_id, None), + SliceType::B => ( + self.dpb.iter().enumerate().filter(|(_, r)| r.poc < poc).max_by_key(|(_, r)| r.poc).map(|(i, _)| i), + self.dpb.iter().enumerate().filter(|(_, r)| r.poc > poc).min_by_key(|(_, r)| r.poc).map(|(i, _)| i), + ), + }; + if st == SliceType::B && (l0.is_none() || l1.is_none()) { + st = SliceType::P; + l0 = by_id; + l1 = None; + } + if st == SliceType::P && l0.is_none() { + st = SliceType::I; + } + let is_ref = st != SliceType::B; + let cplx = match st { + SliceType::I => input.cplx.intra, + _ => input.cplx.inter, + } as f64; + let qpf = self.rc.frame_qp(st, cplx, self.coded_count as usize + queued); + let qp = qpf.round().clamp(0.0, 51.0) as u8; + let aq: Vec = if input.aq.is_empty() { + Vec::new() + } else { + input.aq.iter().map(|&o| ((qpf + o as f64).round().clamp(0.0, 51.0) as i32 - qp as i32) as i8).collect() + }; + let nal_ref_idc = if idr { + 3 + } else if is_ref { + 2 + } else { + 0 + }; + let hdr = SliceHeader { + first_mb: 0, + slice_type: st, + nal_ref_idc, + idr, + idr_pic_id: self.idr_count % 2, + frame_num: self.next_frame_num % (1 << LOG2_MAX_FRAME_NUM), + log2_max_frame_num: LOG2_MAX_FRAME_NUM, + poc_lsb: (poc as u32) % (1 << LOG2_MAX_POC_LSB), + log2_max_poc_lsb: LOG2_MAX_POC_LSB, + l0_modification: None, + cabac: self.params.cabac, + slice_qp: qp as i32, + pic_init_qp: 26, + disable_deblocking: if self.slices > 1 { 2 } else { 0 }, + alpha_offset_div2: 0, + beta_offset_div2: 0, + }; + let rec = self.new_frame(); + let mbs = vec![MbInfo::default(); self.params.mbw * self.params.mbh]; + Job { input, st, idr, qpf, qp, cplx, aq, hdr, nal_ref_idc, l0, l1, poc, rec: Some(rec), mbs, results: Vec::new() } + } + + /// Encode all slices of the batch's pictures in parallel (with deblocking of each slice band), storing + /// the results and reconstructions back into `jobs`. + fn encode_slices(&self, jobs: &mut [Job]) { + // ---- parallel slice encoding (+ deblocking of each slice band) + let (mbw, mbh) = (self.params.mbw, self.params.mbh); + let n = self.slices; + let bounds: Vec<(usize, usize)> = (0..n).map(|s| (s * mbh / n, (s + 1) * mbh / n)).filter(|(a, b)| b > a).collect(); + let capture = self.recon.is_some(); + let mut slice_results: Vec> = (0..jobs.len()).map(|_| Vec::new()).collect(); + // Every job carries its reconstruction buffer (`rec: Some(..)` above). + let mut bufs: Vec<(Frame, Vec)> = jobs.iter_mut().filter_map(|j| Some((j.rec.take()?, std::mem::take(&mut j.mbs)))).collect(); + { + let this = self; + let fencs: Vec = jobs + .iter() + .map(|j| FrameEnc { + p: &this.params, + qt: &this.qt, + cost_tab: &this.cost_tab, + st: j.hdr.slice_type, + src: &j.input.frame, + l0: j.l0.map(|i| &this.dpb[i]), + l1: j.l1.map(|i| &this.dpb[i]), + qp: j.qp, + aq: &j.aq, + hdr: j.hdr.clone(), + nal_ref_idc: j.nal_ref_idc, + nal_type: if j.idr { NAL_IDR } else { NAL_SLICE }, + cabac_table: if j.hdr.slice_type == SliceType::I { &this.cabac_i } else { &this.cabac_pb }, + lam: &this.lam, + }) + .collect(); + let dps: Vec = jobs + .iter() + .map(|j| DeblockParams { + alpha_offset: 0, + beta_offset: 0, + chroma_qp_offset: this.params.chroma_qp_offset, + ref_ids: [j.l0.map_or(-1, |i| this.dpb[i].id as i32), j.l1.map_or(-1, |i| this.dpb[i].id as i32)], + }) + .collect(); + let deblock: Vec = jobs.iter().map(|j| j.hdr.slice_type != SliceType::B || capture).collect(); + type Task<'t> = (usize, usize, usize, usize, &'t mut [u8], &'t mut [u8], &'t mut [u8], &'t mut [MbInfo]); + let mut tasks: Vec = Vec::new(); + for (pi, (rec, mbs)) in bufs.iter_mut().enumerate() { + let ys = rec.y.stride; + let cs = rec.u.stride; + let mut yrest = &mut rec.y.data[LUMA_PAD * ys..]; + let mut urest = &mut rec.u.data[CHROMA_PAD * cs..]; + let mut vrest = &mut rec.v.data[CHROMA_PAD * cs..]; + let mut mrest = &mut mbs[..]; + for (k, &(r0, r1)) in bounds.iter().enumerate() { + let rows = r1 - r0; + let (a, b) = std::mem::take(&mut yrest).split_at_mut(rows * 16 * ys); + yrest = b; + let (ua, ub) = std::mem::take(&mut urest).split_at_mut(rows * 8 * cs); + urest = ub; + let (va, vb) = std::mem::take(&mut vrest).split_at_mut(rows * 8 * cs); + vrest = vb; + let (ma, mb) = std::mem::take(&mut mrest).split_at_mut(rows * mbw); + mrest = mb; + tasks.push((pi, k, r0, r1, a, ua, va, ma)); + } + } + let ys = jobs_stride_y(mbw); + let cs = jobs_stride_c(mbw); + let run = |(pi, k, r0, r1, y, u, v, m): Task| -> (usize, usize, SliceOut) { + let out = SliceEnc::new(&fencs[pi], Band { y: &mut *y, u: &mut *u, v: &mut *v, ys, cs }, &mut *m, r0, r1, k as u16).encode(); + if deblock[pi] { + let mut by = BandPlane { data: y, stride: ys, pad: LUMA_PAD }; + let mut bu = BandPlane { data: u, stride: cs, pad: CHROMA_PAD }; + let mut bv = BandPlane { data: v, stride: cs, pad: CHROMA_PAD }; + deblock_band(&mut by, &mut bu, &mut bv, m, mbw, r1 - r0, &dps[pi]); + } + (pi, k, out) + }; + #[cfg(feature = "threads")] + let results: Vec<(usize, usize, SliceOut)> = { + use rayon::prelude::*; + if tasks.len() > 1 { this.pool.install(|| tasks.into_par_iter().map(run).collect()) } else { tasks.into_iter().map(run).collect() } + }; + #[cfg(not(feature = "threads"))] + let results: Vec<(usize, usize, SliceOut)> = tasks.into_iter().map(run).collect(); + let mut results = results; + results.sort_by_key(|r| (r.0, r.1)); + for (pi, _, r) in results { + slice_results[pi].push(r); + } + } + for ((j, (rec, mbs)), res) in jobs.iter_mut().zip(bufs).zip(slice_results) { + j.rec = Some(rec); + j.mbs = mbs; + j.results = res; + } + } + + /// Assemble the packet of one coded picture, update rate control and the reference state. + fn finalise_job(&mut self, job: Job, out: &mut Vec) { + let Job { input, st, idr, qpf, qp, cplx, hdr: _, nal_ref_idc: _, l0: _, l1: _, poc, rec, mbs, results, aq: _ } = job; + let Some(mut rec) = rec else { return }; + let is_ref = st != SliceType::B; + let annexb = self.cfg.format == PacketFormat::AnnexB; + let mut nals: Vec> = Vec::new(); + if annexb && self.cfg.aud { + let t = match st { + SliceType::I => 0, + SliceType::P => 1, + SliceType::B => 2, + }; + nals.push(nal::nal(0, NAL_AUD, &nal::aud_rbsp(t))); + } + if idr && annexb { + nals.push(self.sps_nal.clone()); + nals.push(self.pps_nal.clone()); + } + if !self.sei_sent { + self.sei_sent = true; + let text = format!("filmcraft-h264enc {} - clean-room H.264 encoder - options: {}", env!("CARGO_PKG_VERSION"), self.options()); + nals.push(nal::nal(0, NAL_SEI, &nal::sei_user_data_rbsp(&text))); + if self.cfg.mastering_display.is_some() || self.cfg.content_light.is_some() { + nals.push(nal::nal(0, NAL_SEI, &nal::sei_hdr_rbsp(self.cfg.mastering_display.as_ref(), self.cfg.content_light))); + } + } + for r in results { + nals.push(r.nal); + } + let mut data = Vec::with_capacity(nals.iter().map(|n| n.len() + 4).sum()); + for n in &nals { + if annexb { + data.extend_from_slice(&[0, 0, 0, 1]); + } else { + data.extend_from_slice(&(n.len() as u32).to_be_bytes()); + } + data.extend_from_slice(n); + } + let total_bits = data.len() as u64 * 8; + self.rc.update(st, qpf, cplx, total_bits); + if self.record_pass1 { + self.rc.record_pass1(st, qpf, cplx, total_bits); + } + if let Some(rv) = self.recon.as_mut() { + let (w, h) = (self.cfg.width as usize, self.cfg.height as usize); + let mut y = Vec::with_capacity(w * h); + let mut u = Vec::with_capacity(w * h / 4); + let mut v = Vec::with_capacity(w * h / 4); + rec.y.copy_out(w, h, &mut y); + rec.u.copy_out(w / 2, h / 2, &mut u); + rec.v.copy_out(w / 2, h / 2, &mut v); + rv.push(ReconFrame { pts: input.pts, y, u, v }); + } + if is_ref { + let hp = self.hpel_pool.pop(); + let hpel = self.pool.install(|| { + rec.extend_edges(); + build_hpel(&rec.y, hp) + }); + self.ref_uid += 1; + let rp = RefPic { frame: rec, hpel, mbs, poc, id: self.ref_uid }; + if self.dpb.len() >= self.max_refs { + // sliding window: drop the oldest + if let Some(oldest) = self.dpb.iter().enumerate().min_by_key(|(_, r)| r.id).map(|(i, _)| i) { + let r = self.dpb.remove(oldest); + self.recycle_ref(r); + } + } + self.dpb.push(rp); + self.next_frame_num = (self.next_frame_num + 1) % (1 << LOG2_MAX_FRAME_NUM); + } else { + self.recycle(rec); + } + if idr { + self.idr_count += 1; + } + // decoding timestamp base: the k-th smallest presentation time (offset applied in `release`) + // One display time is queued per input picture, so the queue is never empty here. + let pos = self.display_pts.iter().enumerate().min_by_key(|(_, p)| **p).map(|(i, _)| i); + let dts_base = pos.and_then(|p| self.display_pts.remove(p)).unwrap_or(input.pts); + self.coded_count += 1; + let frame_type = if idr { + FrameType::Idr + } else { + match st { + SliceType::I => FrameType::I, + SliceType::P => FrameType::P, + SliceType::B => FrameType::B, + } + }; + self.recycle(input.frame); + out.push(Packet { data, pts: input.pts, dts: dts_base, keyframe: idr, frame_type, qp }); + } + + fn recycle_ref(&mut self, r: RefPic) { + self.recycle(r.frame); + if self.hpel_pool.len() < 4 { + self.hpel_pool.push(r.hpel); + } + } +} + +/// One picture of a batch while it is planned, sliced and finalised. +struct Job { + input: Input, + st: SliceType, + idr: bool, + qpf: f64, + qp: u8, + cplx: f64, + aq: Vec, + hdr: SliceHeader, + nal_ref_idc: u8, + l0: Option, + l1: Option, + poc: i32, + rec: Option, + mbs: Vec, + results: Vec, +} + +fn jobs_stride_y(mbw: usize) -> usize { + mbw * 16 + 2 * LUMA_PAD +} + +fn jobs_stride_c(mbw: usize) -> usize { + mbw * 8 + 2 * CHROMA_PAD +} diff --git a/crates/h264enc/src/lookahead.rs b/crates/h264enc/src/lookahead.rs new file mode 100644 index 0000000000..c2140b9ac9 --- /dev/null +++ b/crates/h264enc/src/lookahead.rs @@ -0,0 +1,156 @@ +//! Cheap half-resolution analysis used for rate control, scene-cut detection and adaptive quantisation. + +use crate::dsp::{block_var, sad, satd}; +use crate::picture::Frame; + +/// Half-resolution luma (2x2 box filtered) of the macroblock-aligned picture. +pub struct LowRes { + pub w: usize, + pub h: usize, + pub data: Vec, +} + +impl LowRes { + pub fn new(f: &Frame) -> Self { + let (w, h) = (f.y.w / 2, f.y.h / 2); + let mut data = vec![0u8; w * h]; + let row = |y: usize, out: &mut [u8]| { + let r0 = f.y.idx(0, (2 * y) as isize); + let r1 = f.y.idx(0, (2 * y + 1) as isize); + for x in 0..w { + let s = f.y.data[r0 + 2 * x] as u32 + f.y.data[r0 + 2 * x + 1] as u32 + f.y.data[r1 + 2 * x] as u32 + f.y.data[r1 + 2 * x + 1] as u32; + out[x] = ((s + 2) >> 2) as u8; + } + }; + #[cfg(feature = "threads")] + { + use rayon::prelude::*; + data.par_chunks_mut(w).enumerate().for_each(|(y, r)| row(y, r)); + } + #[cfg(not(feature = "threads"))] + for (y, r) in data.chunks_mut(w).enumerate() { + row(y, r); + } + LowRes { w, h, data } + } +} + +/// Per-frame complexity estimates (sum over 8x8 low-res blocks, i.e. per macroblock). +#[derive(Clone, Copy, Debug, Default)] +pub struct Complexity { + pub intra: u64, + /// Inter cost against the previous frame (min with intra per block); equals `intra` when there is no previous frame. + pub inter: u64, +} + +fn block_rows (u64, u64) + Sync + Send>(rows: usize, f: F) -> (u64, u64) { + #[cfg(feature = "threads")] + { + use rayon::prelude::*; + (0..rows).into_par_iter().map(f).reduce(|| (0, 0), |a, b| (a.0 + b.0, a.1 + b.1)) + } + #[cfg(not(feature = "threads"))] + { + (0..rows).map(f).fold((0, 0), |a, b| (a.0 + b.0, a.1 + b.1)) + } +} + +pub fn complexity(cur: &LowRes, prev: Option<&LowRes>) -> Complexity { + let bw = cur.w / 8; + let bh = cur.h / 8; + let (intra, inter) = block_rows(bh, |by| { + let mut ci = 0u64; + let mut cp = 0u64; + let mut blk = [0u8; 64]; + for bx in 0..bw { + let (x0, y0) = (bx * 8, by * 8); + let base = y0 * cur.w + x0; + // intra: best of DC / H / V predictions from source neighbours + let mut best = u32::MAX; + for mode in 0..3 { + if (mode == 1 && x0 == 0) || (mode == 2 && y0 == 0) { + continue; + } + for y in 0..8 { + for x in 0..8 { + blk[y * 8 + x] = match mode { + 0 => { + if x0 > 0 && y0 > 0 { + ((cur.data[(y0 - 1) * cur.w + x0 + x] as u32 + cur.data[(y0 + y) * cur.w + x0 - 1] as u32) / 2) as u8 + } else { + 128 + } + } + 1 => cur.data[(y0 + y) * cur.w + x0 - 1], + _ => cur.data[(y0 - 1) * cur.w + x0 + x], + }; + } + } + best = best.min(satd(&cur.data[base..], cur.w, &blk, 8, 8, 8)); + } + let intra_c = best as u64 + 16; + ci += intra_c; + if let Some(p) = prev { + // small diamond search on SAD, then SATD of the best + let mut bmv = (0i32, 0i32); + let cost_at = |mx: i32, my: i32| -> u32 { + let x = x0 as i32 + mx; + let y = y0 as i32 + my; + if x < 0 || y < 0 || x + 8 > p.w as i32 || y + 8 > p.h as i32 { + return u32::MAX; + } + sad(&cur.data[base..], cur.w, &p.data[y as usize * p.w + x as usize..], p.w, 8, 8) + }; + let mut bc = cost_at(0, 0); + for _ in 0..8 { + let c = bmv; + for (dx, dy) in [(-1, 0), (1, 0), (0, -1), (0, 1), (-2, 0), (2, 0), (0, -2), (0, 2)] { + let k = cost_at(c.0 + dx, c.1 + dy); + if k < bc { + bc = k; + bmv = (c.0 + dx, c.1 + dy); + } + } + if bmv == c { + break; + } + } + let x = (x0 as i32 + bmv.0) as usize; + let y = (y0 as i32 + bmv.1) as usize; + let inter_c = satd(&cur.data[base..], cur.w, &p.data[y * p.w + x..], p.w, 8, 8) as u64 + 8; + cp += inter_c.min(intra_c); + } else { + cp += intra_c; + } + } + (ci, cp) + }); + Complexity { intra, inter } +} + +/// Variance-based adaptive quantisation offsets per macroblock (QP units, mean ~0). +pub fn aq_offsets(f: &Frame, mbw: usize, mbh: usize, strength: f32) -> Vec { + let mut e = vec![0f32; mbw * mbh]; + let row = |my: usize, e: &mut [f32]| { + for mx in 0..mbw { + let i = f.y.idx((mx * 16) as isize, (my * 16) as isize); + let vy = block_var(&f.y.data[i..], f.y.stride, 16, 16) as f32 / 256.0; + let iu = f.u.idx((mx * 8) as isize, (my * 8) as isize); + let vu = block_var(&f.u.data[iu..], f.u.stride, 8, 8) as f32 / 64.0; + let iv = f.v.idx((mx * 8) as isize, (my * 8) as isize); + let vv = block_var(&f.v.data[iv..], f.v.stride, 8, 8) as f32 / 64.0; + e[mx] = (vy + 0.5 * (vu + vv) + 1.0).log2(); + } + }; + #[cfg(feature = "threads")] + { + use rayon::prelude::*; + e.par_chunks_mut(mbw).enumerate().for_each(|(my, r)| row(my, r)); + } + #[cfg(not(feature = "threads"))] + for (my, r) in e.chunks_mut(mbw).enumerate() { + row(my, r); + } + let mean = e.iter().sum::() / e.len() as f32; + e.iter().map(|&v| (strength * (v - mean)).clamp(-8.0, 8.0)).collect() +} diff --git a/crates/h264enc/src/mbinfo.rs b/crates/h264enc/src/mbinfo.rs new file mode 100644 index 0000000000..99264d9ef0 --- /dev/null +++ b/crates/h264enc/src/mbinfo.rs @@ -0,0 +1,126 @@ +//! Per-macroblock state kept for neighbour context derivation, deblocking and co-located (direct) prediction. + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default, Hash)] +pub struct Mv { + pub x: i16, + pub y: i16, +} + +impl Mv { + pub const ZERO: Mv = Mv { x: 0, y: 0 }; + #[inline] + pub const fn new(x: i32, y: i32) -> Mv { + Mv { x: x as i16, y: y as i16 } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub enum MbKind { + #[default] + None, + I4x4, + I8x8, + I16x16, + PSkip, + PInter, + BSkip, + BDirect, + BInter, +} + +impl MbKind { + #[inline] + pub fn is_intra(self) -> bool { + matches!(self, MbKind::I4x4 | MbKind::I8x8 | MbKind::I16x16) + } + #[inline] + pub fn is_skip(self) -> bool { + matches!(self, MbKind::PSkip | MbKind::BSkip) + } + #[inline] + pub fn is_inxn(self) -> bool { + matches!(self, MbKind::I4x4 | MbKind::I8x8) + } +} + +/// Partition shapes for inter macroblocks. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub enum Part { + #[default] + P16x16, + P16x8, + P8x16, + P8x8, +} + +impl Part { + pub fn count(self) -> usize { + match self { + Part::P16x16 => 1, + Part::P16x8 | Part::P8x16 => 2, + Part::P8x8 => 4, + } + } + /// (x, y, w, h) in pixels of partition `i`. + pub fn rect(self, i: usize) -> (usize, usize, usize, usize) { + match self { + Part::P16x16 => (0, 0, 16, 16), + Part::P16x8 => (0, 8 * i, 16, 8), + Part::P8x16 => (8 * i, 0, 8, 16), + Part::P8x8 => (8 * (i & 1), 8 * (i >> 1), 8, 8), + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct MbInfo { + pub kind: MbKind, + /// Slice number + 1 (0 = not yet coded). + pub slice: u16, + pub qp: u8, + /// Luma cbp in bits 0..3, chroma cbp in bits 4..5. + pub cbp: u8, + pub t8x8: bool, + pub chroma_mode: u8, + /// Non-zero coefficient counts per luma 4x4 (raster y*4+x). + pub nnz: [u8; 16], + /// Chroma AC non-zero counts: Cb raster 2x2 then Cr raster 2x2. + pub nnz_c: [u8; 8], + /// bit0 luma DC (Intra16x16), bit1 Cb DC, bit2 Cr DC. + pub dc_cbf: u8, + /// Intra 4x4/8x8 prediction modes per 4x4 (raster), -1 when the MB is not I_NxN. + pub imodes: [i8; 16], + /// Reference index per 8x8 (raster) per list; -1 = list unused. + pub ref_idx: [[i8; 4]; 2], + pub mv: [[Mv; 16]; 2], + /// |mvd| components per 4x4 per list, clamped to 127. + pub mvd: [[[u8; 2]; 16]; 2], +} + +impl Default for MbInfo { + fn default() -> Self { + MbInfo { + kind: MbKind::None, + slice: 0, + qp: 0, + cbp: 0, + t8x8: false, + chroma_mode: 0, + nnz: [0; 16], + nnz_c: [0; 8], + dc_cbf: 0, + imodes: [-1; 16], + ref_idx: [[-1; 4]; 2], + mv: [[Mv::ZERO; 16]; 2], + mvd: [[[0; 2]; 16]; 2], + } + } +} + +impl MbInfo { + #[inline] + pub fn ref_at(&self, list: usize, blk4_raster: usize) -> i8 { + let (x, y) = (blk4_raster & 3, blk4_raster >> 2); + self.ref_idx[list][(y >> 1) * 2 + (x >> 1)] + } +} diff --git a/crates/h264enc/src/nal.rs b/crates/h264enc/src/nal.rs new file mode 100644 index 0000000000..dc753416a3 --- /dev/null +++ b/crates/h264enc/src/nal.rs @@ -0,0 +1,337 @@ +//! NAL unit and parameter-set/slice-header generation (§7.3), SEI and the AVCDecoderConfigurationRecord. + +use filmcraft_bitstream::{BitWriter, escape_rbsp}; + +pub const NAL_SLICE: u8 = 1; +pub const NAL_IDR: u8 = 5; +pub const NAL_SEI: u8 = 6; +pub const NAL_SPS: u8 = 7; +pub const NAL_PPS: u8 = 8; +pub const NAL_AUD: u8 = 9; + +/// Build a NAL unit (header + escaped RBSP), without start code. +pub fn nal(nal_ref_idc: u8, nal_type: u8, rbsp: &[u8]) -> Vec { + let mut out = Vec::with_capacity(rbsp.len() + rbsp.len() / 64 + 2); + out.push((nal_ref_idc << 5) | nal_type); + out.extend_from_slice(&escape_rbsp(rbsp)); + out +} + +#[derive(Clone, Debug)] +pub struct Vui { + pub sar: (u16, u16), + pub video_full_range: bool, + pub colour_primaries: u8, + pub transfer: u8, + pub matrix: u8, + pub num_units_in_tick: u32, + pub time_scale: u32, + pub max_num_reorder_frames: u32, + pub max_dec_frame_buffering: u32, +} + +#[derive(Clone, Debug)] +pub struct Sps { + pub profile_idc: u8, + pub constraint_flags: u8, + pub level_idc: u8, + pub width_mbs: u32, + pub height_mbs: u32, + pub crop_right: u32, + pub crop_bottom: u32, + pub log2_max_frame_num: u32, + pub log2_max_poc_lsb: u32, + pub max_num_ref_frames: u32, + pub vui: Vui, +} + +impl Sps { + pub fn rbsp(&self) -> Vec { + let mut w = BitWriter::new(); + w.write_bits(self.profile_idc as u32, 8); + w.write_bits(self.constraint_flags as u32, 8); + w.write_bits(self.level_idc as u32, 8); + w.write_ue(0); // seq_parameter_set_id + if matches!(self.profile_idc, 100 | 110 | 122 | 244 | 44 | 83 | 86 | 118 | 128 | 138 | 139 | 134 | 135) { + w.write_ue(1); // chroma_format_idc 4:2:0 + w.write_ue(0); // bit_depth_luma_minus8 + w.write_ue(0); // bit_depth_chroma_minus8 + w.write_bit(false); // qpprime_y_zero_transform_bypass_flag + w.write_bit(false); // seq_scaling_matrix_present_flag + } + w.write_ue(self.log2_max_frame_num - 4); + w.write_ue(0); // pic_order_cnt_type + w.write_ue(self.log2_max_poc_lsb - 4); + w.write_ue(self.max_num_ref_frames); + w.write_bit(false); // gaps_in_frame_num_value_allowed_flag + w.write_ue(self.width_mbs - 1); + w.write_ue(self.height_mbs - 1); + w.write_bit(true); // frame_mbs_only_flag + w.write_bit(true); // direct_8x8_inference_flag + let crop = self.crop_right > 0 || self.crop_bottom > 0; + w.write_bit(crop); + if crop { + w.write_ue(0); + w.write_ue(self.crop_right / 2); + w.write_ue(0); + w.write_ue(self.crop_bottom / 2); + } + w.write_bit(true); // vui_parameters_present_flag + let v = &self.vui; + let has_sar = v.sar != (0, 0); + w.write_bit(has_sar); + if has_sar { + if v.sar == (1, 1) { + w.write_bits(1, 8); + } else { + w.write_bits(255, 8); // Extended_SAR + w.write_bits(v.sar.0 as u32, 16); + w.write_bits(v.sar.1 as u32, 16); + } + } + w.write_bit(false); // overscan_info_present_flag + w.write_bit(true); // video_signal_type_present_flag + w.write_bits(5, 3); // video_format: unspecified + w.write_bit(v.video_full_range); + w.write_bit(true); // colour_description_present_flag + w.write_bits(v.colour_primaries as u32, 8); + w.write_bits(v.transfer as u32, 8); + w.write_bits(v.matrix as u32, 8); + w.write_bit(false); // chroma_loc_info_present_flag + w.write_bit(true); // timing_info_present_flag + w.write_bits(v.num_units_in_tick, 32); + w.write_bits(v.time_scale, 32); + w.write_bit(true); // fixed_frame_rate_flag + w.write_bit(false); // nal_hrd_parameters_present_flag + w.write_bit(false); // vcl_hrd_parameters_present_flag + w.write_bit(false); // pic_struct_present_flag + w.write_bit(true); // bitstream_restriction_flag + w.write_bit(true); // motion_vectors_over_pic_boundaries_flag + w.write_ue(0); // max_bytes_per_pic_denom + w.write_ue(0); // max_bits_per_mb_denom + w.write_ue(11); // log2_max_mv_length_horizontal (±512 samples) + w.write_ue(11); // log2_max_mv_length_vertical + w.write_ue(v.max_num_reorder_frames); + w.write_ue(v.max_dec_frame_buffering); + w.rbsp_trailing(); + w.finish() + } +} + +#[derive(Clone, Debug)] +pub struct Pps { + pub cabac: bool, + pub pic_init_qp: i32, + pub chroma_qp_offset: i32, + pub transform_8x8: bool, + pub high: bool, +} + +impl Pps { + pub fn rbsp(&self) -> Vec { + let mut w = BitWriter::new(); + w.write_ue(0); // pic_parameter_set_id + w.write_ue(0); // seq_parameter_set_id + w.write_bit(self.cabac); + w.write_bit(false); // bottom_field_pic_order_in_frame_present_flag + w.write_ue(0); // num_slice_groups_minus1 + w.write_ue(0); // num_ref_idx_l0_default_active_minus1 + w.write_ue(0); // num_ref_idx_l1_default_active_minus1 + w.write_bit(false); // weighted_pred_flag + w.write_bits(0, 2); // weighted_bipred_idc + w.write_se(self.pic_init_qp - 26); + w.write_se(0); // pic_init_qs_minus26 + w.write_se(self.chroma_qp_offset); + w.write_bit(true); // deblocking_filter_control_present_flag + w.write_bit(false); // constrained_intra_pred_flag + w.write_bit(false); // redundant_pic_cnt_present_flag + if self.high { + w.write_bit(self.transform_8x8); + w.write_bit(false); // pic_scaling_matrix_present_flag + w.write_se(self.chroma_qp_offset); // second_chroma_qp_index_offset + } + w.rbsp_trailing(); + w.finish() + } +} + +/// Access unit delimiter RBSP. `primary_pic_type`: 0 = I, 1 = I/P, 2 = I/P/B. +pub fn aud_rbsp(primary_pic_type: u8) -> Vec { + let mut w = BitWriter::new(); + w.write_bits(primary_pic_type as u32, 3); + w.rbsp_trailing(); + w.finish() +} + +/// user_data_unregistered SEI carrying an encoder identification string. +pub fn sei_user_data_rbsp(text: &str) -> Vec { + const UUID: [u8; 16] = [0x7a, 0x3c, 0x1f, 0x52, 0xd4, 0x0b, 0x4e, 0x8e, 0x9d, 0x61, 0x2f, 0xc4, 0x93, 0x05, 0xb7, 0x1a]; + let mut payload = UUID.to_vec(); + payload.extend_from_slice(text.as_bytes()); + let mut w = BitWriter::new(); + w.write_bits(5, 8); // payloadType user_data_unregistered + let mut size = payload.len(); + while size >= 255 { + w.write_bits(255, 8); + size -= 255; + } + w.write_bits(size as u32, 8); + w.write_bytes(&payload); + w.rbsp_trailing(); + w.finish() +} + +/// SEI with HDR static metadata: mastering_display_colour_volume (payloadType 137, the 24-byte +/// ST 2086 payload) and/or content_light_level_info (payloadType 144), H.264 Annex D. +pub fn sei_hdr_rbsp(mastering: Option<&[u8; 24]>, cll: Option<(u16, u16)>) -> Vec { + let mut w = BitWriter::new(); + if let Some(m) = mastering { + w.write_bits(137, 8); + w.write_bits(24, 8); + w.write_bytes(m); + } + if let Some((a, b)) = cll { + w.write_bits(144, 8); + w.write_bits(4, 8); + w.write_bits(a as u32, 16); + w.write_bits(b as u32, 16); + } + w.rbsp_trailing(); + w.finish() +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum SliceType { + P = 0, + B = 1, + I = 2, +} + +#[derive(Clone, Debug)] +pub struct SliceHeader { + pub first_mb: u32, + pub slice_type: SliceType, + pub nal_ref_idc: u8, + pub idr: bool, + pub idr_pic_id: u32, + pub frame_num: u32, + pub log2_max_frame_num: u32, + pub poc_lsb: u32, + pub log2_max_poc_lsb: u32, + /// Reorder RefPicList0 so that this picNum difference (abs_diff_pic_num_minus1, subtract) comes first. + pub l0_modification: Option, + pub cabac: bool, + pub slice_qp: i32, + pub pic_init_qp: i32, + pub disable_deblocking: u32, + pub alpha_offset_div2: i32, + pub beta_offset_div2: i32, +} + +impl SliceHeader { + /// Write the slice header into `w`; for CABAC the caller must byte-align with one-bits afterwards. + pub fn write(&self, w: &mut BitWriter) { + w.write_ue(self.first_mb); + w.write_ue(self.slice_type as u32); + w.write_ue(0); // pic_parameter_set_id + w.write_bits(self.frame_num, self.log2_max_frame_num); + if self.idr { + w.write_ue(self.idr_pic_id); + } + w.write_bits(self.poc_lsb, self.log2_max_poc_lsb); + if self.slice_type == SliceType::B { + w.write_bit(true); // direct_spatial_mv_pred_flag + } + if self.slice_type != SliceType::I { + w.write_bit(false); // num_ref_idx_active_override_flag + // ref_pic_list_modification + match self.l0_modification { + Some(d) => { + w.write_bit(true); + w.write_ue(0); // modification_of_pic_nums_idc: subtract + w.write_ue(d); + w.write_ue(3); + } + None => w.write_bit(false), + } + if self.slice_type == SliceType::B { + w.write_bit(false); + } + } + if self.nal_ref_idc != 0 { + if self.idr { + w.write_bit(false); // no_output_of_prior_pics_flag + w.write_bit(false); // long_term_reference_flag + } else { + w.write_bit(false); // adaptive_ref_pic_marking_mode_flag + } + } + if self.cabac && self.slice_type != SliceType::I { + w.write_ue(0); // cabac_init_idc + } + w.write_se(self.slice_qp - self.pic_init_qp); + w.write_ue(self.disable_deblocking); + if self.disable_deblocking != 1 { + w.write_se(self.alpha_offset_div2); + w.write_se(self.beta_offset_div2); + } + } +} + +/// AVCDecoderConfigurationRecord (ISO/IEC 14496-15 §5.3.3.1) with 4-byte NAL lengths. +pub fn avcc(sps_nal: &[u8], pps_nal: &[u8], profile_idc: u8, constraint: u8, level: u8) -> Vec { + let mut v = vec![1, profile_idc, constraint, level, 0xFF, 0xE1]; + v.extend_from_slice(&(sps_nal.len() as u16).to_be_bytes()); + v.extend_from_slice(sps_nal); + v.push(1); + v.extend_from_slice(&(pps_nal.len() as u16).to_be_bytes()); + v.extend_from_slice(pps_nal); + if matches!(profile_idc, 100 | 110 | 122 | 144) { + v.push(0xFC | 1); // chroma_format 4:2:0 + v.push(0xF8); // bit_depth_luma_minus8 + v.push(0xF8); // bit_depth_chroma_minus8 + v.push(0); // numOfSequenceParameterSetExt + } + v +} + +/// The requested level when it is at least `needed` (and a valid level_idc), else `needed`. +/// Level 1b (`11` with constraint_set3) is not distinguished; 9 is not a level here. +pub fn level_at_least(requested: u8, needed: u8) -> u8 { + const VALID: [u8; 19] = [10, 11, 12, 13, 20, 21, 22, 30, 31, 32, 40, 41, 42, 50, 51, 52, 60, 61, 62]; + if VALID.contains(&requested) && requested >= needed { requested } else { needed } +} + +/// Pick the lowest level satisfying frame size, macroblock rate, DPB size and (optionally) bitrate. +pub fn pick_level(width_mbs: u32, height_mbs: u32, fps: f64, dpb_frames: u32, kbps: Option, high: bool) -> u8 { + // (level_idc, MaxMBPS, MaxFS, MaxDpbMbs, MaxBR kbit/s) + const LEVELS: [(u8, u64, u64, u64, u64); 17] = [ + (10, 1485, 99, 396, 64), + (11, 3000, 396, 900, 192), + (12, 6000, 396, 2376, 384), + (13, 11880, 396, 2376, 768), + (20, 11880, 396, 2376, 2000), + (21, 19800, 792, 4752, 4000), + (22, 20250, 1620, 8100, 4000), + (30, 40500, 1620, 8100, 10000), + (31, 108000, 3600, 18000, 14000), + (32, 216000, 5120, 20480, 20000), + (40, 245760, 8192, 32768, 20000), + (41, 245760, 8192, 32768, 50000), + (42, 522240, 8704, 34816, 50000), + (50, 589824, 22080, 110400, 135000), + (51, 983040, 36864, 184320, 240000), + (52, 2073600, 36864, 184320, 240000), + (62, 16711680, 139264, 696320, 800000), + ]; + let fs = (width_mbs * height_mbs) as u64; + let mbps = (fs as f64 * fps).ceil() as u64; + for &(l, max_mbps, max_fs, max_dpb, max_br) in &LEVELS { + let br_ok = kbps.is_none_or(|k| (k as u64) * 1000 <= max_br * if high { 1250 } else { 1000 }); + let dim_ok = (width_mbs as u64).pow(2) <= 8 * max_fs && (height_mbs as u64).pow(2) <= 8 * max_fs; + if fs <= max_fs && mbps <= max_mbps && fs * dpb_frames as u64 <= max_dpb && br_ok && dim_ok { + return l; + } + } + 62 +} diff --git a/crates/h264enc/src/picture.rs b/crates/h264enc/src/picture.rs new file mode 100644 index 0000000000..7443d478bb --- /dev/null +++ b/crates/h264enc/src/picture.rs @@ -0,0 +1,313 @@ +//! Padded picture planes, reference pictures and motion-compensated prediction (§8.4.2.2). + +use crate::mbinfo::{MbInfo, Mv}; + +pub const LUMA_PAD: usize = 64; +pub const CHROMA_PAD: usize = 32; + +/// A plane with `pad` replicated samples on every side. +#[derive(Clone)] +pub struct Plane { + pub data: Vec, + pub stride: usize, + pub w: usize, + pub h: usize, + pub pad: usize, +} + +impl Plane { + pub fn new(w: usize, h: usize, pad: usize) -> Self { + let stride = w + 2 * pad; + Plane { data: vec![0; stride * (h + 2 * pad)], stride, w, h, pad } + } + #[inline] + pub fn idx(&self, x: isize, y: isize) -> usize { + ((y + self.pad as isize) as usize) * self.stride + (x + self.pad as isize) as usize + } + /// Replicate edge samples into the padding. + pub fn extend_edges(&mut self) { + let (w, h, pad, stride) = (self.w, self.h, self.pad, self.stride); + for y in 0..h { + let r = (y + pad) * stride; + let l = self.data[r + pad]; + let rr = self.data[r + pad + w - 1]; + self.data[r..r + pad].fill(l); + self.data[r + pad + w..r + stride].fill(rr); + } + let first = pad * stride; + let last = (pad + h - 1) * stride; + for y in 0..pad { + self.data.copy_within(first..first + stride, y * stride); + self.data.copy_within(last..last + stride, (pad + h + y) * stride); + } + } + /// Copy the picture area out as a tightly packed plane of `cw x ch` (cropping). + pub fn copy_out(&self, cw: usize, ch: usize, out: &mut Vec) { + for y in 0..ch { + let o = self.idx(0, y as isize); + out.extend_from_slice(&self.data[o..o + cw]); + } + } +} + +/// A 4:2:0 frame with padded planes. +#[derive(Clone)] +pub struct Frame { + pub y: Plane, + pub u: Plane, + pub v: Plane, +} + +impl Frame { + pub fn new(w: usize, h: usize) -> Self { + Frame { y: Plane::new(w, h, LUMA_PAD), u: Plane::new(w / 2, h / 2, CHROMA_PAD), v: Plane::new(w / 2, h / 2, CHROMA_PAD) } + } + pub fn extend_edges(&mut self) { + self.y.extend_edges(); + self.u.extend_edges(); + self.v.extend_edges(); + } +} + +/// A decoded reference picture with half-sample planes for fast sub-pel access. +pub struct RefPic { + pub frame: Frame, + /// Horizontal half (b), vertical half (h) and centre (j) planes, same geometry as luma. + pub hpel: [Plane; 3], + pub mbs: Vec, + pub poc: i32, + pub id: u32, +} + +#[inline] +fn clip8(v: i32) -> u8 { + v.clamp(0, 255) as u8 +} + +/// Compute the H, V and centre half-sample planes over the whole padded area (clamped at the buffer edge, +/// which equals the normative picture-edge clamping because the padding is replicated). +pub fn build_hpel(f: &Plane, reuse: Option<[Plane; 3]>) -> [Plane; 3] { + let [mut hp, mut vp, mut cp] = reuse.filter(|r| r[0].data.len() == f.data.len()).unwrap_or_else(|| { + let mk = || Plane { data: vec![0; f.data.len()], stride: f.stride, w: f.w, h: f.h, pad: f.pad }; + [mk(), mk(), mk()] + }); + let rows = f.data.len() / f.stride; + let band = 16usize; + let work = |b: usize, hrows: &mut [u8], vrows: &mut [u8], crows: &mut [u8], scratch: &mut HpelScratch| { + let r0 = b * band; + let r1 = (r0 + band).min(rows); + hpel_rows(f, r0, r1, hrows, vrows, crows, scratch); + }; + let chunk = band * f.stride; + #[cfg(feature = "threads")] + { + use rayon::prelude::*; + hp.data + .par_chunks_mut(chunk) + .zip(vp.data.par_chunks_mut(chunk)) + .zip(cp.data.par_chunks_mut(chunk)) + .enumerate() + .for_each_init(HpelScratch::default, |s, (b, ((h, v), c))| work(b, h, v, c, s)); + } + #[cfg(not(feature = "threads"))] + { + let mut s = HpelScratch::default(); + for (b, ((h, v), c)) in hp.data.chunks_mut(chunk).zip(vp.data.chunks_mut(chunk)).zip(cp.data.chunks_mut(chunk)).enumerate() { + work(b, h, v, c, &mut s); + } + } + [hp, vp, cp] +} + +/// Per-worker buffers for `hpel_rows`, reused across bands. +#[derive(Default)] +struct HpelScratch { + hs: Vec, + ext: Vec, +} + +fn hpel_rows(f: &Plane, r0: usize, r1: usize, hout: &mut [u8], vout: &mut [u8], cout: &mut [u8], scratch: &mut HpelScratch) { + let stride = f.stride; + let rows = f.data.len() / stride; + let clampr = |r: isize| r.clamp(0, rows as isize - 1) as usize; + // Horizontal intermediate (unclipped, unshifted) for rows r0-2 .. r1+3. + let nrows = r1 - r0 + 5; + scratch.hs.resize(nrows * stride, 0); + scratch.ext.resize(stride + 5, 0); + let (hs, ext) = (&mut scratch.hs[..], &mut scratch.ext[..]); + for k in 0..nrows { + let r = clampr(r0 as isize - 2 + k as isize); + let row = &f.data[r * stride..r * stride + stride]; + ext[0] = row[0]; + ext[1] = row[0]; + ext[2..2 + stride].copy_from_slice(row); + ext[stride + 2] = row[stride - 1]; + ext[stride + 3] = row[stride - 1]; + ext[stride + 4] = row[stride - 1]; + let o = &mut hs[k * stride..k * stride + stride]; + for x in 0..stride { + let e = &ext[x..x + 6]; + o[x] = (e[0] as i16 + e[5] as i16) - 5 * (e[1] as i16 + e[4] as i16) + 20 * (e[2] as i16 + e[3] as i16); + } + } + for r in r0..r1 { + let lo = (r - r0) * stride; + let k = r - r0 + 2; + let hrow = &hs[k * stride..k * stride + stride]; + for x in 0..stride { + hout[lo + x] = clip8((hrow[x] as i32 + 16) >> 5); + } + // vertical + let rr: [usize; 6] = std::array::from_fn(|i| clampr(r as isize - 2 + i as isize) * stride); + for x in 0..stride { + let s = f.data[rr[0] + x] as i32 + f.data[rr[5] + x] as i32 - 5 * (f.data[rr[1] + x] as i32 + f.data[rr[4] + x] as i32) + + 20 * (f.data[rr[2] + x] as i32 + f.data[rr[3] + x] as i32); + vout[lo + x] = clip8((s + 16) >> 5); + } + // centre from horizontal intermediates + let hk: [&[i16]; 6] = std::array::from_fn(|i| &hs[(k - 2 + i) * stride..(k - 1 + i) * stride]); + for x in 0..stride { + let s = hk[0][x] as i32 + hk[5][x] as i32 - 5 * (hk[1][x] as i32 + hk[4][x] as i32) + 20 * (hk[2][x] as i32 + hk[3][x] as i32); + cout[lo + x] = clip8((s + 512) >> 10); + } + } +} + +/// Plane selection per quarter-sample position: two (plane, dx, dy) sources to average (identical = no average). +/// Plane indices: 0 full, 1 horizontal half, 2 vertical half, 3 centre. +const QPEL_SRC: [[(u8, u8, u8); 2]; 16] = [ + // fy = 0 + [(0, 0, 0), (0, 0, 0)], + [(0, 0, 0), (1, 0, 0)], + [(1, 0, 0), (1, 0, 0)], + [(1, 0, 0), (0, 1, 0)], + // fy = 1 + [(0, 0, 0), (2, 0, 0)], + [(1, 0, 0), (2, 0, 0)], + [(1, 0, 0), (3, 0, 0)], + [(1, 0, 0), (2, 1, 0)], + // fy = 2 + [(2, 0, 0), (2, 0, 0)], + [(2, 0, 0), (3, 0, 0)], + [(3, 0, 0), (3, 0, 0)], + [(3, 0, 0), (2, 1, 0)], + // fy = 3 + [(2, 0, 0), (0, 0, 1)], + [(2, 0, 0), (1, 0, 1)], + [(3, 0, 0), (1, 0, 1)], + [(2, 1, 0), (1, 0, 1)], +]; + +/// A `w` x `h` block at sample position (`x`, `y`) of a plane. +#[derive(Clone, Copy, Debug)] +pub struct McBlock { + pub x: usize, + pub y: usize, + pub w: usize, + pub h: usize, +} + +impl RefPic { + #[inline] + fn plane(&self, p: u8) -> &Plane { + match p { + 0 => &self.frame.y, + n => &self.hpel[n as usize - 1], + } + } + + /// Luma prediction of a w x h block at (x, y) displaced by `mv`, as (samples, stride). Full- and half-sample + /// positions are read straight from their plane; other positions are interpolated into `buf` (stride 16). + #[inline] + pub fn luma_ref<'s>(&'s self, x: usize, y: usize, mv: Mv, w: usize, h: usize, buf: &'s mut [u8]) -> (&'s [u8], usize) { + let [(pa, dxa, dya), (pb, dxb, dyb)] = QPEL_SRC[((mv.y & 3) * 4 + (mv.x & 3)) as usize]; + if pa == pb && dxa == dxb && dya == dyb { + let a = self.plane(pa); + let i = a.idx(x as isize + (mv.x as isize >> 2) + dxa as isize, y as isize + (mv.y as isize >> 2) + dya as isize); + (&a.data[i..], a.stride) + } else { + self.mc_luma(McBlock { x, y, w, h }, mv, buf, 16); + (buf, 16) + } + } + + /// Luma prediction of a w x h block at picture position (x, y) displaced by `mv` (quarter samples). + pub fn mc_luma(&self, blk: McBlock, mv: Mv, out: &mut [u8], ostride: usize) { + match blk.w { + 16 => self.mc_luma_w(McBlock { w: 16, ..blk }, mv, out, ostride), + 8 => self.mc_luma_w(McBlock { w: 8, ..blk }, mv, out, ostride), + _ => self.mc_luma_w(blk, mv, out, ostride), + } + } + + #[inline(always)] + fn mc_luma_w(&self, blk: McBlock, mv: Mv, out: &mut [u8], ostride: usize) { + let McBlock { x, y, w, h } = blk; + let xi = x as isize + (mv.x as isize >> 2); + let yi = y as isize + (mv.y as isize >> 2); + let q = ((mv.y & 3) * 4 + (mv.x & 3)) as usize; + let [(pa, dxa, dya), (pb, dxb, dyb)] = QPEL_SRC[q]; + let a = self.plane(pa); + let stride = a.stride; + let ia = a.idx(xi + dxa as isize, yi + dya as isize); + if pa == pb && dxa == dxb && dya == dyb { + for r in 0..h { + out[r * ostride..r * ostride + w].copy_from_slice(&a.data[ia + r * stride..ia + r * stride + w]); + } + } else { + let b = self.plane(pb); + let ib = b.idx(xi + dxb as isize, yi + dyb as isize); + for r in 0..h { + let ra = &a.data[ia + r * stride..ia + r * stride + w]; + let rb = &b.data[ib + r * stride..ib + r * stride + w]; + let o = &mut out[r * ostride..r * ostride + w]; + for i in 0..w { + o[i] = ((ra[i] as u16 + rb[i] as u16 + 1) >> 1) as u8; + } + } + } + } + + /// Chroma prediction (both planes) of a w x h chroma block at chroma position (x, y); mv in luma quarter units. + pub fn mc_chroma(&self, blk: McBlock, mv: Mv, out: (&mut [u8], &mut [u8]), ostride: usize) { + match blk.w { + 8 => self.mc_chroma_w(McBlock { w: 8, ..blk }, mv, out, ostride), + 4 => self.mc_chroma_w(McBlock { w: 4, ..blk }, mv, out, ostride), + _ => self.mc_chroma_w(blk, mv, out, ostride), + } + } + + #[inline(always)] + fn mc_chroma_w(&self, blk: McBlock, mv: Mv, out: (&mut [u8], &mut [u8]), ostride: usize) { + let McBlock { x, y, w, h } = blk; + let (out_u, out_v) = out; + let xi = x as isize + (mv.x as isize >> 3); + let yi = y as isize + (mv.y as isize >> 3); + let fx = (mv.x & 7) as u32; + let fy = (mv.y & 7) as u32; + let w00 = (8 - fx) * (8 - fy); + let w01 = fx * (8 - fy); + let w10 = (8 - fx) * fy; + let w11 = fx * fy; + for (p, out) in [(&self.frame.u, out_u), (&self.frame.v, out_v)] { + let s = p.stride; + let i0 = p.idx(xi, yi); + for r in 0..h { + let a = &p.data[i0 + r * s..i0 + r * s + w + 1]; + let b = &p.data[i0 + (r + 1) * s..i0 + (r + 1) * s + w + 1]; + let o = &mut out[r * ostride..r * ostride + w]; + for i in 0..w { + o[i] = ((w00 * a[i] as u32 + w01 * a[i + 1] as u32 + w10 * b[i] as u32 + w11 * b[i + 1] as u32 + 32) >> 6) as u8; + } + } + } + } +} + +/// Average two predictions in place (default bi-prediction). +#[inline] +pub fn avg_into(dst: &mut [u8], src: &[u8]) { + for (d, &s) in dst.iter_mut().zip(src) { + *d = ((*d as u16 + s as u16 + 1) >> 1) as u8; + } +} diff --git a/crates/h264enc/src/ratecontrol.rs b/crates/h264enc/src/ratecontrol.rs new file mode 100644 index 0000000000..b9f61580c3 --- /dev/null +++ b/crates/h264enc/src/ratecontrol.rs @@ -0,0 +1,322 @@ +//! Frame-level rate control: constant QP, constant quality (CRF-like), 1-pass ABR with VBV, and 2-pass VBR. + +use crate::nal::SliceType; + +pub fn qp2qscale(qp: f64) -> f64 { + 0.85 * 2f64.powf((qp - 12.0) / 6.0) +} +pub fn qscale2qp(q: f64) -> f64 { + 12.0 + 6.0 * (q / 0.85).log2() +} + +const QCOMP: f64 = 0.6; + +fn type_idx(t: SliceType) -> usize { + match t { + SliceType::I => 0, + SliceType::P => 1, + SliceType::B => 2, + } +} + +/// Relative qscale factor per frame type (I frames finer, B frames coarser). +fn type_factor(t: SliceType) -> f64 { + match t { + SliceType::I => 1.0 / 1.4, + SliceType::P => 1.0, + SliceType::B => 1.3, + } +} + +/// Statistics of one frame from the first pass. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct FrameStat { + pub slice_type: SliceType, + pub qscale: f64, + pub bits: u64, + pub cplx: f64, +} + +/// First-pass statistics, serialisable as text. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct PassStats { + pub frames: Vec, + pub fps: f64, +} + +impl PassStats { + pub fn to_text(&self) -> String { + let mut s = format!("h264enc-pass1 fps={}\n", self.fps); + for f in &self.frames { + let t = match f.slice_type { + SliceType::I => 'I', + SliceType::P => 'P', + SliceType::B => 'B', + }; + s.push_str(&format!("{t} {} {} {}\n", f.qscale, f.bits, f.cplx)); + } + s + } + pub fn from_text(text: &str) -> Option { + let mut lines = text.lines(); + let head = lines.next()?; + let fps = head.strip_prefix("h264enc-pass1 fps=")?.trim().parse().ok()?; + let mut frames = Vec::new(); + for l in lines { + let mut it = l.split_whitespace(); + let t = match it.next()? { + "I" => SliceType::I, + "P" => SliceType::P, + "B" => SliceType::B, + _ => return None, + }; + frames.push(FrameStat { + slice_type: t, + qscale: it.next()?.parse().ok()?, + bits: it.next()?.parse().ok()?, + cplx: it.next()?.parse().ok()?, + }); + } + Some(PassStats { frames, fps }) + } +} + +#[derive(Clone, Debug)] +pub enum RcKind { + Qp(u8), + Crf(f32), + Abr { kbps: u32 }, + TwoPass { kbps: u32, plan: Vec, planned_bits: Vec, planned_total: f64, remaining_bits: Vec }, +} + +pub struct RateControl { + kind: RcKind, + fps: f64, + /// bits ≈ coef * cplx / qscale per frame type. + coef: [f64; 3], + coef_n: [u32; 3], + total_bits: f64, + frames: u64, + sum_s: f64, + // VBV + vbv_max_bits_per_frame: f64, + vbv_size: f64, + vbv_fill: f64, + pub pass1: Vec, + model_done: f64, + inflight: std::collections::VecDeque, + last_q: f64, + /// Virtual history for 1-pass ABR: (frame count, complexity of one virtual frame), fixed at the first frame. + prior: Option<(f64, f64)>, + /// Sum and count of the rate-model terms (`coef · cplx^QCOMP / type_factor`) of the coded P and B frames. + inter_s: f64, + inter_n: u64, +} + +impl RateControl { + pub fn new(kind: RcKind, fps: f64, vbv_maxrate_kbps: Option, vbv_buf_kbits: Option) -> Self { + let max = vbv_maxrate_kbps.map_or(0.0, |k| k as f64 * 1000.0 / fps); + let size = vbv_buf_kbits.map_or(0.0, |k| k as f64 * 1000.0); + RateControl { + kind, + fps, + coef: [1.6, 1.4, 1.2], + coef_n: [0; 3], + total_bits: 0.0, + frames: 0, + sum_s: 0.0, + vbv_max_bits_per_frame: max, + vbv_size: size, + vbv_fill: size * 0.9, + pass1: Vec::new(), + model_done: 0.0, + inflight: std::collections::VecDeque::new(), + last_q: 0.0, + prior: None, + inter_s: 0.0, + inter_n: 0, + } + } + + /// Plan a second pass from first-pass statistics. + pub fn plan_two_pass(stats: &PassStats, kbps: u32) -> RcKind { + let n = stats.frames.len().max(1); + let target = kbps as f64 * 1000.0 * n as f64 / stats.fps; + let c: Vec = stats.frames.iter().map(|f| (f.bits as f64 * f.qscale).max(1.0)).collect(); + let qs = |rf: f64| -> Vec { + stats + .frames + .iter() + .zip(&c) + .map(|(f, &ci)| (rf * ci.powf(1.0 - QCOMP) * type_factor(f.slice_type)).clamp(qp2qscale(0.0), qp2qscale(51.0))) + .collect() + }; + let bits_for = |q: &[f64]| -> f64 { stats.frames.iter().zip(q).map(|(f, &qi)| f.bits as f64 * f.qscale / qi).sum() }; + // bisection on log(rf) + let (mut lo, mut hi) = (-30.0f64, 30.0f64); + for _ in 0..100 { + let mid = 0.5 * (lo + hi); + if bits_for(&qs(mid.exp())) > target { + lo = mid; + } else { + hi = mid; + } + } + let plan = qs((0.5 * (lo + hi)).exp()); + let planned_bits: Vec = stats.frames.iter().zip(&plan).map(|(f, &q)| f.bits as f64 * f.qscale / q).collect(); + let planned_total = planned_bits.iter().sum(); + // `remaining_bits[i]` is the inclusive suffix for frame `i`; retain an + // empty suffix so callers can index it without a special case. + let mut remaining_bits = vec![0.0; planned_bits.len() + 1]; + for i in (0..planned_bits.len()).rev() { + remaining_bits[i] = remaining_bits[i + 1] + planned_bits[i]; + } + RcKind::TwoPass { kbps, plan, planned_bits, planned_total, remaining_bits } + } + + /// Choose the frame QP (fractional) for frame `index` (display order) of `t` with lookahead complexity `cplx`. + pub fn frame_qp(&mut self, t: SliceType, cplx: f64, index_in_coding_order: usize) -> f64 { + let ti = type_idx(t); + let cplx = cplx.max(1.0); + let qp = match &self.kind { + RcKind::Qp(q) => { + let q = *q as f64; + return if t == SliceType::B { (q + 2.0).min(51.0) } else { q }; + } + RcKind::Crf(c) => { + let c = *c as f64; + return (c + match t { + SliceType::I => -2.0, + SliceType::P => 0.0, + SliceType::B => 1.5, + }) + .clamp(0.0, 51.0); + } + RcKind::Abr { kbps } => { + let bpf = *kbps as f64 * 1000.0 / self.fps; + // rate factor so that past frames at this rf would have hit the target + let s_here = self.coef[ti] * cplx.powf(QCOMP) / type_factor(t); + // Virtual history of one second of inter frames so the first I frame gets a realistic share of the + // budget instead of a single frame's worth. A virtual frame costs what the P and B frames coded so far + // cost on average; before the first of them, a P frame of 40% of the first frame's complexity under + // the current model. Re-estimated every frame: the initial guess of `coef` can be several times off, + // and a history frozen at that guess starves the opening second of bits. + let fps = self.fps; + let (pn0, c0) = *self.prior.get_or_insert_with(|| (fps.max(1.0), if t == SliceType::I { 0.4 * cplx } else { cplx })); + let per_frame = if self.inter_n > 0 { self.inter_s / self.inter_n as f64 } else { self.coef[1] * c0.powf(QCOMP) }; + let fade = (1.0 - self.frames as f64 / pn0).max(0.0); + let (pn, ps) = (pn0 * fade, pn0 * fade * per_frame); + let s = self.sum_s + s_here + ps; + let wanted = bpf * (self.frames as f64 + 1.0 + pn); + let mut rf = s / wanted; + // A quarter second of bits: with a longer horizon the correction is so weak that a short clip + // that starts over target never pays the excess back (+10..14% on 2 s of high motion at 1 s). + // Never shorter than one frame's budget: at a low frame rate the horizon would be a fraction of a + // frame and the correction would overshoot in the other direction every frame. + let abr_buffer = (bpf * self.fps.max(1.0) * 0.25).max(bpf); + let overflow = (1.0 + (self.total_bits - bpf * self.frames as f64) / abr_buffer).clamp(0.5, 2.0); + rf *= overflow; + let q = rf * cplx.powf(1.0 - QCOMP) * type_factor(t); + qscale2qp(q) + } + RcKind::TwoPass { kbps, plan, planned_bits, planned_total, remaining_bits } => { + // Closed-loop second pass: `m` measures how far real frame sizes deviate from the first-pass model + // (bits ∝ 1/qscale) at the qscales actually used; the remaining plan is rescaled so its predicted + // size fits the remaining budget. + let i = index_in_coding_order.min(plan.len().saturating_sub(1)); + let bpf = *kbps as f64 * 1000.0 / self.fps; + let target_total = *planned_total; + let smooth = 0.5 * bpf * self.fps.max(1.0); + let m = ((self.total_bits + smooth) / (self.model_done + smooth)).clamp(0.3, 3.0); + let inflight: f64 = self.inflight.iter().sum(); + let remaining_plan = remaining_bits[i]; + let remaining_budget = (target_total - self.total_bits - m * inflight).max(0.05 * remaining_plan.max(1.0)); + let f = (m * remaining_plan / remaining_budget).clamp(0.25, 4.0); + let q = plan[i] * f; + self.inflight.push_back(planned_bits[i] * plan[i] / q); + qscale2qp(q) + } + }; + let mut qp = qp.clamp(1.0, 51.0); + // VBV: make sure the predicted frame size fits in the buffer. + if self.vbv_size > 0.0 { + let fill = self.vbv_fill; + for _ in 0..60 { + let pred = self.coef[ti] * cplx / qp2qscale(qp); + if pred <= fill * 0.8 || qp >= 51.0 { + break; + } + qp += 0.5; + } + } + self.last_q = qp; + qp + } + + /// Update after coding a frame with actual `bits`. + pub fn update(&mut self, t: SliceType, qp: f64, cplx: f64, bits: u64) { + let ti = type_idx(t); + let cplx = cplx.max(1.0); + let q = qp2qscale(qp); + let c = bits as f64 * q / cplx; + let n = self.coef_n[ti]; + let w = if n == 0 { 1.0 } else { 0.35f64.max(1.0 / (n + 1) as f64) }; + self.coef[ti] = self.coef[ti] * (1.0 - w) + c * w; + self.coef_n[ti] += 1; + // If a type has never been observed, keep it proportional to observed ones. + for k in 0..3 { + if self.coef_n[k] == 0 { + self.coef[k] = self.coef[ti] * [1.2, 1.0, 0.85][k] / [1.2, 1.0, 0.85][ti]; + } + } + let s_frame = self.coef[ti] * cplx.powf(QCOMP) / type_factor(t); + self.sum_s += s_frame; + if t != SliceType::I { + self.inter_s += s_frame; + self.inter_n += 1; + } + self.total_bits += bits as f64; + if let Some(p) = self.inflight.pop_front() { + self.model_done += p; + } + self.frames += 1; + if self.vbv_size > 0.0 { + self.vbv_fill = (self.vbv_fill - bits as f64 + self.vbv_max_bits_per_frame).min(self.vbv_size); + if self.vbv_fill < 0.0 { + self.vbv_fill = 0.0; + } + } + } + + pub fn record_pass1(&mut self, t: SliceType, qp: f64, cplx: f64, bits: u64) { + self.pass1.push(FrameStat { slice_type: t, qscale: qp2qscale(qp), bits, cplx }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn two_pass_plan_caches_total_and_inclusive_suffixes() { + let stats = PassStats { + fps: 24.0, + frames: vec![ + FrameStat { slice_type: SliceType::I, qscale: 1.0, bits: 2_000, cplx: 20.0 }, + FrameStat { slice_type: SliceType::P, qscale: 1.5, bits: 1_000, cplx: 10.0 }, + FrameStat { slice_type: SliceType::B, qscale: 2.0, bits: 500, cplx: 5.0 }, + ], + }; + + let RcKind::TwoPass { planned_bits, planned_total, remaining_bits, .. } = RateControl::plan_two_pass(&stats, 1_000) else { + panic!("two-pass planning must return a two-pass rate-control plan"); + }; + + assert_eq!(remaining_bits.len(), planned_bits.len() + 1); + assert!((planned_total - planned_bits.iter().sum::()).abs() < f64::EPSILON); + for i in 0..planned_bits.len() { + assert!((remaining_bits[i] - planned_bits[i..].iter().sum::()).abs() < f64::EPSILON); + } + assert_eq!(remaining_bits[planned_bits.len()], 0.0); + } +} diff --git a/crates/h264enc/src/slice.rs b/crates/h264enc/src/slice.rs new file mode 100644 index 0000000000..ef6d21889f --- /dev/null +++ b/crates/h264enc/src/slice.rs @@ -0,0 +1,486 @@ +//! Slice encoding: macroblock analysis, mode decision, residual coding, reconstruction and entropy coding. + +use filmcraft_bitstream::BitWriter; + +use crate::cabac::{BinSink, CabacEncoder, CabacEstimator, Contexts, CostTable}; +use crate::dsp::{sad, sad8, sad16, satd, ssd}; +use crate::intra::{Edge4, Edge8, filter8, mode_available, pred_chroma, pred4x4, pred8x8, pred16x16}; +use crate::mbinfo::{MbInfo, MbKind, Mv, Part}; +use crate::nal::{SliceHeader, SliceType, nal}; +use crate::picture::{CHROMA_PAD, Frame, LUMA_PAD, RefPic, avg_into}; +use crate::syntax::{MbCode, NbCtx}; +use crate::tables::{BLK4_XY, NUM_CTX, RASTER_TO_BLK4, ZIGZAG4, ZIGZAG8, chroma_qp}; +use crate::transform::{ + Deadzone, QuantTables, dequant_chroma_dc, dequant_luma_dc, dequant4, dequant8, fdct4, fdct8, idct4, idct8, quant_chroma_dc, quant_luma_dc, + quant4, quant8, +}; +use crate::{cabac_mb, cavlc}; + +mod b_slices; +mod intra; +mod macroblock; +mod motion; +mod residual; + +/// Encoder tuning derived from the preset. +#[derive(Clone, Debug)] +pub struct EncParams { + pub mbw: usize, + pub mbh: usize, + pub cabac: bool, + pub t8x8: bool, + pub i4x4: bool, + pub i8x8: bool, + pub partitions: bool, + pub me_range: i32, + /// 0 = full-pel, 1 = half-pel, 2 = quarter-pel refinement. + pub subpel: u8, + /// Extra quarter-pel refinement rounds. + pub subpel_rounds: u8, + pub rd: bool, + pub chroma_qp_offset: i32, + pub decimate: bool, + /// Adaptive 4x4/8x8 transform decision for inter macroblocks. + pub adaptive_t8: bool, + /// Always try intra in P/B frames (otherwise only when inter looks poor). + pub always_intra: bool, + /// Maximum CAVLC level magnitude (Baseline). + pub cavlc_clamp: bool, +} + +/// Per-frame shared encoding state. +pub struct FrameEnc<'a> { + pub p: &'a EncParams, + pub qt: &'a QuantTables, + pub cost_tab: &'a CostTable, + pub st: SliceType, + pub src: &'a Frame, + pub l0: Option<&'a RefPic>, + pub l1: Option<&'a RefPic>, + pub qp: u8, + pub aq: &'a [i8], + pub hdr: SliceHeader, + pub nal_ref_idc: u8, + pub nal_type: u8, + pub cabac_table: &'a [(i8, i8); NUM_CTX], + pub lam: &'a Lambdas, +} + +/// Lambda tables indexed by QP. +pub struct Lambdas { + /// SATD/SAD-domain lambda, Q8. + pub sad: [u32; 52], + /// SSD-domain lambda (for RD), Q8 per 1/256 bit. + pub ssd: [f64; 52], +} + +impl Lambdas { + pub fn new() -> Self { + let mut sad = [0u32; 52]; + let mut ssd_ = [0f64; 52]; + for qp in 0..52 { + let l = 0.85 * 2f64.powf((qp as f64 - 12.0) / 3.0); + ssd_[qp] = l; + sad[qp] = ((l.sqrt()) * 256.0).round().max(16.0) as u32; + } + Lambdas { sad, ssd: ssd_ } + } +} + +impl Default for Lambdas { + fn default() -> Self { + Self::new() + } +} + +/// Mutable band of reconstructed rows belonging to one slice. +pub struct Band<'b> { + pub y: &'b mut [u8], + pub u: &'b mut [u8], + pub v: &'b mut [u8], + pub ys: usize, + pub cs: usize, +} + +#[derive(Default, Clone, Debug)] +pub struct SliceStats { + pub bits: u64, + pub intra: u32, + pub skip: u32, + pub inter: u32, + pub qp_sum: u64, + pub mbs: u32, +} + +pub struct SliceOut { + pub nal: Vec, + #[allow(dead_code)] + pub stats: SliceStats, +} + +#[derive(Clone, Copy)] +struct Avail { + left: bool, + top: bool, + tr: bool, + tl: bool, +} + +struct MbEdges { + top: [u8; 24], + left: [u8; 16], + tl: u8, + ctop: [[u8; 8]; 2], + cleft: [[u8; 8]; 2], + ctl: [u8; 2], +} + +#[derive(Clone)] +struct Cand { + code: MbCode, + info: MbInfo, + ry: [u8; 256], + ru: [u8; 64], + rv: [u8; 64], + cost: u64, +} + +impl Cand { + fn new() -> Box { + Box::new(Cand { code: MbCode::default(), info: MbInfo::default(), ry: [0; 256], ru: [0; 64], rv: [0; 64], cost: u64::MAX }) + } +} + +#[inline] +fn se_bits(v: i32) -> u32 { + let k = if v > 0 { 2 * v as u32 - 1 } else { (-2 * v) as u32 }; + 2 * (31 - (k + 1).leading_zeros()) + 1 +} + +#[inline] +fn mv_bits(mv: Mv, mvp: Mv) -> u32 { + se_bits(mv.x as i32 - mvp.x as i32) + se_bits(mv.y as i32 - mvp.y as i32) +} + +fn median(a: i16, b: i16, c: i16) -> i16 { + a.max(b).min(a.min(b).max(c)) +} + +fn decimate_score(levels: &[i16]) -> u32 { + const T: [u32; 16] = [3, 2, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut score = 0; + let mut run = 0usize; + for &l in levels { + if l == 0 { + run += 1; + continue; + } + if l.unsigned_abs() > 1 { + return 64; + } + score += T[run.min(15)]; + run = 0; + } + score +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Shape { + Normal, + Top16x8, + Bottom16x8, + Left8x16, + Right8x16, +} + +pub struct SliceEnc<'a, 'b> { + f: &'a FrameEnc<'a>, + band: Band<'b>, + mbs: &'b mut [MbInfo], + y0: usize, + y1: usize, + slice_no: u16, + last_qp: u8, + prev_dqp_nz: bool, + cabac: Option, + cavlc: Option, + skip_run: u32, + stats: SliceStats, +} + +impl<'a, 'b> SliceEnc<'a, 'b> { + pub fn new(f: &'a FrameEnc<'a>, band: Band<'b>, mbs: &'b mut [MbInfo], y0: usize, y1: usize, slice_no: u16) -> Self { + SliceEnc { + f, + band, + mbs, + y0, + y1, + slice_no, + last_qp: f.qp, + prev_dqp_nz: false, + cabac: None, + cavlc: None, + skip_run: 0, + stats: SliceStats::default(), + } + } + + pub fn encode(mut self) -> SliceOut { + let f = self.f; + let mbw = f.p.mbw; + let mut hdr = f.hdr.clone(); + hdr.first_mb = (self.y0 * mbw) as u32; + let mut bw = BitWriter::new(); + hdr.write(&mut bw); + if f.p.cabac { + while !bw.is_byte_aligned() { + bw.write_bit(true); + } + let ctx = Contexts::init(f.cabac_table, f.qp as i32); + self.cabac = Some(CabacEncoder::new(ctx, bw.finish())); + } else { + self.cavlc = Some(bw); + } + for my in self.y0..self.y1 { + for mx in 0..mbw { + self.encode_mb(mx, my); + if let Some(c) = self.cabac.as_mut() { + let last = my + 1 == self.y1 && mx + 1 == mbw; + if !last { + c.terminate(0); + } + } + } + } + let rbsp = if let Some(c) = self.cabac.take() { + c.finish() + } else { + let mut w = self.cavlc.take().unwrap_or_default(); + if self.skip_run > 0 { + w.write_ue(self.skip_run); + } + w.rbsp_trailing(); + w.finish() + }; + self.stats.bits = rbsp.len() as u64 * 8; + SliceOut { nal: nal(f.nal_ref_idc, f.nal_type, &rbsp), stats: self.stats } + } + + // ------------------------------------------------------------------ neighbour access + + #[inline] + fn mb(&self, mx: usize, my: usize) -> &MbInfo { + &self.mbs[(my - self.y0) * self.f.p.mbw + mx] + } + + fn avail(&self, mx: usize, my: usize) -> Avail { + let top = my > self.y0; + Avail { left: mx > 0, top, tr: top && mx + 1 < self.f.p.mbw, tl: top && mx > 0 } + } + + #[inline] + fn ry(&self, x: usize, y: usize) -> usize { + (y - self.y0 * 16) * self.band.ys + LUMA_PAD + x + } + #[inline] + fn rc(&self, x: usize, y: usize) -> usize { + (y - self.y0 * 8) * self.band.cs + CHROMA_PAD + x + } + + fn edges(&self, mx: usize, my: usize, av: Avail) -> MbEdges { + let (px, py) = (mx * 16, my * 16); + let mut e = MbEdges { top: [128; 24], left: [128; 16], tl: 128, ctop: [[128; 8]; 2], cleft: [[128; 8]; 2], ctl: [128; 2] }; + if av.top { + let i = self.ry(px, py - 1); + e.top[..16].copy_from_slice(&self.band.y[i..i + 16]); + if av.tr { + e.top[16..24].copy_from_slice(&self.band.y[i + 16..i + 24]); + } else { + let v = e.top[15]; + e.top[16..24].fill(v); + } + for (c, pl) in [&*self.band.u, &*self.band.v].into_iter().enumerate() { + let i = self.rc(px / 2, py / 2 - 1); + e.ctop[c].copy_from_slice(&pl[i..i + 8]); + } + } + if av.left { + for y in 0..16 { + e.left[y] = self.band.y[self.ry(px - 1, py + y)]; + } + for (c, pl) in [&*self.band.u, &*self.band.v].into_iter().enumerate() { + for y in 0..8 { + e.cleft[c][y] = pl[self.rc(px / 2 - 1, py / 2 + y)]; + } + } + } + if av.tl { + e.tl = self.band.y[self.ry(px - 1, py - 1)]; + e.ctl[0] = self.band.u[self.rc(px / 2 - 1, py / 2 - 1)]; + e.ctl[1] = self.band.v[self.rc(px / 2 - 1, py / 2 - 1)]; + } + e + } + + /// Neighbouring partition data at 4x4 offset (bx, by) relative to the MB (range -1..=4). + /// None = not available; Some((ref, mv)) with ref = -1 for intra / list unused. + fn nb_part(&self, nb: MvNb, bx: i32, by: i32, list: usize) -> Option<(i8, Mv)> { + let MvNb { cur, mask, mx, my } = nb; + if (0..4).contains(&bx) && (0..4).contains(&by) { + let r = (by * 4 + bx) as usize; + if mask & (1 << r) != 0 { + return Some((cur.ref_at(list, r), cur.mv[list][r])); + } + return None; + } + let dx = if bx < 0 { + -1 + } else if bx >= 4 { + 1 + } else { + 0 + }; + let dy = if by < 0 { -1 } else { 0 }; + if by >= 4 || (dy == 0 && dx == 1) { + return None; + } + let nx = mx as i32 + dx; + let ny = my as i32 + dy; + if nx < 0 || nx >= self.f.p.mbw as i32 || ny < self.y0 as i32 { + return None; + } + let m = self.mb(nx as usize, ny as usize); + let r = (((by + 4) % 4) * 4 + (bx + 4) % 4) as usize; + if m.kind.is_intra() { + return Some((-1, Mv::ZERO)); + } + let rf = m.ref_at(list, r); + Some((rf, if rf >= 0 { m.mv[list][r] } else { Mv::ZERO })) + } + + /// Motion vector predictor (§8.4.1.3) for a partition at 4x4 coords (x, y) of size (w, h) (in 4x4 units). + fn mvp(&self, nb: MvNb, geom: PartGeom, list: usize, rf: i8) -> Mv { + let PartGeom { x, y, w, shape } = geom; + let a = self.nb_part(nb, x - 1, y, list); + let b = self.nb_part(nb, x, y - 1, list); + let mut c = self.nb_part(nb, x + w, y - 1, list); + if c.is_none() { + c = self.nb_part(nb, x - 1, y - 1, list); + } + let (mut b, mut c) = (b, c); + if b.is_none() && c.is_none() && a.is_some() { + b = a; + c = a; + } + let a = a.unwrap_or((-1, Mv::ZERO)); + let b = b.unwrap_or((-1, Mv::ZERO)); + let c = c.unwrap_or((-1, Mv::ZERO)); + match shape { + Shape::Top16x8 if b.0 == rf => return b.1, + Shape::Bottom16x8 if a.0 == rf => return a.1, + Shape::Left8x16 if a.0 == rf => return a.1, + Shape::Right8x16 if c.0 == rf => return c.1, + _ => {} + } + let n = (a.0 == rf) as u32 + (b.0 == rf) as u32 + (c.0 == rf) as u32; + if n == 1 { + if a.0 == rf { + return a.1; + } + if b.0 == rf { + return b.1; + } + return c.1; + } + Mv { x: median(a.1.x, b.1.x, c.1.x), y: median(a.1.y, b.1.y, c.1.y) } + } + + fn pskip_mv(&self, mx: usize, my: usize) -> Mv { + let cur = MbInfo::default(); + let nb = MvNb { cur: &cur, mask: 0, mx, my }; + let a = self.nb_part(nb, -1, 0, 0); + let b = self.nb_part(nb, 0, -1, 0); + match (a, b) { + (None, _) | (_, None) => Mv::ZERO, + (Some((0, m)), _) if m == Mv::ZERO => Mv::ZERO, + (_, Some((0, m))) if m == Mv::ZERO => Mv::ZERO, + _ => self.mvp(nb, PartGeom { x: 0, y: 0, w: 4, shape: Shape::Normal }, 0, 0), + } + } + + /// Whether a luma block at picture (x, y) of size w x h with `mv` stays within the padded reference. + fn mv_ok(&self, x: usize, y: usize, w: usize, h: usize, mv: Mv) -> bool { + let pw = (self.f.p.mbw * 16) as i32; + let ph = (self.f.p.mbh * 16) as i32; + let m = LUMA_PAD as i32 - 20; + let x0 = x as i32 + (mv.x as i32 >> 2); + let y0 = y as i32 + (mv.y as i32 >> 2); + mv.x.unsigned_abs() < 2000 && mv.y.unsigned_abs() < 2000 && x0 >= -m && y0 >= -m && x0 + w as i32 <= pw + m && y0 + h as i32 <= ph + m + } +} + +/// The macroblock (`mx`, `my`) being analysed: quantiser, neighbour availability and edge samples, and source. +#[derive(Clone, Copy)] +struct MbCtx<'x> { + mx: usize, + my: usize, + qp: u8, + av: Avail, + e: &'x MbEdges, + src: &'x Src, +} + +/// Where a motion-vector-predictor neighbour lookup happens: the macroblock (`mx`, `my`) whose already +/// decided partitions (`mask` over the 4x4 blocks of `cur`) are visible. +#[derive(Clone, Copy)] +struct MvNb<'c> { + cur: &'c MbInfo, + mask: u16, + mx: usize, + my: usize, +} + +/// A partition for motion-vector prediction: 4x4-block origin (`x`, `y`), width `w` (in 4x4 units), and shape. +#[derive(Clone, Copy)] +struct PartGeom { + x: i32, + y: i32, + w: i32, + shape: Shape, +} + +/// A luma block to motion-search at picture position (`x`, `y`) against `src` (row stride `sstride`). +#[derive(Clone, Copy)] +struct SearchBlock<'s> { + x: usize, + y: usize, + w: usize, + h: usize, + src: &'s [u8], + sstride: usize, +} + +/// Parameters of a sequential per-partition search of macroblock (`mx`, `my`) for one reference list. +#[derive(Clone, Copy)] +struct PartSearch { + mx: usize, + my: usize, + part: Part, + list: usize, + lam: u32, + seed: Mv, +} + +/// Destination buffers (luma 16x16, chroma 8x8 each) of a motion-compensated prediction. +struct PredOut<'p> { + y: &'p mut [u8; 256], + u: &'p mut [u8; 64], + v: &'p mut [u8; 64], +} + +struct Src { + y: [u8; 256], + u: [u8; 64], + v: [u8; 64], +} diff --git a/crates/h264enc/src/slice/b_slices.rs b/crates/h264enc/src/slice/b_slices.rs new file mode 100644 index 0000000000..95e42af7cd --- /dev/null +++ b/crates/h264enc/src/slice/b_slices.rs @@ -0,0 +1,321 @@ +use super::{ + Avail, Cand, MbCtx, MbEdges, MbInfo, MbKind, Mv, MvNb, Part, PartGeom, PartSearch, PredOut, Shape, SliceEnc, Src, avg_into, mv_bits, sad16, satd, +}; + +impl<'a, 'b> SliceEnc<'a, 'b> { + // ------------------------------------------------------------------ B slices + + /// Spatial direct prediction (§8.4.1.2.2) with direct_8x8_inference. Returns per-8x8 refs and mvs. + fn direct_spatial(&self, mx: usize, my: usize) -> ([[i8; 4]; 2], [[Mv; 4]; 2]) { + let cur = MbInfo::default(); + let nb = MvNb { cur: &cur, mask: 0, mx, my }; + let mut refs = [-1i8; 2]; + let mut mvp = [Mv::ZERO; 2]; + for l in 0..2 { + let a = self.nb_part(nb, -1, 0, l).map_or(-1, |v| v.0); + let b = self.nb_part(nb, 0, -1, l).map_or(-1, |v| v.0); + let c = self.nb_part(nb, 4, -1, l).or_else(|| self.nb_part(nb, -1, -1, l)).map_or(-1, |v| v.0); + let minp = |x: i8, y: i8| if x >= 0 && y >= 0 { x.min(y) } else { x.max(y) }; + refs[l] = minp(a, minp(b, c)); + } + let zero = refs[0] < 0 && refs[1] < 0; + if zero { + refs = [0, 0]; + } else { + for l in 0..2 { + if refs[l] >= 0 { + mvp[l] = self.mvp(nb, PartGeom { x: 0, y: 0, w: 4, shape: Shape::Normal }, l, refs[l]); + } + } + } + let Some(l1) = self.f.l1 else { return ([[0; 4]; 2], [[Mv::ZERO; 4]; 2]) }; + let col = &l1.mbs[my * self.f.p.mbw + mx]; + let mut out_ref = [[-1i8; 4]; 2]; + let mut out_mv = [[Mv::ZERO; 4]; 2]; + for q in 0..4 { + let corner = [0usize, 3, 12, 15][q]; + let col_zero = if col.kind.is_intra() { + false + } else { + let (rc, mvc) = if col.ref_idx[0][q] >= 0 { (col.ref_idx[0][q], col.mv[0][corner]) } else { (col.ref_idx[1][q], col.mv[1][corner]) }; + rc == 0 && mvc.x.abs() <= 1 && mvc.y.abs() <= 1 + }; + for l in 0..2 { + out_ref[l][q] = refs[l]; + if refs[l] >= 0 { + out_mv[l][q] = if zero || (refs[l] == 0 && col_zero) { Mv::ZERO } else { mvp[l] }; + } + } + } + (out_ref, out_mv) + } + + /// B 16x8 / 8x16 candidate: search each partition in both lists, choose L0/L1/Bi per partition, then derive + /// the motion vector predictors in decoding order from the chosen directions. With `beat`, a + /// candidate whose cost does not come below it is `None` before its residual is coded. + fn b_partition(&self, m: &MbCtx, part: Part, lam: u32, seeds: [Mv; 2], beat: Option) -> Option> { + let MbCtx { mx, my, qp, src, .. } = *m; + let f = self.f; + let rp = [f.l0?, f.l1?]; + let (px, py) = (mx * 16, my * 16); + let mut mvs = [[Mv::ZERO; 4]; 2]; + for l in 0..2 { + mvs[l] = self.search_part(rp[l], &PartSearch { mx, my, part, list: l, lam, seed: seeds[l] }, src).0; + } + let mut preds = [([0u8; 256], [0u8; 64], [0u8; 64]), ([0u8; 256], [0u8; 64], [0u8; 64])]; + for l in 0..2 { + let (a, b, c) = &mut preds[l]; + for pi in 0..2 { + self.pred_block(rp[l], (px, py), part.rect(pi), mvs[l][pi], PredOut { y: a, u: b, v: c }); + } + } + let mut bi = preds[0]; + avg_into(&mut bi.0, &preds[1].0); + avg_into(&mut bi.1, &preds[1].1); + avg_into(&mut bi.2, &preds[1].2); + let srcs = [&preds[0], &preds[1], &bi]; + let (mut py_, mut pu, mut pv) = ([0u8; 256], [0u8; 64], [0u8; 64]); + let mut dirs = [0u8; 4]; + let mut cost = ((lam * 8) >> 8) as u64; + for pi in 0..2 { + let (x, y, w, h) = part.rect(pi); + let mut bestd = (u64::MAX, 0usize); + for (d, s) in srcs.iter().enumerate() { + let bits = if d == 2 { 14 } else { 7 }; + let c = satd(&src.y[y * 16 + x..], 16, &s.0[y * 16 + x..], 16, w, h) as u64 + ((lam * bits) >> 8) as u64; + if c < bestd.0 { + bestd = (c, d); + } + } + cost += bestd.0; + dirs[pi] = bestd.1 as u8; + if beat.is_some_and(|b| cost >= b) { + return None; + } + if beat.is_some_and(|b| cost >= b) { + return None; + } + let s = srcs[bestd.1]; + for r in y..y + h { + py_[r * 16 + x..r * 16 + x + w].copy_from_slice(&s.0[r * 16 + x..r * 16 + x + w]); + } + for r in y / 2..(y + h) / 2 { + let o = r * 8 + x / 2; + pu[o..o + w / 2].copy_from_slice(&s.1[o..o + w / 2]); + pv[o..o + w / 2].copy_from_slice(&s.2[o..o + w / 2]); + } + } + // predictors in decoding order with the actual per-list references + let mut info = MbInfo::default(); + let mut refs = [[-1i8; 4]; 2]; + let mut fmvs = [[Mv::ZERO; 4]; 2]; + let mut mvds = [[Mv::ZERO; 4]; 2]; + for pi in 0..2 { + let (x, y, w, h) = part.rect(pi); + let mask = Self::mask_of(part, pi); + for l in 0..2 { + if dirs[pi] == 2 || dirs[pi] as usize == l { + let mvp = self.mvp( + MvNb { cur: &info, mask, mx, my }, + PartGeom { x: (x / 4) as i32, y: (y / 4) as i32, w: (w / 4) as i32, shape: Self::shape_of(part, pi) }, + l, + 0, + ); + let mv = mvs[l][pi]; + mvds[l][pi] = Mv::new(mv.x as i32 - mvp.x as i32, mv.y as i32 - mvp.y as i32); + refs[l][pi] = 0; + fmvs[l][pi] = mv; + } + } + for l in 0..2 { + for by in y / 4..(y + h) / 4 { + for bx in x / 4..(x + w) / 4 { + info.mv[l][by * 4 + bx] = fmvs[l][pi]; + } + } + for by8 in y / 8..(y + h).div_ceil(8) { + for bx8 in x / 8..(x + w).div_ceil(8) { + info.ref_idx[l][by8 * 2 + bx8] = refs[l][pi]; + } + } + } + } + let mut c = Cand::new(); + c.code.kind = MbKind::BInter; + c.code.part = part; + c.code.bdir = dirs; + c.code.mvd = mvds; + Self::cand_inter_info(&mut c, part, refs, fmvs, mvds); + self.code_inter(src, &py_, &pu, &pv, qp, &mut c); + c.cost = cost; + Some(c) + } + + pub(super) fn analyse_b(&self, mx: usize, my: usize, qp: u8, av: Avail, e: &MbEdges, src: &Src) -> Box { + let f = self.f; + // B slices always have both references; without them the macroblock is coded intra. + let (Some(r0), Some(r1)) = (f.l0, f.l1) else { return self.analyse_intra(&MbCtx { mx, my, qp, av, e, src }, u64::MAX) }; + let lam = f.lam.sad[qp as usize]; + let (px, py) = (mx * 16, my * 16); + // --- direct + let (dref, dmv) = self.direct_spatial(mx, my); + let mut direct_ok = true; + for q in 0..4 { + let rect = Part::P8x8.rect(q); + for l in 0..2 { + if dref[l][q] >= 0 && !self.mv_ok(px + rect.0, py + rect.1, 8, 8, dmv[l][q]) { + direct_ok = false; + } + } + } + let mut best: Option> = None; + if direct_ok { + let mut y = [0u8; 256]; + let mut u = [0u8; 64]; + let mut v = [0u8; 64]; + let mut ty = [0u8; 256]; + let mut tu = [0u8; 64]; + let mut tv = [0u8; 64]; + for q in 0..4 { + let rect = Part::P8x8.rect(q); + let (u0, u1) = (dref[0][q] >= 0, dref[1][q] >= 0); + if u0 { + self.pred_block(r0, (px, py), rect, dmv[0][q], PredOut { y: &mut y, u: &mut u, v: &mut v }); + } + if u1 { + if u0 { + self.pred_block(r1, (px, py), rect, dmv[1][q], PredOut { y: &mut ty, u: &mut tu, v: &mut tv }); + let (x, yy, w, h) = rect; + for rr in 0..h { + avg_into(&mut y[(yy + rr) * 16 + x..(yy + rr) * 16 + x + w], &ty[(yy + rr) * 16 + x..(yy + rr) * 16 + x + w]); + } + for rr in 0..h / 2 { + let o = (yy / 2 + rr) * 8 + x / 2; + avg_into(&mut u[o..o + w / 2], &tu[o..o + w / 2]); + avg_into(&mut v[o..o + w / 2], &tv[o..o + w / 2]); + } + } else { + self.pred_block(r1, (px, py), rect, dmv[1][q], PredOut { y: &mut y, u: &mut u, v: &mut v }); + } + } + } + let mut c = Cand::new(); + c.code.kind = MbKind::BDirect; + c.code.part = Part::P8x8; + // store per-8x8 refs/mvs + let refs = [[dref[0][0], dref[0][1], dref[0][2], dref[0][3]], [dref[1][0], dref[1][1], dref[1][2], dref[1][3]]]; + Self::cand_inter_info(&mut c, Part::P8x8, refs, dmv, [[Mv::ZERO; 4]; 2]); + c.code.part = Part::P16x16; + c.cost = satd(&src.y, 16, &y, 16, 16, 16) as u64 + ((lam * 2) >> 8) as u64; + self.code_inter(src, &y, &u, &v, qp, &mut c); + if c.code.cbp == 0 { + c.code.kind = MbKind::BSkip; + c.info.t8x8 = false; + // a skip is cheap; accept immediately when the prediction is good + let thresh = (f.qt.v4[(qp % 6) as usize][0] << (qp / 6)) as u64 * 24; + if sad16(&src.y, 16, &y, 16, 16) as u64 <= thresh { + return c; + } + } + best = Some(c); + } + // --- L0 / L1 16x16 + let mut res = [(Mv::ZERO, Mv::ZERO, u32::MAX); 2]; + for (l, rp) in [(0usize, r0), (1usize, r1)] { + let (m, p, c) = self.search_part(rp, &PartSearch { mx, my, part: Part::P16x16, list: l, lam, seed: dmv[l][0] }, src); + res[l] = (m[0], p[0], c); + } + let mut preds = [([0u8; 256], [0u8; 64], [0u8; 64]), ([0u8; 256], [0u8; 64], [0u8; 64])]; + for (l, rp) in [(0usize, r0), (1usize, r1)] { + let (a, b, c) = &mut preds[l]; + self.pred_block(rp, (px, py), (0, 0, 16, 16), res[l].0, PredOut { y: a, u: b, v: c }); + } + let mut bi = preds[0]; + avg_into(&mut bi.0, &preds[1].0); + avg_into(&mut bi.1, &preds[1].1); + avg_into(&mut bi.2, &preds[1].2); + let bits = |l: usize| mv_bits(res[l].0, res[l].1); + let costs = [ + res[0].2 as u64 + ((lam * 3) >> 8) as u64, + res[1].2 as u64 + ((lam * 3) >> 8) as u64, + satd(&src.y, 16, &bi.0, 16, 16, 16) as u64 + ((lam * (bits(0) + bits(1) + 5)) >> 8) as u64, + ]; + let dir = (0..3).min_by_key(|&d| costs[d]).unwrap_or(0); + let mut ci = Cand::new(); + ci.code.kind = MbKind::BInter; + ci.code.part = Part::P16x16; + ci.code.bdir[0] = dir as u8; + let mut refs = [[-1i8; 4]; 2]; + let mut mvs = [[Mv::ZERO; 4]; 2]; + let mut mvds = [[Mv::ZERO; 4]; 2]; + for l in 0..2 { + if dir == 2 || dir == l { + refs[l] = [0; 4]; + mvs[l] = [res[l].0; 4]; + mvds[l][0] = Mv::new(res[l].0.x as i32 - res[l].1.x as i32, res[l].0.y as i32 - res[l].1.y as i32); + } + } + ci.code.mvd = mvds; + Self::cand_inter_info(&mut ci, Part::P16x16, refs, mvs, mvds); + let (py_, pu, pv) = if dir == 2 { bi } else { preds[dir] }; + ci.cost = costs[dir]; + let better_than_direct = best.as_ref().is_none_or(|b| ci.cost < b.cost); + // Without RD only costs decide from here on: the 16x16 candidate's residual is coded at + // the end, if it is still the best then. + let mut deferred = false; + if better_than_direct || f.p.rd { + if f.p.rd { + self.code_inter(src, &py_, &pu, &pv, qp, &mut ci); + } else { + deferred = true; + } + best = Some(match best { + None => ci, + Some(b) => { + if f.p.rd { + if self.rd_cost(mx, my, &ci, src, qp) < self.rd_cost(mx, my, &b, src, qp) { ci } else { b } + } else { + ci + } + } + }); + } + let mut best = best.unwrap_or_else(|| self.analyse_intra(&MbCtx { mx, my, qp, av, e, src }, u64::MAX)); + // --- 16x8 / 8x16 partitions with a per-partition prediction direction + if f.p.partitions && best.cost > 256 * 4 { + for part in [Part::P16x8, Part::P8x16] { + // Without RD only the cost decides, so a losing candidate's residual is not coded. + let beat = (!f.p.rd).then_some(best.cost); + if let Some(c) = self.b_partition(&MbCtx { mx, my, qp, av, e, src }, part, lam, [res[0].0, res[1].0], beat) { + let better = if f.p.rd { self.rd_cost(mx, my, &c, src, qp) < self.rd_cost(mx, my, &best, src, qp) } else { c.cost < best.cost }; + if better { + best = c; + deferred = false; + } + } + } + } + let ctx = MbCtx { mx, my, qp, av, e, src }; + if let Some(intra) = self.better_intra_candidate(&ctx, &best, lam) { + return intra; + } + if deferred { + self.code_inter(src, &py_, &pu, &pv, qp, &mut best); + } + best + } + + fn better_intra_candidate(&self, ctx: &MbCtx<'_>, best: &Cand, lam: u32) -> Option> { + let threshold = (256 * 3 + 16 * lam as u64 / 256) * 3; + if !self.f.p.always_intra && best.cost <= threshold { + return None; + } + let intra = self.analyse_intra(ctx, best.cost); + let improves = if self.f.p.rd { + self.rd_cost(ctx.mx, ctx.my, &intra, ctx.src, ctx.qp) < self.rd_cost(ctx.mx, ctx.my, best, ctx.src, ctx.qp) + } else { + intra.cost < best.cost + }; + improves.then_some(intra) + } +} diff --git a/crates/h264enc/src/slice/intra.rs b/crates/h264enc/src/slice/intra.rs new file mode 100644 index 0000000000..5150ace103 --- /dev/null +++ b/crates/h264enc/src/slice/intra.rs @@ -0,0 +1,481 @@ +use super::{ + Avail, BLK4_XY, Cand, Deadzone, Edge4, Edge8, MbCtx, MbEdges, MbKind, RASTER_TO_BLK4, SliceEnc, Src, ZIGZAG4, ZIGZAG8, cavlc, dequant_luma_dc, + dequant4, dequant8, fdct4, fdct8, filter8, idct4, idct8, mode_available, pred_chroma, pred4x4, pred8x8, pred16x16, quant_luma_dc, quant4, quant8, + satd, +}; + +impl<'a, 'b> SliceEnc<'a, 'b> { + // ------------------------------------------------------------------ intra + + fn pred_intra_mode(&self, mx: usize, my: usize, av: Avail, local: &[i8; 16], bx: usize, by: usize) -> u8 { + let a = if bx > 0 { + Some(local[by * 4 + bx - 1]) + } else if av.left { + Some(self.mb(mx - 1, my).imodes[by * 4 + 3]) + } else { + None + }; + let b = if by > 0 { + Some(local[(by - 1) * 4 + bx]) + } else if av.top { + Some(self.mb(mx, my - 1).imodes[12 + bx]) + } else { + None + }; + match (a, b) { + (Some(a), Some(b)) => { + let a = if a < 0 { 2 } else { a }; + let b = if b < 0 { 2 } else { b }; + a.min(b) as u8 + } + _ => 2, + } + } + + /// Luma I16x16 coding. Returns luma cbp (0 or 15). + fn code_i16(&self, src: &[u8; 256], pred: &[u8; 256], qp: u8, c: &mut Cand) -> u8 { + let qt = self.f.qt; + let mut coefs = [[0i32; 16]; 16]; + let mut dc = [0i32; 16]; + for blk in 0..16 { + let (bx, by) = BLK4_XY[blk]; + let mut d = [0i32; 16]; + for y in 0..4 { + for x in 0..4 { + let i = (by * 4 + y) * 16 + bx * 4 + x; + d[y * 4 + x] = src[i] as i32 - pred[i] as i32; + } + } + let mut t = fdct4(&d); + dc[by * 4 + bx] = t[0]; + quant4(&mut t, qt, qp, Deadzone::INTRA, true); + if self.f.p.cavlc_clamp { + for v in t.iter_mut() { + *v = (*v).clamp(-cavlc::MAX_LEVEL, cavlc::MAX_LEVEL); + } + } + coefs[blk] = t; + } + let (mut dcl, _) = quant_luma_dc(&dc, qt, qp, Deadzone::INTRA); + if self.f.p.cavlc_clamp { + for v in dcl.iter_mut() { + *v = (*v).clamp(-cavlc::MAX_LEVEL, cavlc::MAX_LEVEL); + } + } + for k in 0..16 { + c.code.luma_dc[k] = dcl[ZIGZAG4[k]] as i16; + } + let ac_any = coefs.iter().any(|t| t[1..].iter().any(|&v| v != 0)); + let dcq = dequant_luma_dc(&dcl, qt, qp); + c.info.dc_cbf = (c.info.dc_cbf & !1) | dcl.iter().any(|&v| v != 0) as u8; + for blk in 0..16 { + let (bx, by) = BLK4_XY[blk]; + let mut t = coefs[blk]; + let mut scan = [0i16; 16]; + for k in 1..16 { + scan[k] = t[ZIGZAG4[k]] as i16; + } + c.code.luma4[blk] = if ac_any { scan } else { [0; 16] }; + let nz = if ac_any { scan.iter().filter(|&&v| v != 0).count() as u8 } else { 0 }; + c.info.nnz[by * 4 + bx] = nz; + let off = by * 4 * 16 + bx * 4; + if nz == 0 { + let dcv = (dcq[by * 4 + bx] + 32) >> 6; + for y in 0..4 { + for x in 0..4 { + let i = off + y * 16 + x; + c.ry[i] = (pred[i] as i32 + dcv).clamp(0, 255) as u8; + } + } + } else { + dequant4(&mut t, qt, qp, true); + t[0] = dcq[by * 4 + bx]; + let r = idct4(&t); + for y in 0..4 { + for x in 0..4 { + let i = off + y * 16 + x; + c.ry[i] = (pred[i] as i32 + r[y * 4 + x]).clamp(0, 255) as u8; + } + } + } + } + if ac_any { 15 } else { 0 } + } + + fn best_chroma_mode(&self, e: &MbEdges, av: Avail, src: &Src, lam: u32) -> (u8, [u8; 64], [u8; 64]) { + let mut best = (u32::MAX, 0u8, [0u8; 64], [0u8; 64]); + for mode in 0..4u8 { + let ok = match mode { + 0 => true, + 1 => av.left, + 2 => av.top, + _ => av.left && av.top && av.tl, + }; + if !ok { + continue; + } + let mut pu = [0u8; 64]; + let mut pv = [0u8; 64]; + pred_chroma(mode, &e.ctop[0], &e.cleft[0], e.ctl[0], av.top, av.left, &mut pu); + pred_chroma(mode, &e.ctop[1], &e.cleft[1], e.ctl[1], av.top, av.left, &mut pv); + let cost = satd(&src.u, 8, &pu, 8, 8, 8) + satd(&src.v, 8, &pv, 8, 8, 8) + ((lam * (1 + 2 * mode as u32)) >> 8); + if cost < best.0 { + best = (cost, mode, pu, pv); + } + } + (best.1, best.2, best.3) + } + + fn edge4(&self, e: &MbEdges, av: Avail, rec: &[u8; 256], bx: usize, by: usize) -> Edge4 { + let mut top = [0u8; 8]; + let mut left = [0u8; 4]; + let has_top = by > 0 || av.top; + let has_left = bx > 0 || av.left; + let has_tl = if bx > 0 && by > 0 { + true + } else if by == 0 && bx > 0 { + av.top + } else if bx == 0 && by > 0 { + av.left + } else { + av.tl + }; + let tr_avail = + if by == 0 { if bx < 3 { av.top } else { av.tr } } else { bx < 3 && RASTER_TO_BLK4[(by - 1) * 4 + bx + 1] < RASTER_TO_BLK4[by * 4 + bx] }; + if has_top { + for i in 0..4 { + top[i] = if by > 0 { rec[(by * 4 - 1) * 16 + bx * 4 + i] } else { e.top[bx * 4 + i] }; + } + for i in 4..8 { + top[i] = if tr_avail { if by > 0 { rec[(by * 4 - 1) * 16 + bx * 4 + i] } else { e.top[bx * 4 + i] } } else { top[3] }; + } + } + if has_left { + for i in 0..4 { + left[i] = if bx > 0 { rec[(by * 4 + i) * 16 + bx * 4 - 1] } else { e.left[by * 4 + i] }; + } + } + let tl = if !has_tl { + 0 + } else if bx > 0 && by > 0 { + rec[(by * 4 - 1) * 16 + bx * 4 - 1] + } else if by == 0 && bx > 0 { + e.top[bx * 4 - 1] + } else if bx == 0 && by > 0 { + e.left[by * 4 - 1] + } else { + e.tl + }; + Edge4 { top, left, tl, has_top, has_left, has_tl } + } + + fn edge8(&self, e: &MbEdges, av: Avail, rec: &[u8; 256], b8: usize) -> Edge8 { + let (x8, y8) = (b8 & 1, b8 >> 1); + let mut top = [0u8; 16]; + let mut left = [0u8; 8]; + let has_top = y8 > 0 || av.top; + let has_left = x8 > 0 || av.left; + let has_tl = match b8 { + 0 => av.tl, + 1 => av.top, + 2 => av.left, + _ => true, + }; + let tr_avail = match b8 { + 0 => av.top, + 1 => av.tr, + 2 => true, + _ => false, + }; + if has_top { + for i in 0..8 { + top[i] = if y8 > 0 { rec[7 * 16 + x8 * 8 + i] } else { e.top[x8 * 8 + i] }; + } + for i in 8..16 { + top[i] = if tr_avail { if y8 > 0 { rec[7 * 16 + x8 * 8 + i] } else { e.top[x8 * 8 + i] } } else { top[7] }; + } + } + if has_left { + for i in 0..8 { + left[i] = if x8 > 0 { rec[(y8 * 8 + i) * 16 + 7] } else { e.left[y8 * 8 + i] }; + } + } + let tl = if !has_tl { + 0 + } else { + match b8 { + 0 => e.tl, + 1 => e.top[7], + 2 => e.left[7], + _ => rec[7 * 16 + 7], + } + }; + Edge8 { top, left, tl, has_top, has_left, has_tl } + } + + /// Intra 4x4 analysis + coding. Returns cost (SATD domain) or None if it exceeded `limit`. + fn intra4x4(&self, m: &MbCtx, c: &mut Cand, limit: u64) -> Option { + let MbCtx { mx, my, qp, av, e, src } = *m; + let qt = self.f.qt; + let lam = self.f.lam.sad[qp as usize]; + let mut modes = [-1i8; 16]; + let mut total = ((lam * 24) >> 8) as u64; + for blk in 0..16 { + let (bx, by) = BLK4_XY[blk]; + let ed = self.edge4(e, av, &c.ry, bx, by); + let pm = self.pred_intra_mode(mx, my, av, &modes, bx, by); + let mut srcb = [0u8; 16]; + for y in 0..4 { + srcb[y * 4..y * 4 + 4].copy_from_slice(&src.y[(by * 4 + y) * 16 + bx * 4..(by * 4 + y) * 16 + bx * 4 + 4]); + } + let mut best = (u32::MAX, 2u8, [0u8; 16]); + for mode in 0..9u8 { + if !mode_available(mode, ed.has_top, ed.has_left, ed.has_tl) { + continue; + } + let mut p = [0u8; 16]; + pred4x4(mode, &ed, &mut p); + let bits = if mode == pm { 1 } else { 4 }; + let cost = satd(&srcb, 4, &p, 4, 4, 4) + ((lam * bits) >> 8); + if cost < best.0 { + best = (cost, mode, p); + } + } + total += best.0 as u64; + if total > limit { + return None; + } + let (mode, p) = (best.1, best.2); + modes[by * 4 + bx] = mode as i8; + c.code.ipred[blk] = if mode == pm { + -1 + } else if mode < pm { + mode as i8 + } else { + mode as i8 - 1 + }; + // code block + let mut d = [0i32; 16]; + for i in 0..16 { + d[i] = srcb[i] as i32 - p[i] as i32; + } + let mut t = fdct4(&d); + let nz = quant4(&mut t, qt, qp, Deadzone::INTRA, false); + if self.f.p.cavlc_clamp { + for v in t.iter_mut() { + *v = (*v).clamp(-cavlc::MAX_LEVEL, cavlc::MAX_LEVEL); + } + } + let mut scan = [0i16; 16]; + for k in 0..16 { + scan[k] = t[ZIGZAG4[k]] as i16; + } + c.code.luma4[blk] = scan; + c.info.nnz[by * 4 + bx] = nz as u8; + let off = by * 4 * 16 + bx * 4; + if nz == 0 { + for y in 0..4 { + c.ry[off + y * 16..off + y * 16 + 4].copy_from_slice(&p[y * 4..y * 4 + 4]); + } + } else { + dequant4(&mut t, qt, qp, false); + let r = idct4(&t); + for y in 0..4 { + for x in 0..4 { + c.ry[off + y * 16 + x] = (p[y * 4 + x] as i32 + r[y * 4 + x]).clamp(0, 255) as u8; + } + } + } + } + c.info.imodes = modes; + let mut cbp = 0; + for b8 in 0..4 { + if (0..4).any(|i| c.code.luma4[b8 * 4 + i].iter().any(|&v| v != 0)) { + cbp |= 1 << b8; + } + } + c.code.cbp = cbp; + c.code.kind = MbKind::I4x4; + c.code.t8x8 = false; + c.info.t8x8 = false; + Some(total) + } + + /// Intra 8x8 analysis + coding. + fn intra8x8(&self, m: &MbCtx, c: &mut Cand, limit: u64) -> Option { + let MbCtx { mx, my, qp, av, e, src } = *m; + let qt = self.f.qt; + let lam = self.f.lam.sad[qp as usize]; + let mut modes = [-1i8; 16]; + let mut total = ((lam * 12) >> 8) as u64; + for b8 in 0..4 { + let (x8, y8) = (b8 & 1, b8 >> 1); + let ed = self.edge8(e, av, &c.ry, b8); + let fe = filter8(&ed); + let pm = self.pred_intra_mode(mx, my, av, &modes, x8 * 2, y8 * 2); + let mut srcb = [0u8; 64]; + for y in 0..8 { + srcb[y * 8..y * 8 + 8].copy_from_slice(&src.y[(y8 * 8 + y) * 16 + x8 * 8..(y8 * 8 + y) * 16 + x8 * 8 + 8]); + } + let mut best = (u32::MAX, 2u8, [0u8; 64]); + for mode in 0..9u8 { + if !mode_available(mode, ed.has_top, ed.has_left, ed.has_tl) { + continue; + } + let mut p = [0u8; 64]; + pred8x8(mode, &fe, &mut p); + let bits = if mode == pm { 1 } else { 4 }; + let cost = satd(&srcb, 8, &p, 8, 8, 8) + ((lam * bits) >> 8); + if cost < best.0 { + best = (cost, mode, p); + } + } + total += best.0 as u64; + if total > limit { + return None; + } + let (mode, p) = (best.1, best.2); + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + modes[(y8 * 2 + dy) * 4 + x8 * 2 + dx] = mode as i8; + } + c.code.ipred[b8] = if mode == pm { + -1 + } else if mode < pm { + mode as i8 + } else { + mode as i8 - 1 + }; + let mut d = [0i32; 64]; + for i in 0..64 { + d[i] = srcb[i] as i32 - p[i] as i32; + } + let mut t = fdct8(&d); + let nz = quant8(&mut t, qt, qp, Deadzone::INTRA); + let mut scan = [0i16; 64]; + for k in 0..64 { + scan[k] = t[ZIGZAG8[k]] as i16; + } + c.code.luma8[b8] = scan; + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + c.info.nnz[(y8 * 2 + dy) * 4 + x8 * 2 + dx] = nz as u8; + } + let off = y8 * 8 * 16 + x8 * 8; + if nz == 0 { + for y in 0..8 { + c.ry[off + y * 16..off + y * 16 + 8].copy_from_slice(&p[y * 8..y * 8 + 8]); + } + } else { + dequant8(&mut t, qt, qp); + let r = idct8(&t); + for y in 0..8 { + for x in 0..8 { + c.ry[off + y * 16 + x] = (p[y * 8 + x] as i32 + r[y * 8 + x]).clamp(0, 255) as u8; + } + } + } + } + c.info.imodes = modes; + let mut cbp = 0; + for b8 in 0..4 { + if c.code.luma8[b8].iter().any(|&v| v != 0) { + cbp |= 1 << b8; + } + } + c.code.cbp = cbp; + c.code.kind = MbKind::I8x8; + c.code.t8x8 = true; + c.info.t8x8 = true; + Some(total) + } + + /// Full intra analysis; returns the best coded intra candidate. `inter_cost` lets P/B skip expensive modes. + pub(super) fn analyse_intra(&self, m: &MbCtx, inter_cost: u64) -> Box { + let MbCtx { mx, my, qp, av, e, src } = *m; + let lam = self.f.lam.sad[qp as usize]; + let p = self.f.p; + // I16x16 mode + let mut top16 = [0u8; 16]; + top16.copy_from_slice(&e.top[..16]); + let mut b16 = (u32::MAX, 2u8, [0u8; 256]); + for mode in 0..4u8 { + let ok = match mode { + 0 => av.top, + 1 => av.left, + 2 => true, + _ => av.top && av.left && av.tl, + }; + if !ok { + continue; + } + let mut pr = [0u8; 256]; + pred16x16(mode, &top16, &e.left, e.tl, av.top, av.left, &mut pr); + let cost = satd(&src.y, 16, &pr, 16, 16, 16); + if cost < b16.0 { + b16 = (cost, mode, pr); + } + } + let cost16 = b16.0 as u64 + ((lam * 6) >> 8) as u64; + let (cmode, pu, pv) = self.best_chroma_mode(e, av, src, lam); + let mk16 = || { + let mut c = Cand::new(); + c.code.kind = MbKind::I16x16; + c.code.i16_mode = b16.1; + c.code.chroma_mode = cmode; + let cbpl = self.code_i16(&src.y, &b16.2, qp, &mut c); + let cbpc = self.code_chroma(src, &pu, &pv, qp, true, &mut c); + c.code.cbp = cbpl | (cbpc << 4); + c.info.chroma_mode = cmode; + c.info.imodes = [-1; 16]; + c.cost = cost16; + c + }; + let mut best: Option> = None; + let try_nxn = p.i4x4 && (inter_cost == u64::MAX || cost16 < inter_cost + inter_cost / 4 || p.always_intra); + if !try_nxn { + return mk16(); + } + let limit = cost16.min(inter_cost.saturating_add(inter_cost / 8)); + let mut limit = if p.rd { cost16 + cost16 / 4 } else { limit }; + if p.i8x8 && p.t8x8 { + let mut c = Cand::new(); + if let Some(cost) = self.intra8x8(m, &mut c, limit) { + c.cost = cost; + limit = limit.min(cost); + best = Some(c); + } + } + { + let mut c = Cand::new(); + if let Some(cost) = self.intra4x4(m, &mut c, limit) { + c.cost = cost; + if best.as_ref().is_none_or(|b| cost < b.cost) { + best = Some(c); + } + } + } + let finish = |mut c: Box| { + c.code.chroma_mode = cmode; + c.info.chroma_mode = cmode; + let cbpc = self.code_chroma(src, &pu, &pv, qp, true, &mut c); + c.code.cbp = (c.code.cbp & 15) | (cbpc << 4); + c + }; + match best { + None => mk16(), + Some(c) => { + let c = finish(c); + if p.rd { + let c16 = mk16(); + let (j1, j2) = (self.rd_cost(mx, my, &c, src, qp), self.rd_cost(mx, my, &c16, src, qp)); + let mut w = if j1 <= j2 { c } else { c16 }; + w.cost = w.cost.min(cost16); + w + } else if c.cost < cost16 { + c + } else { + mk16() + } + } + } + } +} diff --git a/crates/h264enc/src/slice/macroblock.rs b/crates/h264enc/src/slice/macroblock.rs new file mode 100644 index 0000000000..42151bacee --- /dev/null +++ b/crates/h264enc/src/slice/macroblock.rs @@ -0,0 +1,148 @@ +use super::{BitWriter, CabacEstimator, Cand, MbCtx, MbKind, NbCtx, SliceEnc, SliceType, Src, cabac_mb, cavlc, ssd}; + +impl<'a, 'b> SliceEnc<'a, 'b> { + // ------------------------------------------------------------------ macroblock + + pub(super) fn encode_mb(&mut self, mx: usize, my: usize) { + let f = self.f; + let mbw = f.p.mbw; + let aq = if f.aq.is_empty() { 0 } else { f.aq[my * mbw + mx] as i32 }; + let qp = (f.qp as i32 + aq).clamp(0, 51) as u8; + let av = self.avail(mx, my); + let (px, py) = (mx * 16, my * 16); + let mut src_y = [0u8; 256]; + let mut src_u = [0u8; 64]; + let mut src_v = [0u8; 64]; + { + let s = &f.src.y; + for r in 0..16 { + let i = s.idx(px as isize, (py + r) as isize); + src_y[r * 16..r * 16 + 16].copy_from_slice(&s.data[i..i + 16]); + } + for r in 0..8 { + let i = f.src.u.idx((px / 2) as isize, (py / 2 + r) as isize); + src_u[r * 8..r * 8 + 8].copy_from_slice(&f.src.u.data[i..i + 8]); + let i = f.src.v.idx((px / 2) as isize, (py / 2 + r) as isize); + src_v[r * 8..r * 8 + 8].copy_from_slice(&f.src.v.data[i..i + 8]); + } + } + let src = Src { y: src_y, u: src_u, v: src_v }; + let edges = self.edges(mx, my, av); + let best = match f.st { + SliceType::I => self.analyse_intra(&MbCtx { mx, my, qp, av, e: &edges, src: &src }, u64::MAX), + SliceType::P => self.analyse_p(mx, my, qp, av, &edges, &src), + SliceType::B => self.analyse_b(mx, my, qp, av, &edges, &src), + }; + self.commit(mx, my, qp, *best); + } + + fn commit(&mut self, mx: usize, my: usize, qp: u8, mut c: Cand) { + let (px, py) = (mx * 16, my * 16); + // reconstruction + for r in 0..16 { + let i = self.ry(px, py + r); + self.band.y[i..i + 16].copy_from_slice(&c.ry[r * 16..r * 16 + 16]); + } + for r in 0..8 { + let i = self.rc(px / 2, py / 2 + r); + self.band.u[i..i + 8].copy_from_slice(&c.ru[r * 8..r * 8 + 8]); + self.band.v[i..i + 8].copy_from_slice(&c.rv[r * 8..r * 8 + 8]); + } + let kind = c.code.kind; + let has_dqp = !kind.is_skip() && (c.code.cbp != 0 || kind == MbKind::I16x16); + let prev_dqp_nz = self.prev_dqp_nz; + if has_dqp { + let mut d = qp as i32 - self.last_qp as i32; + if d < -26 { + d += 52; + } else if d > 25 { + d -= 52; + } + c.code.qp_delta = d; + self.prev_dqp_nz = d != 0; + self.last_qp = qp; + } else { + c.code.qp_delta = 0; + self.prev_dqp_nz = false; + } + c.info.qp = self.last_qp; + c.info.slice = self.slice_no + 1; + c.info.kind = kind; + c.info.cbp = if kind.is_skip() { 0 } else { c.code.cbp }; + self.stats.mbs += 1; + self.stats.qp_sum += c.info.qp as u64; + if kind.is_intra() { + self.stats.intra += 1; + } else if kind.is_skip() { + self.stats.skip += 1; + } else { + self.stats.inter += 1; + } + let idx = (my - self.y0) * self.f.p.mbw + mx; + self.mbs[idx] = c.info; + // entropy coding + let av = self.avail(mx, my); + let left = if av.left { Some(&self.mbs[idx - 1]) } else { None }; + let top = if av.top { Some(&self.mbs[idx - self.f.p.mbw]) } else { None }; + let nb = NbCtx { left, top, cur: &self.mbs[idx], slice_type: self.f.st, prev_dqp_nz, t8x8_mode: self.f.p.t8x8 }; + if let Some(cab) = self.cabac.as_mut() { + if nb.slice_type != SliceType::I { + cabac_mb::write_skip_flag(cab, &nb, kind.is_skip()); + } + if !kind.is_skip() { + cabac_mb::write_mb(cab, &nb, &c.code); + } + } else if let Some(w) = self.cavlc.as_mut() { + if kind.is_skip() { + self.skip_run += 1; + } else { + if nb.slice_type != SliceType::I { + w.write_ue(self.skip_run); + self.skip_run = 0; + } + cavlc::write_mb(w, &nb, &c.code); + } + } + } + + /// Estimated bits (1/256 units) of coding candidate `c` at the current entropy state. + fn est_bits(&self, mx: usize, my: usize, c: &Cand, qp: u8) -> u64 { + let av = self.avail(mx, my); + let idx = (my - self.y0) * self.f.p.mbw + mx; + let left = if av.left { Some(&self.mbs[idx - 1]) } else { None }; + let top = if av.top { Some(&self.mbs[idx - self.f.p.mbw]) } else { None }; + let mut info = c.info; + info.kind = c.code.kind; + info.cbp = c.code.cbp; + let mut code = c.code.clone(); + code.qp_delta = qp as i32 - self.last_qp as i32; + let nb = NbCtx { left, top, cur: &info, slice_type: self.f.st, prev_dqp_nz: self.prev_dqp_nz, t8x8_mode: self.f.p.t8x8 }; + if let Some(cab) = self.cabac.as_ref() { + let mut est = CabacEstimator::new(cab.ctx.clone(), self.f.cost_tab); + if nb.slice_type != SliceType::I { + cabac_mb::write_skip_flag(&mut est, &nb, code.kind.is_skip()); + } + if !code.kind.is_skip() { + cabac_mb::write_mb(&mut est, &nb, &code); + } + est.cost + } else { + if code.kind.is_skip() { + return 256; + } + let mut w = BitWriter::new(); + if nb.slice_type != SliceType::I { + w.write_ue(self.skip_run); + } + cavlc::write_mb(&mut w, &nb, &code); + w.bit_len() as u64 * 256 + } + } + + pub(super) fn rd_cost(&self, mx: usize, my: usize, c: &Cand, src: &Src, qp: u8) -> u64 { + let d = ssd(&src.y, 16, &c.ry, 16, 16, 16) + ssd(&src.u, 8, &c.ru, 8, 8, 8) + ssd(&src.v, 8, &c.rv, 8, 8, 8); + let bits = self.est_bits(mx, my, c, qp); + let lam = self.f.lam.ssd[qp as usize]; + (d as f64 * 256.0 + lam * bits as f64) as u64 + } +} diff --git a/crates/h264enc/src/slice/motion.rs b/crates/h264enc/src/slice/motion.rs new file mode 100644 index 0000000000..3ec86bc237 --- /dev/null +++ b/crates/h264enc/src/slice/motion.rs @@ -0,0 +1,386 @@ +use super::{ + Avail, Cand, MbCtx, MbEdges, MbInfo, MbKind, Mv, MvNb, Part, PartGeom, PartSearch, PredOut, RefPic, SearchBlock, Shape, SliceEnc, Src, mv_bits, + sad, sad8, sad16, satd, +}; +use crate::picture::McBlock; + +/// Motion vectors already costed in one search: one bit per point of the square of +/// `2 * half + 1` points (`step` quarter samples apart) around `centre`. A point outside the +/// square, off the grid or past the `W` words always counts as new. +struct Visited { + centre: Mv, + half: i32, + step: i32, + bits: [u64; W], +} + +impl Visited { + fn new(centre: Mv, half: i32, step: i32) -> Self { + Visited { centre, half, step: step.max(1), bits: [0; W] } + } + + /// Marks `m`; false when it was marked before. + fn first(&mut self, m: Mv) -> bool { + let (dx, dy) = (m.x as i32 - self.centre.x as i32, m.y as i32 - self.centre.y as i32); + if dx % self.step != 0 || dy % self.step != 0 { + return true; + } + let side = 2 * self.half + 1; + let (gx, gy) = (dx / self.step + self.half, dy / self.step + self.half); + if gx < 0 || gy < 0 || gx >= side || gy >= side { + return true; + } + let i = (gy * side + gx) as usize; + let Some(word) = self.bits.get_mut(i / 64) else { return true }; + let bit = 1u64 << (i % 64); + let fresh = *word & bit == 0; + *word |= bit; + fresh + } +} + +impl<'a, 'b> SliceEnc<'a, 'b> { + // ------------------------------------------------------------------ motion estimation + + /// Integer + sub-pel search for a w x h block at picture (bx, by). Returns (mv, cost, satd). + fn search(&self, r: &RefPic, blk: &SearchBlock, mvp: Mv, cands: &[Mv], lam: u32) -> (Mv, u32) { + let SearchBlock { x: bx, y: by, w, h, src, sstride } = *blk; + let pl = &r.frame.y; + let p = self.f.p; + let fcost = |mv: Mv| -> u32 { + let i = pl.idx(bx as isize + (mv.x as isize >> 2), by as isize + (mv.y as isize >> 2)); + let d = match w { + 16 => sad16(src, sstride, &pl.data[i..], pl.stride, h), + 8 => sad8(src, sstride, &pl.data[i..], pl.stride, h), + _ => sad(src, sstride, &pl.data[i..], pl.stride, w, h), + }; + d + ((lam * mv_bits(mv, mvp)) >> 8) + }; + let mut best = Mv::new((mvp.x as i32 + 2) & !3, (mvp.y as i32 + 2) & !3); + if !self.mv_ok(bx, by, w, h, best) { + best = Mv::ZERO; + } + let mut bcost = fcost(best); + for &c in cands { + let c = Mv::new((c.x as i32 + 2) & !3, (c.y as i32 + 2) & !3); + if c != best && self.mv_ok(bx, by, w, h, c) { + let k = fcost(c); + if k < bcost { + bcost = k; + best = c; + } + } + } + // hexagon search (full-pel units = 4 quarter samples) + let range = p.me_range; + let centre0 = best; + // a point costed once is never better than the best since: skip it when it comes up again + let mut seen = Visited::<40>::new(centre0, range, 4); + seen.first(centre0); + const HEX: [(i32, i32); 6] = [(-2, 0), (2, 0), (-1, -2), (1, -2), (-1, 2), (1, 2)]; + for _ in 0..range { + let mut moved = false; + let c = best; + for &(dx, dy) in &HEX { + let m = Mv::new(c.x as i32 + dx * 4, c.y as i32 + dy * 4); + if ((m.x as i32 - centre0.x as i32).abs() >> 2) > range + || ((m.y as i32 - centre0.y as i32).abs() >> 2) > range + || !self.mv_ok(bx, by, w, h, m) + || !seen.first(m) + { + continue; + } + let k = fcost(m); + if k < bcost { + bcost = k; + best = m; + moved = true; + } + } + if !moved { + break; + } + } + // small diamond / square refinement + for _ in 0..2 { + let c = best; + for (dx, dy) in [(-1, 0), (1, 0), (0, -1), (0, 1), (-1, -1), (1, -1), (-1, 1), (1, 1)] { + let m = Mv::new(c.x as i32 + dx * 4, c.y as i32 + dy * 4); + if !self.mv_ok(bx, by, w, h, m) || !seen.first(m) { + continue; + } + let k = fcost(m); + if k < bcost { + bcost = k; + best = m; + } + } + if best == c { + break; + } + } + // sub-pel refinement with SATD + let mut buf = [0u8; 256]; + let scost = |mv: Mv, buf: &mut [u8; 256]| -> u32 { + let (pr, ps) = r.luma_ref(bx, by, mv, w, h, buf); + satd(src, sstride, pr, ps, w, h) + ((lam * mv_bits(mv, mvp)) >> 8) + }; + let mut bs = scost(best, &mut buf); + let mut seen = Visited::<2>::new(best, 4, 1); + seen.first(best); + if p.subpel >= 1 { + for step in [2, 1] { + if step == 1 && p.subpel < 2 { + break; + } + let rounds = if step == 1 { 1 + p.subpel_rounds as usize } else { 1 }; + for _ in 0..rounds { + let c = best; + for (dx, dy) in [(-1, 0), (1, 0), (0, -1), (0, 1), (-1, -1), (1, -1), (-1, 1), (1, 1)] { + let m = Mv::new(c.x as i32 + dx * step, c.y as i32 + dy * step); + if !self.mv_ok(bx, by, w, h, m) || !seen.first(m) { + continue; + } + let k = scost(m, &mut buf); + if k < bs { + bs = k; + best = m; + } + } + if best == c { + break; + } + } + } + } + (best, bs) + } + + /// Motion-compensated luma + chroma prediction of `rect` (x, y, w, h within the macroblock whose + /// top-left luma sample is `origin`) into `out`. + pub(super) fn pred_block(&self, r: &RefPic, origin: (usize, usize), rect: (usize, usize, usize, usize), mv: Mv, out: PredOut) { + let (px, py) = origin; + let (x, yy, w, h) = rect; + r.mc_luma(McBlock { x: px + x, y: py + yy, w, h }, mv, &mut out.y[yy * 16 + x..], 16); + let co = (yy / 2) * 8 + x / 2; + r.mc_chroma(McBlock { x: (px + x) / 2, y: (py + yy) / 2, w: w / 2, h: h / 2 }, mv, (&mut out.u[co..], &mut out.v[co..]), 8); + } + + pub(super) fn cand_inter_info(c: &mut Cand, part: Part, refs: [[i8; 4]; 2], mvs: [[Mv; 4]; 2], mvds: [[Mv; 4]; 2]) { + for p in 0..part.count() { + let (x, y, w, h) = part.rect(p); + for by in y / 4..(y + h) / 4 { + for bx in x / 4..(x + w) / 4 { + let r = by * 4 + bx; + for l in 0..2 { + c.info.mv[l][r] = if refs[l][p] >= 0 { mvs[l][p] } else { Mv::ZERO }; + let d = if refs[l][p] >= 0 { mvds[l][p] } else { Mv::ZERO }; + c.info.mvd[l][r] = [d.x.unsigned_abs().min(127) as u8, d.y.unsigned_abs().min(127) as u8]; + } + } + } + for by8 in y / 8..(y + h).div_ceil(8) { + for bx8 in x / 8..(x + w).div_ceil(8) { + for l in 0..2 { + c.info.ref_idx[l][by8 * 2 + bx8] = refs[l][p]; + } + } + } + } + } + + pub(super) fn mask_of(part: Part, upto: usize) -> u16 { + let mut m = 0u16; + for p in 0..upto { + let (x, y, w, h) = part.rect(p); + for by in y / 4..(y + h) / 4 { + for bx in x / 4..(x + w) / 4 { + m |= 1 << (by * 4 + bx); + } + } + } + m + } + + pub(super) fn shape_of(part: Part, p: usize) -> Shape { + match (part, p) { + (Part::P16x8, 0) => Shape::Top16x8, + (Part::P16x8, _) => Shape::Bottom16x8, + (Part::P8x16, 0) => Shape::Left8x16, + (Part::P8x16, _) => Shape::Right8x16, + _ => Shape::Normal, + } + } + + /// Search each partition of `ps.part` for list `ps.list` sequentially (mvp depends on earlier partitions). + pub(super) fn search_part(&self, r: &RefPic, ps: &PartSearch, src: &Src) -> ([Mv; 4], [Mv; 4], u32) { + let PartSearch { mx, my, part, list, lam, seed } = *ps; + let (px, py) = (mx * 16, my * 16); + let mut info = MbInfo::default(); + let mut mvs = [Mv::ZERO; 4]; + let mut mvps = [Mv::ZERO; 4]; + let mut total = 0; + for p in 0..part.count() { + let (x, y, w, h) = part.rect(p); + let nb = MvNb { cur: &info, mask: Self::mask_of(part, p), mx, my }; + let geom = PartGeom { x: (x / 4) as i32, y: (y / 4) as i32, w: (w / 4) as i32, shape: Self::shape_of(part, p) }; + let mvp = self.mvp(nb, geom, list, 0); + let blk = SearchBlock { x: px + x, y: py + y, w, h, src: &src.y[y * 16 + x..], sstride: 16 }; + let (mv, cost) = self.search(r, &blk, mvp, &[seed, mvs[0]], lam); + mvs[p] = mv; + mvps[p] = mvp; + total += cost; + // record in scratch info for the next partition's predictor + for by in y / 4..(y + h) / 4 { + for bx in x / 4..(x + w) / 4 { + info.mv[list][by * 4 + bx] = mv; + } + } + for by8 in y / 8..(y + h).div_ceil(8) { + for bx8 in x / 8..(x + w).div_ceil(8) { + info.ref_idx[list][by8 * 2 + bx8] = 0; + } + } + } + (mvs, mvps, total) + } + + pub(super) fn analyse_p(&self, mx: usize, my: usize, qp: u8, av: Avail, e: &MbEdges, src: &Src) -> Box { + let f = self.f; + // P slices always have a reference; without one the macroblock is coded intra. + let Some(r) = f.l0 else { return self.analyse_intra(&MbCtx { mx, my, qp, av, e, src }, u64::MAX) }; + let lam = f.lam.sad[qp as usize]; + let (px, py) = (mx * 16, my * 16); + let skip_mv = self.pskip_mv(mx, my); + let skip_ok = self.mv_ok(px, py, 16, 16, skip_mv); + // --- try P_Skip early + let mut skip_pred = None; + if skip_ok { + let mut y = [0u8; 256]; + let mut u = [0u8; 64]; + let mut v = [0u8; 64]; + self.pred_block(r, (px, py), (0, 0, 16, 16), skip_mv, PredOut { y: &mut y, u: &mut u, v: &mut v }); + let s = sad16(&src.y, 16, &y, 16, 16); + let qstep_thresh = (f.qt.v4[(qp % 6) as usize][0] << (qp / 6)) as u32 * 16; + if s < qstep_thresh * 2 { + let mut c = Cand::new(); + c.code.kind = MbKind::PInter; + c.code.part = Part::P16x16; + self.code_inter(src, &y, &u, &v, qp, &mut c); + if c.code.cbp == 0 { + c.code.kind = MbKind::PSkip; + Self::cand_inter_info(&mut c, Part::P16x16, [[0; 4], [-1; 4]], [[skip_mv; 4], [Mv::ZERO; 4]], [[Mv::ZERO; 4]; 2]); + c.info.nnz = [0; 16]; + c.info.nnz_c = [0; 8]; + c.info.dc_cbf = 0; + c.info.t8x8 = false; + c.ry = y; + c.ru = u; + c.rv = v; + return c; + } + } + skip_pred = Some((y, u, v)); + } + // --- 16x16 search + let mut cands = vec![skip_mv, Mv::ZERO]; + let colo = r.mbs[my * f.p.mbw + mx]; + if colo.ref_idx[0][0] >= 0 { + cands.push(colo.mv[0][0]); + } + let (mv16, mvp16, cost16) = { + let (m, p, c) = self.search_part(r, &PartSearch { mx, my, part: Part::P16x16, list: 0, lam, seed: skip_mv }, src); + (m, p, c) + }; + let _ = cands; + let mut best_part = Part::P16x16; + let mut best_mvs = mv16; + let mut best_mvps = mvp16; + let mut best_cost = cost16 as u64 + (lam >> 8) as u64; + if f.p.partitions { + let big = cost16 as u64 > (256 * 4) as u64; + if big { + for part in [Part::P16x8, Part::P8x16, Part::P8x8] { + let (m, p, c) = self.search_part(r, &PartSearch { mx, my, part, list: 0, lam, seed: mv16[0] }, src); + let extra = match part { + Part::P8x8 => 10, + _ => 3, + }; + let c = c as u64 + ((lam * extra) >> 8) as u64; + if c < best_cost { + best_cost = c; + best_part = part; + best_mvs = m; + best_mvps = p; + } + } + } + } + // --- build inter candidate + let mut ci = Cand::new(); + ci.code.kind = MbKind::PInter; + ci.code.part = best_part; + let mut py_ = [0u8; 256]; + let mut pu = [0u8; 64]; + let mut pv = [0u8; 64]; + if let (Part::P16x16, true, Some((a, b, c))) = (best_part, best_mvs[0] == skip_mv, skip_pred) { + py_ = a; + pu = b; + pv = c; + } else { + for p in 0..best_part.count() { + self.pred_block(r, (px, py), best_part.rect(p), best_mvs[p], PredOut { y: &mut py_, u: &mut pu, v: &mut pv }); + } + } + let mut mvds = [[Mv::ZERO; 4]; 2]; + for p in 0..best_part.count() { + mvds[0][p] = Mv::new(best_mvs[p].x as i32 - best_mvps[p].x as i32, best_mvs[p].y as i32 - best_mvps[p].y as i32); + } + ci.code.mvd = mvds; + Self::cand_inter_info(&mut ci, best_part, [[0; 4], [-1; 4]], [best_mvs, [Mv::ZERO; 4]], mvds); + // The residual decides a P_Skip and the RD comparisons; otherwise it only matters when the + // inter candidate is kept, so it is coded after an intra candidate had its chance. + let skip_candidate = best_part == Part::P16x16 && best_mvs[0] == skip_mv && skip_ok; + let coded_now = skip_candidate || f.p.rd; + if coded_now { + self.code_inter(src, &py_, &pu, &pv, qp, &mut ci); + } + ci.cost = best_cost; + if skip_candidate && ci.code.cbp == 0 { + ci.code.kind = MbKind::PSkip; + ci.info.t8x8 = false; + ci.info.mvd = [[[0; 2]; 16]; 2]; + return ci; + } + // --- intra + let need_intra = f.p.always_intra || best_cost > (256 * 3 + 16 * lam as u64 / 256) * 2; + if need_intra { + let ic = self.analyse_intra(&MbCtx { mx, my, qp, av, e, src }, best_cost); + if f.p.rd { + if self.rd_cost(mx, my, &ic, src, qp) < self.rd_cost(mx, my, &ci, src, qp) { + return ic; + } + } else if ic.cost < best_cost { + return ic; + } + } + if !coded_now { + self.code_inter(src, &py_, &pu, &pv, qp, &mut ci); + } + if f.p.rd && skip_ok && ci.code.cbp != 0 { + // RD check against P_Skip + if let Some((y, u, v)) = skip_pred { + let mut sc = Cand::new(); + sc.code.kind = MbKind::PSkip; + Self::cand_inter_info(&mut sc, Part::P16x16, [[0; 4], [-1; 4]], [[skip_mv; 4], [Mv::ZERO; 4]], [[Mv::ZERO; 4]; 2]); + sc.ry = y; + sc.ru = u; + sc.rv = v; + if self.rd_cost(mx, my, &sc, src, qp) < self.rd_cost(mx, my, &ci, src, qp) { + return sc; + } + } + } + ci + } +} diff --git a/crates/h264enc/src/slice/residual.rs b/crates/h264enc/src/slice/residual.rs new file mode 100644 index 0000000000..b0c463ffc1 --- /dev/null +++ b/crates/h264enc/src/slice/residual.rs @@ -0,0 +1,320 @@ +use super::{ + BLK4_XY, Cand, Deadzone, SliceEnc, Src, ZIGZAG4, ZIGZAG8, cavlc, chroma_qp, decimate_score, dequant_chroma_dc, dequant4, dequant8, fdct4, fdct8, + idct4, idct8, quant_chroma_dc, quant4, quant8, ssd, +}; + +/// The transform trial's bit estimate of a level `v`: `0.4` for zero, `3 + 2·log2|v|` otherwise. +/// Levels below 256 (nearly all) read it from a table of exactly those values. +#[inline] +fn level_bits(table: &[f64; 256], v: i16) -> f64 { + let a = v.unsigned_abs(); + match table.get(usize::from(a)) { + Some(&b) => b, + None => 3.0 + 2.0 * (a as f64).log2(), + } +} + +fn level_bits_table() -> &'static [f64; 256] { + static T: std::sync::OnceLock<[f64; 256]> = std::sync::OnceLock::new(); + T.get_or_init(|| std::array::from_fn(|a| if a == 0 { 0.4 } else { 3.0 + 2.0 * (a as f64).log2() })) +} + +/// `rec = clamp(pred + residual(x, y))` for the 4x4 chroma block at (`bx`, `by`) of an 8x8 chroma plane. +fn add_chroma_residual(rec: &mut [u8], pred: &[u8; 64], bx: usize, by: usize, residual: impl Fn(usize, usize) -> i32) { + for y in 0..4 { + for x in 0..4 { + let i = (by + y) * 8 + bx + x; + rec[i] = (pred[i] as i32 + residual(x, y)).clamp(0, 255) as u8; + } + } +} + +impl<'a, 'b> SliceEnc<'a, 'b> { + // ------------------------------------------------------------------ residual coding helpers + + /// Transform + quantise chroma 4x4 block `b` of a component. Returns (unquantised DC, quantised + /// coefficients in raster order, AC coefficients in scan order). + fn quant_chroma_ac_block(&self, s: &[u8; 64], p: &[u8; 64], b: usize, qpc: u8, dz: Deadzone) -> (i32, [i32; 16], [i16; 16]) { + let (bx, by) = ((b & 1) * 4, (b >> 1) * 4); + let mut d = [0i32; 16]; + for y in 0..4 { + for x in 0..4 { + d[y * 4 + x] = s[(by + y) * 8 + bx + x] as i32 - p[(by + y) * 8 + bx + x] as i32; + } + } + let mut t = fdct4(&d); + let dc = t[0]; + quant4(&mut t, self.f.qt, qpc, dz, true); + if self.f.p.cavlc_clamp { + for v in t.iter_mut().skip(1) { + *v = (*v).clamp(-cavlc::MAX_LEVEL, cavlc::MAX_LEVEL); + } + } + let mut scan = [0i16; 16]; + for k in 1..16 { + scan[k] = t[ZIGZAG4[k]] as i16; + } + (dc, t, scan) + } + + /// Code chroma residual for prediction `pu`, `pv`; fills code/info and reconstruction. Returns chroma cbp. + pub(super) fn code_chroma(&self, src: &Src, pu: &[u8; 64], pv: &[u8; 64], qp: u8, intra: bool, c: &mut Cand) -> u8 { + let qt = self.f.qt; + let qpc = chroma_qp(qp as i32 + self.f.p.chroma_qp_offset); + let dz = if intra { Deadzone::INTRA } else { Deadzone::INTER }; + let mut dc_any = false; + let mut ac_any = false; + let mut coefs = [[[0i32; 16]; 4]; 2]; + let mut dcl = [[0i32; 4]; 2]; + for comp in 0..2 { + let (s, p) = if comp == 0 { (&src.u, pu) } else { (&src.v, pv) }; + let mut dc = [0i32; 4]; + let mut ac_score = 0; + for b in 0..4 { + let (dc_coef, t, scan) = self.quant_chroma_ac_block(s, p, b, qpc, dz); + dc[b] = dc_coef; + ac_score += decimate_score(&scan[1..]); + c.code.cac[comp * 4 + b] = scan; + coefs[comp][b] = t; + } + if !intra && self.f.p.decimate && ac_score < 7 { + for b in 0..4 { + c.code.cac[comp * 4 + b] = [0; 16]; + for k in 1..16 { + coefs[comp][b][k] = 0; + } + } + } + let (mut lv, _) = quant_chroma_dc(&dc, qt, qpc, dz); + if self.f.p.cavlc_clamp { + for v in lv.iter_mut() { + *v = (*v).clamp(-cavlc::MAX_LEVEL, cavlc::MAX_LEVEL); + } + } + dcl[comp] = lv; + for b in 0..4 { + c.code.cdc[comp][b] = lv[b] as i16; + } + dc_any |= lv.iter().any(|&v| v != 0); + ac_any |= c.code.cac[comp * 4..comp * 4 + 4].iter().any(|b| b.iter().any(|&v| v != 0)); + } + let cbp = if ac_any { + 2 + } else if dc_any { + 1 + } else { + 0 + }; + c.info.dc_cbf &= 1; + for comp in 0..2 { + let p = if comp == 0 { pu } else { pv }; + let dcq = dequant_chroma_dc(&dcl[comp], qt, qpc); + if dcl[comp].iter().any(|&v| v != 0) { + c.info.dc_cbf |= 2 << comp; + } + for b in 0..4 { + let (bx, by) = ((b & 1) * 4, (b >> 1) * 4); + let mut t = coefs[comp][b]; + let nz = if cbp == 2 { t.iter().skip(1).filter(|&&v| v != 0).count() as u8 } else { 0 }; + c.info.nnz_c[comp * 4 + b] = nz; + if cbp != 2 { + t = [0; 16]; + } else { + dequant4(&mut t, qt, qpc, true); + } + t[0] = dcq[b]; + let rec = if comp == 0 { &mut c.ru } else { &mut c.rv }; + if nz == 0 { + let dcv = (t[0] + 32) >> 6; + add_chroma_residual(rec, p, bx, by, |_, _| dcv); + } else { + let r = idct4(&t); + add_chroma_residual(rec, p, bx, by, |x, y| r[y * 4 + x]); + } + } + } + if cbp == 0 { + c.code.cdc = [[0; 4]; 2]; + } + if cbp != 2 { + c.code.cac = [[0; 16]; 8]; + } + cbp + } + + /// Inter luma residual with the 4x4 transform. Returns luma cbp. + fn code_luma4(&self, src: &[u8; 256], pred: &[u8; 256], qp: u8, dz: Deadzone, decimate: bool, c: &mut Cand) -> u8 { + let qt = self.f.qt; + let mut coefs = [[0i32; 16]; 16]; + let mut score8 = [0u32; 4]; + for blk in 0..16 { + let (bx, by) = BLK4_XY[blk]; + let mut d = [0i32; 16]; + for y in 0..4 { + for x in 0..4 { + let i = (by * 4 + y) * 16 + bx * 4 + x; + d[y * 4 + x] = src[i] as i32 - pred[i] as i32; + } + } + let mut t = fdct4(&d); + quant4(&mut t, qt, qp, dz, false); + if self.f.p.cavlc_clamp { + for v in t.iter_mut() { + *v = (*v).clamp(-cavlc::MAX_LEVEL, cavlc::MAX_LEVEL); + } + } + let mut scan = [0i16; 16]; + for k in 0..16 { + scan[k] = t[ZIGZAG4[k]] as i16; + } + score8[blk / 4] += decimate_score(&scan); + c.code.luma4[blk] = scan; + coefs[blk] = t; + } + let mut cbp = 0u8; + let total: u32 = score8.iter().sum(); + for b8 in 0..4 { + let kill = decimate && (score8[b8] < 4 || total < 6); + for i in 0..4 { + let blk = b8 * 4 + i; + if kill { + c.code.luma4[blk] = [0; 16]; + coefs[blk] = [0; 16]; + } + if c.code.luma4[blk].iter().any(|&v| v != 0) { + cbp |= 1 << b8; + } + } + } + for blk in 0..16 { + let (bx, by) = BLK4_XY[blk]; + let nz = c.code.luma4[blk].iter().filter(|&&v| v != 0).count() as u8; + c.info.nnz[by * 4 + bx] = nz; + let off = by * 4 * 16 + bx * 4; + if nz == 0 { + for y in 0..4 { + c.ry[off + y * 16..off + y * 16 + 4].copy_from_slice(&pred[off + y * 16..off + y * 16 + 4]); + } + } else { + let mut t = coefs[blk]; + dequant4(&mut t, qt, qp, false); + let r = idct4(&t); + for y in 0..4 { + for x in 0..4 { + let i = off + y * 16 + x; + c.ry[i] = (pred[i] as i32 + r[y * 4 + x]).clamp(0, 255) as u8; + } + } + } + } + cbp + } + + /// Inter luma residual with the 8x8 transform. Returns luma cbp. + fn code_luma8(&self, src: &[u8; 256], pred: &[u8; 256], qp: u8, dz: Deadzone, decimate: bool, c: &mut Cand) -> u8 { + let qt = self.f.qt; + let mut coefs = [[0i32; 64]; 4]; + let mut score = [0u32; 4]; + for b8 in 0..4 { + let (ox, oy) = ((b8 & 1) * 8, (b8 >> 1) * 8); + let mut d = [0i32; 64]; + for y in 0..8 { + for x in 0..8 { + let i = (oy + y) * 16 + ox + x; + d[y * 8 + x] = src[i] as i32 - pred[i] as i32; + } + } + let mut t = fdct8(&d); + quant8(&mut t, qt, qp, dz); + let mut scan = [0i16; 64]; + for k in 0..64 { + scan[k] = t[ZIGZAG8[k]] as i16; + } + score[b8] = decimate_score(&scan); + c.code.luma8[b8] = scan; + coefs[b8] = t; + } + let total: u32 = score.iter().sum(); + let mut cbp = 0; + for b8 in 0..4 { + let (ox, oy) = ((b8 & 1) * 8, (b8 >> 1) * 8); + if decimate && (score[b8] < 4 || total < 6) { + c.code.luma8[b8] = [0; 64]; + coefs[b8] = [0; 64]; + } + let nz = c.code.luma8[b8].iter().filter(|&&v| v != 0).count() as u8; + let (bx, by) = (ox / 4, oy / 4); + for (dx, dy) in [(0, 0), (1, 0), (0, 1), (1, 1)] { + c.info.nnz[(by + dy) * 4 + bx + dx] = nz; + } + if nz == 0 { + for y in 0..8 { + let o = (oy + y) * 16 + ox; + c.ry[o..o + 8].copy_from_slice(&pred[o..o + 8]); + } + continue; + } + cbp |= 1 << b8; + let mut t = coefs[b8]; + dequant8(&mut t, qt, qp); + let r = idct8(&t); + for y in 0..8 { + for x in 0..8 { + let i = (oy + y) * 16 + ox + x; + c.ry[i] = (pred[i] as i32 + r[y * 8 + x]).clamp(0, 255) as u8; + } + } + } + cbp + } + + /// Code an inter candidate's residual given its luma/chroma prediction. + pub(super) fn code_inter(&self, src: &Src, py: &[u8; 256], pu: &[u8; 64], pv: &[u8; 64], qp: u8, c: &mut Cand) { + let t8_allowed = self.f.p.t8x8; + let (use8, cbpl) = if t8_allowed { + if self.f.p.adaptive_t8 { + // Compare 4x4 and 8x8 transform by SSD + lambda * (approximate) bits. Neither + // coder reads the fields it writes, so the 4x4 trial is coded into `c` itself + // and the 8x8 trial into a copy that replaces `c` when it wins: the winner is + // not coded a second time. + let mut c8 = c.clone(); + let cbp8 = self.code_luma8(&src.y, py, qp, Deadzone::INTER, self.f.p.decimate, &mut c8); + let cbp4 = self.code_luma4(&src.y, py, qp, Deadzone::INTER, self.f.p.decimate, c); + let lam = self.f.lam.ssd[qp as usize]; + let lb = level_bits_table(); + let cost = |cc: &Cand, cbp: u8, eight: bool| { + let d = ssd(&src.y, 16, &cc.ry, 16, 16, 16) as f64; + let mut bits = 0f64; + if eight { + for b in 0..4 { + if (cbp >> b) & 1 != 0 { + bits += cc.code.luma8[b].iter().map(|&v| level_bits(lb, v)).sum::(); + } + } + } else { + for b in 0..16 { + if (cbp >> (b / 4)) & 1 != 0 { + bits += 1.0 + cc.code.luma4[b].iter().map(|&v| level_bits(lb, v)).sum::(); + } + } + } + d + lam * bits + }; + if cost(&c8, cbp8, true) <= cost(c, cbp4, false) { + *c = c8; + (true, cbp8) + } else { + (false, cbp4) + } + } else { + (true, self.code_luma8(&src.y, py, qp, Deadzone::INTER, self.f.p.decimate, c)) + } + } else { + (false, self.code_luma4(&src.y, py, qp, Deadzone::INTER, self.f.p.decimate, c)) + }; + c.code.t8x8 = use8 && cbpl != 0; + c.info.t8x8 = c.code.t8x8; + let cbpc = self.code_chroma(src, pu, pv, qp, false, c); + c.code.cbp = cbpl | (cbpc << 4); + } +} diff --git a/crates/h264enc/src/syntax.rs b/crates/h264enc/src/syntax.rs new file mode 100644 index 0000000000..c35d6bd9bf --- /dev/null +++ b/crates/h264enc/src/syntax.rs @@ -0,0 +1,120 @@ +//! Macroblock-layer syntax values produced by mode decision and consumed by the entropy coders. + +use crate::mbinfo::{MbInfo, MbKind, Mv, Part}; +use crate::nal::SliceType; + +#[derive(Clone)] +pub struct MbCode { + pub kind: MbKind, + pub part: Part, + /// Prediction direction per partition: 0 = L0, 1 = L1, 2 = Bi. + pub bdir: [u8; 4], + pub i16_mode: u8, + pub chroma_mode: u8, + /// Per 4x4 (or 8x8, first four entries) block in decoding order: -1 = use predicted mode, else rem_intra_pred_mode. + pub ipred: [i8; 16], + pub t8x8: bool, + /// mvd per list per partition. + pub mvd: [[Mv; 4]; 2], + pub cbp: u8, + pub qp_delta: i32, + /// Intra16x16 DC levels in zig-zag order. + pub luma_dc: [i16; 16], + /// Luma 4x4 levels by luma4x4BlkIdx, zig-zag order (index 0 unused for Intra16x16 AC). + pub luma4: [[i16; 16]; 16], + /// Luma 8x8 levels by 8x8 block, zig-zag order. + pub luma8: [[i16; 64]; 4], + /// Chroma DC levels (Cb, Cr) in raster (c0..c3) order. + pub cdc: [[i16; 4]; 2], + /// Chroma AC levels, Cb blocks 0..3 then Cr 0..3, zig-zag order (index 0 unused). + pub cac: [[i16; 16]; 8], +} + +impl Default for MbCode { + fn default() -> Self { + MbCode { + kind: MbKind::None, + part: Part::P16x16, + bdir: [0; 4], + i16_mode: 0, + chroma_mode: 0, + ipred: [-1; 16], + t8x8: false, + mvd: [[Mv::ZERO; 4]; 2], + cbp: 0, + qp_delta: 0, + luma_dc: [0; 16], + luma4: [[0; 16]; 16], + luma8: [[0; 64]; 4], + cdc: [[0; 4]; 2], + cac: [[0; 16]; 8], + } + } +} + +impl MbCode { + /// B macroblock type number (Table 7-14) for inter B macroblocks. + pub fn b_type_number(&self) -> u32 { + match self.kind { + MbKind::BDirect => 0, + _ => match self.part { + Part::P16x16 => 1 + self.bdir[0] as u32, + Part::P16x8 | Part::P8x16 => { + const PAIRS: [(u8, u8); 9] = [(0, 0), (1, 1), (0, 1), (1, 0), (0, 2), (1, 2), (2, 0), (2, 1), (2, 2)]; + let k = PAIRS.iter().position(|&p| p == (self.bdir[0], self.bdir[1])).unwrap_or(0) as u32; + 4 + 2 * k + (self.part == Part::P8x16) as u32 + } + Part::P8x8 => 22, + }, + } + } + pub fn uses_list(&self, part: usize, list: usize) -> bool { + match self.kind { + MbKind::PInter => list == 0, + MbKind::BInter => self.bdir[part] == 2 || self.bdir[part] as usize == list, + _ => false, + } + } +} + +/// Neighbour context for entropy coding one macroblock. +pub struct NbCtx<'a> { + pub left: Option<&'a MbInfo>, + pub top: Option<&'a MbInfo>, + pub cur: &'a MbInfo, + pub slice_type: SliceType, + pub prev_dqp_nz: bool, + pub t8x8_mode: bool, +} + +impl NbCtx<'_> { + /// coded_block_flag-style condition for an unavailable neighbour. + #[inline] + pub fn unavail_cbf(&self) -> u32 { + self.cur.kind.is_intra() as u32 + } + /// Luma 4x4 neighbour non-zero count (A = left when `left`, else B = above). Returns None if unavailable. + #[inline] + pub fn luma_nnz(&self, raster: usize, left: bool) -> Option { + let (x, y) = (raster & 3, raster >> 2); + if left { + if x > 0 { Some(self.cur.nnz[raster - 1]) } else { self.left.map(|m| m.nnz[y * 4 + 3]) } + } else if y > 0 { + Some(self.cur.nnz[raster - 4]) + } else { + self.top.map(|m| m.nnz[12 + x]) + } + } + /// Chroma AC neighbour count for component `c`, block raster `b` (2x2). + #[inline] + pub fn chroma_nnz(&self, c: usize, b: usize, left: bool) -> Option { + let (x, y) = (b & 1, b >> 1); + if left { + if x > 0 { Some(self.cur.nnz_c[c * 4 + b - 1]) } else { self.left.map(|m| m.nnz_c[c * 4 + y * 2 + 1]) } + } else if y > 0 { + Some(self.cur.nnz_c[c * 4 + b - 2]) + } else { + self.top.map(|m| m.nnz_c[c * 4 + 2 + x]) + } + } +} diff --git a/crates/h264enc/src/tables.rs b/crates/h264enc/src/tables.rs new file mode 100644 index 0000000000..2dde174ca9 --- /dev/null +++ b/crates/h264enc/src/tables.rs @@ -0,0 +1,1097 @@ +//! Normative constant tables from ITU-T H.264 (scans, CABAC state machine and initialisation values, +//! CAVLC code tables, quantisation/deblocking tables). + +/// 4x4 frame zig-zag scan: scan index -> raster index (y*4+x). +pub const ZIGZAG4: [usize; 16] = [0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15]; + +/// 8x8 frame zig-zag scan: scan index -> raster index (y*8+x). +pub const ZIGZAG8: [usize; 64] = [ + 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50, 43, + 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63, +]; + +/// Luma 4x4 block index (decoding order) -> (x, y) in 4x4 units within the macroblock. +pub const BLK4_XY: [(usize, usize); 16] = + [(0, 0), (1, 0), (0, 1), (1, 1), (2, 0), (3, 0), (2, 1), (3, 1), (0, 2), (1, 2), (0, 3), (1, 3), (2, 2), (3, 2), (2, 3), (3, 3)]; + +/// Raster 4x4 position (y*4+x) -> luma 4x4 block index. +pub const RASTER_TO_BLK4: [usize; 16] = [0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15]; + +/// rangeTabLPS[pStateIdx][qCodIRangeIdx]. +pub const RANGE_LPS: [[u8; 4]; 64] = [ + [128, 176, 208, 240], + [128, 167, 197, 227], + [128, 158, 187, 216], + [123, 150, 178, 205], + [116, 142, 169, 195], + [111, 135, 160, 185], + [105, 128, 152, 175], + [100, 122, 144, 166], + [95, 116, 137, 158], + [90, 110, 130, 150], + [85, 104, 123, 142], + [81, 99, 117, 135], + [77, 94, 111, 128], + [73, 89, 105, 122], + [69, 85, 100, 116], + [66, 80, 95, 110], + [62, 76, 90, 104], + [59, 72, 86, 99], + [56, 69, 81, 94], + [53, 65, 77, 89], + [51, 62, 73, 85], + [48, 59, 69, 80], + [46, 56, 66, 76], + [43, 53, 63, 72], + [41, 50, 59, 69], + [39, 48, 56, 65], + [37, 45, 54, 62], + [35, 43, 51, 59], + [33, 41, 48, 56], + [32, 39, 46, 53], + [30, 37, 43, 50], + [29, 35, 41, 48], + [27, 33, 39, 45], + [26, 31, 37, 43], + [24, 30, 35, 41], + [23, 28, 33, 39], + [22, 27, 32, 37], + [21, 26, 30, 35], + [20, 24, 29, 33], + [19, 23, 27, 31], + [18, 22, 26, 30], + [17, 21, 25, 28], + [16, 20, 23, 27], + [15, 19, 22, 25], + [14, 18, 21, 24], + [14, 17, 20, 23], + [13, 16, 19, 22], + [12, 15, 18, 21], + [12, 14, 17, 20], + [11, 14, 16, 19], + [11, 13, 15, 18], + [10, 12, 15, 17], + [10, 12, 14, 16], + [9, 11, 13, 15], + [9, 11, 12, 14], + [8, 10, 12, 14], + [8, 9, 11, 13], + [7, 9, 11, 12], + [7, 9, 10, 12], + [7, 8, 10, 11], + [6, 8, 9, 11], + [6, 7, 9, 10], + [6, 7, 8, 9], + [2, 2, 2, 2], +]; + +/// transIdxLPS. +pub const TRANS_LPS: [u8; 64] = [ + 0, 0, 1, 2, 2, 4, 4, 5, 6, 7, 8, 9, 9, 11, 11, 12, 13, 13, 15, 15, 16, 16, 18, 18, 19, 19, 21, 21, 22, 22, 23, 24, 24, 25, 26, 26, 27, 27, 28, + 29, 29, 30, 30, 30, 31, 32, 32, 33, 33, 33, 34, 34, 35, 35, 35, 36, 36, 36, 37, 37, 37, 38, 38, 63, +]; + +/// Number of CABAC contexts we keep (0..=459 covers every frame-coded 4:2:0 syntax element). +pub const NUM_CTX: usize = 460; + +/// Context initialisation (m, n) for I slices. Field-coding contexts are left as zero (never used). +pub fn cabac_init_i() -> [(i8, i8); NUM_CTX] { + let mut t = [(0i8, 0i8); NUM_CTX]; + set(&mut t, 0, &COMMON_0_10); + set(&mut t, 60, &COMMON_60_69); + set(&mut t, 70, &I_70_104); + set(&mut t, 105, &I_105_165); + set(&mut t, 166, &I_166_226); + set(&mut t, 227, &I_227_275); + set(&mut t, 399, &I_399_401); + set(&mut t, 402, &I_402_435); + t +} + +/// Context initialisation (m, n) for P/B slices with cabac_init_idc = 0. +pub fn cabac_init_pb0() -> [(i8, i8); NUM_CTX] { + let mut t = [(0i8, 0i8); NUM_CTX]; + set(&mut t, 0, &COMMON_0_10); + set(&mut t, 11, &PB0_11_59); + set(&mut t, 60, &COMMON_60_69); + set(&mut t, 70, &PB0_70_104); + set(&mut t, 105, &PB0_105_165); + set(&mut t, 166, &PB0_166_226); + set(&mut t, 227, &PB0_227_275); + set(&mut t, 399, &PB0_399_401); + set(&mut t, 402, &PB0_402_435); + t +} + +fn set(t: &mut [(i8, i8); NUM_CTX], start: usize, v: &[(i8, i8)]) { + t[start..start + v.len()].copy_from_slice(v); +} + +const COMMON_0_10: [(i8, i8); 11] = [(20, -15), (2, 54), (3, 74), (20, -15), (2, 54), (3, 74), (-28, 127), (-23, 104), (-6, 53), (-1, 54), (7, 51)]; + +const COMMON_60_69: [(i8, i8); 10] = [(0, 41), (0, 63), (0, 63), (0, 63), (-9, 83), (4, 86), (0, 97), (-7, 72), (13, 41), (3, 62)]; + +const PB0_11_59: [(i8, i8); 49] = [ + // 11..23 P mb_skip / mb_type / sub_mb_type + (23, 33), + (23, 2), + (21, 0), + (1, 9), + (0, 49), + (-37, 118), + (5, 57), + (-13, 78), + (-11, 65), + (1, 62), + (12, 49), + (-4, 73), + (17, 50), + // 24..39 B mb_skip / mb_type / sub_mb_type + (18, 64), + (9, 43), + (29, 0), + (26, 67), + (16, 90), + (9, 104), + (-46, 127), + (-20, 104), + (1, 67), + (-13, 78), + (-11, 65), + (1, 62), + (-6, 86), + (-17, 95), + (-6, 61), + (9, 45), + // 40..53 mvd + (-3, 69), + (-6, 81), + (-11, 96), + (6, 55), + (7, 67), + (-5, 86), + (2, 88), + (0, 58), + (-3, 76), + (-10, 94), + (5, 54), + (4, 69), + (-3, 81), + (0, 88), + // 54..59 ref_idx + (-7, 67), + (-5, 74), + (-4, 74), + (-5, 80), + (-7, 72), + (1, 58), +]; + +const I_70_104: [(i8, i8); 35] = [ + (0, 11), + (1, 55), + (0, 69), + (-17, 127), + (-13, 102), + (0, 82), + (-7, 74), + (-21, 107), + (-27, 127), + (-31, 127), + (-24, 127), + (-18, 95), + (-27, 127), + (-21, 114), + (-30, 127), + (-17, 123), + (-12, 115), + (-16, 122), + (-11, 115), + (-12, 63), + (-2, 68), + (-15, 84), + (-13, 104), + (-3, 70), + (-8, 93), + (-10, 90), + (-30, 127), + (-1, 74), + (-6, 97), + (-7, 91), + (-20, 127), + (-4, 56), + (-5, 82), + (-7, 76), + (-22, 125), +]; + +const PB0_70_104: [(i8, i8); 35] = [ + (0, 45), + (-4, 78), + (-3, 96), + (-27, 126), + (-28, 98), + (-25, 101), + (-23, 67), + (-28, 82), + (-20, 94), + (-16, 83), + (-22, 110), + (-21, 91), + (-18, 102), + (-13, 93), + (-29, 127), + (-7, 92), + (-5, 89), + (-7, 96), + (-13, 108), + (-3, 46), + (-1, 65), + (-1, 57), + (-9, 93), + (-3, 74), + (-9, 92), + (-8, 87), + (-23, 126), + (5, 54), + (6, 60), + (6, 59), + (6, 69), + (-1, 48), + (0, 68), + (-4, 69), + (-8, 88), +]; + +const I_105_165: [(i8, i8); 61] = [ + (-7, 93), + (-11, 87), + (-3, 77), + (-5, 71), + (-4, 63), + (-4, 68), + (-12, 84), + (-7, 62), + (-7, 65), + (8, 61), + (5, 56), + (-2, 66), + (1, 64), + (0, 61), + (-2, 78), + (1, 50), + (7, 52), + (10, 35), + (0, 44), + (11, 38), + (1, 45), + (0, 46), + (5, 44), + (31, 17), + (1, 51), + (7, 50), + (28, 19), + (16, 33), + (14, 62), + (-13, 108), + (-15, 100), + (-13, 101), + (-13, 91), + (-12, 94), + (-10, 88), + (-16, 84), + (-10, 86), + (-7, 83), + (-13, 87), + (-19, 94), + (1, 70), + (0, 72), + (-5, 74), + (18, 59), + (-8, 102), + (-15, 100), + (0, 95), + (-4, 75), + (2, 72), + (-11, 75), + (-3, 71), + (15, 46), + (-13, 69), + (0, 62), + (0, 65), + (21, 37), + (-15, 72), + (9, 57), + (16, 54), + (0, 62), + (12, 72), +]; + +const PB0_105_165: [(i8, i8); 61] = [ + (-2, 85), + (-6, 78), + (-1, 75), + (-7, 77), + (2, 54), + (5, 50), + (-3, 68), + (1, 50), + (6, 42), + (-4, 81), + (1, 63), + (-4, 70), + (0, 67), + (2, 57), + (-2, 76), + (11, 35), + (4, 64), + (1, 61), + (11, 35), + (18, 25), + (12, 24), + (13, 29), + (13, 36), + (-10, 93), + (-7, 73), + (-2, 73), + (13, 46), + (9, 49), + (-7, 100), + (9, 53), + (2, 53), + (5, 53), + (-2, 61), + (0, 56), + (0, 56), + (-13, 63), + (-5, 60), + (-1, 62), + (4, 57), + (-6, 69), + (4, 57), + (14, 39), + (4, 51), + (13, 68), + (3, 64), + (1, 61), + (9, 63), + (7, 50), + (16, 39), + (5, 44), + (4, 52), + (11, 48), + (-5, 60), + (-1, 59), + (0, 59), + (22, 33), + (5, 44), + (14, 43), + (-1, 78), + (0, 60), + (9, 69), +]; + +const I_166_226: [(i8, i8); 61] = [ + (24, 0), + (15, 9), + (8, 25), + (13, 18), + (15, 9), + (13, 19), + (10, 37), + (12, 18), + (6, 29), + (20, 33), + (15, 30), + (4, 45), + (1, 58), + (0, 62), + (7, 61), + (12, 38), + (11, 45), + (15, 39), + (11, 42), + (13, 44), + (16, 45), + (12, 41), + (10, 49), + (30, 34), + (18, 42), + (10, 55), + (17, 51), + (17, 46), + (0, 89), + (26, -19), + (22, -17), + (26, -17), + (30, -25), + (28, -20), + (33, -23), + (37, -27), + (33, -23), + (40, -28), + (38, -17), + (33, -11), + (40, -15), + (41, -6), + (38, 1), + (41, 17), + (30, -6), + (27, 3), + (26, 22), + (37, -16), + (35, -4), + (38, -8), + (38, -3), + (37, 3), + (38, 5), + (42, 0), + (35, 16), + (39, 22), + (14, 48), + (27, 37), + (21, 60), + (12, 68), + (2, 97), +]; + +const PB0_166_226: [(i8, i8); 61] = [ + (11, 28), + (2, 40), + (3, 44), + (0, 49), + (0, 46), + (2, 44), + (2, 51), + (0, 47), + (4, 39), + (2, 62), + (6, 46), + (0, 54), + (3, 54), + (2, 58), + (4, 63), + (6, 51), + (6, 57), + (7, 53), + (6, 52), + (6, 55), + (11, 45), + (14, 36), + (8, 53), + (-1, 82), + (7, 55), + (-3, 78), + (15, 46), + (22, 31), + (-1, 84), + (25, 7), + (30, -7), + (28, 3), + (28, 4), + (32, 0), + (34, -1), + (30, 6), + (30, 6), + (32, 9), + (31, 19), + (26, 27), + (26, 30), + (37, 20), + (28, 34), + (17, 70), + (1, 67), + (5, 59), + (9, 67), + (16, 30), + (18, 32), + (18, 35), + (22, 29), + (24, 31), + (23, 38), + (18, 43), + (20, 41), + (11, 63), + (9, 59), + (9, 64), + (-1, 94), + (-2, 89), + (-9, 108), +]; + +const I_227_275: [(i8, i8); 49] = [ + (-3, 71), + (-6, 42), + (-5, 50), + (-3, 54), + (-2, 62), + (0, 58), + (1, 63), + (-2, 72), + (-1, 74), + (-9, 91), + (-5, 67), + (-5, 27), + (-3, 39), + (-2, 44), + (0, 46), + (-16, 64), + (-8, 68), + (-10, 78), + (-6, 77), + (-10, 86), + (-12, 92), + (-15, 55), + (-10, 60), + (-6, 62), + (-4, 65), + (-12, 73), + (-8, 76), + (-7, 80), + (-9, 88), + (-17, 110), + (-11, 97), + (-20, 84), + (-11, 79), + (-6, 73), + (-4, 74), + (-13, 86), + (-13, 96), + (-11, 97), + (-19, 117), + (-8, 78), + (-5, 33), + (-4, 48), + (-2, 53), + (-3, 62), + (-13, 71), + (-10, 79), + (-12, 86), + (-13, 90), + (-14, 97), +]; + +const PB0_227_275: [(i8, i8); 49] = [ + (-6, 76), + (-2, 44), + (0, 45), + (0, 52), + (-3, 64), + (-2, 59), + (-4, 70), + (-4, 75), + (-8, 82), + (-17, 102), + (-9, 77), + (3, 24), + (0, 42), + (0, 48), + (0, 55), + (-6, 59), + (-7, 71), + (-12, 83), + (-11, 87), + (-30, 119), + (1, 58), + (-3, 29), + (-1, 36), + (1, 38), + (2, 43), + (-6, 55), + (0, 58), + (0, 64), + (-3, 74), + (-10, 90), + (0, 70), + (-4, 29), + (5, 31), + (7, 42), + (1, 59), + (-2, 58), + (-3, 72), + (-3, 81), + (-11, 97), + (0, 58), + (8, 5), + (10, 14), + (14, 18), + (13, 27), + (2, 40), + (0, 58), + (-3, 70), + (-6, 79), + (-8, 85), +]; + +const I_399_401: [(i8, i8); 3] = [(31, 21), (31, 31), (25, 50)]; +const PB0_399_401: [(i8, i8); 3] = [(12, 40), (11, 51), (14, 59)]; + +const I_402_435: [(i8, i8); 34] = [ + (-17, 120), + (-20, 112), + (-18, 114), + (-11, 85), + (-15, 92), + (-14, 89), + (-26, 71), + (-15, 81), + (-14, 80), + (0, 68), + (-14, 70), + (-24, 56), + (-23, 68), + (-24, 50), + (-11, 74), + (23, -13), + (26, -13), + (40, -15), + (49, -14), + (44, 3), + (45, 6), + (44, 34), + (33, 54), + (19, 82), + (-3, 75), + (-1, 23), + (1, 34), + (1, 43), + (0, 54), + (-2, 55), + (0, 61), + (1, 64), + (0, 68), + (-9, 92), +]; + +const PB0_402_435: [(i8, i8); 34] = [ + (-4, 79), + (-7, 71), + (-5, 69), + (-9, 70), + (-8, 66), + (-10, 68), + (-19, 73), + (-12, 69), + (-16, 70), + (-15, 67), + (-20, 62), + (-19, 70), + (-16, 66), + (-22, 65), + (-20, 63), + (9, -2), + (26, -9), + (33, -9), + (39, -7), + (41, -2), + (45, 3), + (49, 9), + (45, 27), + (36, 59), + (-6, 66), + (-7, 35), + (-7, 42), + (-8, 45), + (-5, 48), + (-12, 56), + (-6, 60), + (-5, 62), + (-8, 66), + (-8, 76), +]; + +/// significant_coeff_flag ctxIdxInc for 8x8 frame-coded blocks (Table 9-43). +pub const SIG8X8_FRAME: [u8; 63] = [ + 0, 1, 2, 3, 4, 5, 5, 4, 4, 3, 3, 4, 4, 4, 5, 5, 4, 4, 4, 4, 3, 3, 6, 7, 7, 7, 8, 9, 10, 9, 8, 7, 7, 6, 11, 12, 13, 11, 6, 7, 8, 9, 14, 10, 9, 8, + 6, 11, 12, 13, 11, 6, 9, 14, 10, 9, 11, 12, 13, 11, 14, 10, 12, +]; + +/// last_significant_coeff_flag ctxIdxInc for 8x8 blocks (Table 9-43). +pub const LAST8X8: [u8; 63] = [ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, + 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, +]; + +// ---------------------------------------------------------------- CAVLC + +/// coeff_token (length, code) indexed [table][total_coeff][trailing_ones] for nC ranges 0..2, 2..4, 4..8. +pub const COEFF_TOKEN_LEN: [[[u8; 4]; 17]; 3] = [ + [ + [1, 0, 0, 0], + [6, 2, 0, 0], + [8, 6, 3, 0], + [9, 8, 7, 5], + [10, 9, 8, 6], + [11, 10, 9, 7], + [13, 11, 10, 8], + [13, 13, 11, 9], + [13, 13, 13, 10], + [14, 14, 13, 11], + [14, 14, 14, 13], + [15, 15, 14, 14], + [15, 15, 15, 14], + [16, 15, 15, 15], + [16, 16, 16, 15], + [16, 16, 16, 16], + [16, 16, 16, 16], + ], + [ + [2, 0, 0, 0], + [6, 2, 0, 0], + [6, 5, 3, 0], + [7, 6, 6, 4], + [8, 6, 6, 4], + [8, 7, 7, 5], + [9, 8, 8, 6], + [11, 9, 9, 6], + [11, 11, 11, 7], + [12, 11, 11, 9], + [12, 12, 12, 11], + [12, 12, 12, 11], + [13, 13, 13, 12], + [13, 13, 13, 13], + [13, 14, 13, 13], + [14, 14, 14, 13], + [14, 14, 14, 14], + ], + [ + [4, 0, 0, 0], + [6, 4, 0, 0], + [6, 5, 4, 0], + [6, 5, 5, 4], + [7, 5, 5, 4], + [7, 5, 5, 4], + [7, 6, 6, 4], + [7, 6, 6, 4], + [8, 7, 7, 5], + [8, 8, 7, 6], + [9, 8, 8, 7], + [9, 9, 8, 8], + [9, 9, 9, 8], + [10, 9, 9, 9], + [10, 10, 10, 10], + [10, 10, 10, 10], + [10, 10, 10, 10], + ], +]; + +pub const COEFF_TOKEN_CODE: [[[u8; 4]; 17]; 3] = [ + [ + [1, 0, 0, 0], + [5, 1, 0, 0], + [7, 4, 1, 0], + [7, 6, 5, 3], + [7, 6, 5, 3], + [7, 6, 5, 4], + [15, 6, 5, 4], + [11, 14, 5, 4], + [8, 10, 13, 4], + [15, 14, 9, 4], + [11, 10, 13, 12], + [15, 14, 9, 12], + [11, 10, 13, 8], + [15, 1, 9, 12], + [11, 14, 13, 8], + [7, 10, 9, 12], + [4, 6, 5, 8], + ], + [ + [3, 0, 0, 0], + [11, 2, 0, 0], + [7, 7, 3, 0], + [7, 10, 9, 5], + [7, 6, 5, 4], + [4, 6, 5, 6], + [7, 6, 5, 8], + [15, 6, 5, 4], + [11, 14, 13, 4], + [15, 10, 9, 4], + [11, 14, 13, 12], + [8, 10, 9, 8], + [15, 14, 13, 12], + [11, 10, 9, 12], + [7, 11, 6, 8], + [9, 8, 10, 1], + [7, 6, 5, 4], + ], + [ + [15, 0, 0, 0], + [15, 14, 0, 0], + [11, 15, 13, 0], + [8, 12, 14, 12], + [15, 10, 11, 11], + [11, 8, 9, 10], + [9, 14, 13, 9], + [8, 10, 9, 8], + [15, 14, 13, 13], + [11, 14, 10, 12], + [15, 10, 13, 12], + [11, 14, 9, 12], + [8, 10, 13, 8], + [13, 7, 9, 12], + [9, 12, 11, 10], + [5, 8, 7, 6], + [1, 4, 3, 2], + ], +]; + +/// Chroma DC (nC = -1) coeff_token [total_coeff][trailing_ones]. +pub const COEFF_TOKEN_CDC_LEN: [[u8; 4]; 5] = [[2, 0, 0, 0], [6, 1, 0, 0], [6, 6, 3, 0], [6, 7, 7, 6], [6, 8, 8, 7]]; +pub const COEFF_TOKEN_CDC_CODE: [[u8; 4]; 5] = [[1, 0, 0, 0], [7, 1, 0, 0], [4, 6, 1, 0], [3, 3, 2, 5], [2, 3, 2, 0]]; + +/// total_zeros for 4x4 blocks: [total_coeff - 1][total_zeros] = (len, code). +pub const TOTAL_ZEROS: [[(u8, u8); 16]; 15] = [ + [(1, 1), (3, 3), (3, 2), (4, 3), (4, 2), (5, 3), (5, 2), (6, 3), (6, 2), (7, 3), (7, 2), (8, 3), (8, 2), (9, 3), (9, 2), (9, 1)], + [(3, 7), (3, 6), (3, 5), (3, 4), (3, 3), (4, 5), (4, 4), (4, 3), (4, 2), (5, 3), (5, 2), (6, 3), (6, 2), (6, 1), (6, 0), (0, 0)], + [(4, 5), (3, 7), (3, 6), (3, 5), (4, 4), (4, 3), (3, 4), (3, 3), (4, 2), (5, 3), (5, 2), (6, 1), (5, 1), (6, 0), (0, 0), (0, 0)], + [(5, 3), (3, 7), (4, 5), (4, 4), (3, 6), (3, 5), (3, 4), (4, 3), (3, 3), (4, 2), (5, 2), (5, 1), (5, 0), (0, 0), (0, 0), (0, 0)], + [(4, 5), (4, 4), (4, 3), (3, 7), (3, 6), (3, 5), (3, 4), (3, 3), (4, 2), (5, 1), (4, 1), (5, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(6, 1), (5, 1), (3, 7), (3, 6), (3, 5), (3, 4), (3, 3), (3, 2), (4, 1), (3, 1), (6, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(6, 1), (5, 1), (3, 5), (3, 4), (3, 3), (2, 3), (3, 2), (4, 1), (3, 1), (6, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(6, 1), (4, 1), (5, 1), (3, 3), (2, 3), (2, 2), (3, 2), (3, 1), (6, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(6, 1), (6, 0), (4, 1), (2, 3), (2, 2), (3, 1), (2, 1), (5, 1), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(5, 1), (5, 0), (3, 1), (2, 3), (2, 2), (2, 1), (4, 1), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(4, 0), (4, 1), (3, 1), (3, 2), (1, 1), (3, 3), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(4, 0), (4, 1), (2, 1), (1, 1), (3, 1), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(3, 0), (3, 1), (1, 1), (2, 1), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(2, 0), (2, 1), (1, 1), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(1, 0), (1, 1), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], +]; + +/// total_zeros for 2x2 chroma DC: [total_coeff - 1][total_zeros]. +pub const TOTAL_ZEROS_CDC: [[(u8, u8); 4]; 3] = + [[(1, 1), (2, 1), (3, 1), (3, 0)], [(1, 1), (2, 1), (2, 0), (0, 0)], [(1, 1), (1, 0), (0, 0), (0, 0)]]; + +/// run_before: [min(zeros_left, 7) - 1][run_before] = (len, code). +pub const RUN_BEFORE: [[(u8, u8); 15]; 7] = [ + [(1, 1), (1, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(1, 1), (2, 1), (2, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(2, 3), (2, 2), (2, 1), (2, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(2, 3), (2, 2), (2, 1), (3, 1), (3, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(2, 3), (2, 2), (3, 3), (3, 2), (3, 1), (3, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(2, 3), (3, 0), (3, 1), (3, 3), (3, 2), (3, 5), (3, 4), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0)], + [(3, 7), (3, 6), (3, 5), (3, 4), (3, 3), (3, 2), (3, 1), (4, 1), (5, 1), (6, 1), (7, 1), (8, 1), (9, 1), (10, 1), (11, 1)], +]; + +/// coded_block_pattern -> codeNum for Intra_4x4/8x8 macroblocks (ChromaArrayType 1/2). +pub const CBP_INTRA_TO_CODE: [u8; 48] = invert_cbp(&CODE_TO_CBP_INTRA); +/// coded_block_pattern -> codeNum for inter macroblocks. +pub const CBP_INTER_TO_CODE: [u8; 48] = invert_cbp(&CODE_TO_CBP_INTER); + +const CODE_TO_CBP_INTRA: [u8; 48] = [ + 47, 31, 15, 0, 23, 27, 29, 30, 7, 11, 13, 14, 39, 43, 45, 46, 16, 3, 5, 10, 12, 19, 21, 26, 28, 35, 37, 42, 44, 1, 2, 4, 8, 17, 18, 20, 24, 6, 9, + 22, 25, 32, 33, 34, 36, 40, 38, 41, +]; +const CODE_TO_CBP_INTER: [u8; 48] = [ + 0, 16, 1, 2, 4, 8, 32, 3, 5, 10, 12, 15, 47, 7, 11, 13, 14, 6, 9, 31, 35, 37, 42, 44, 33, 34, 36, 40, 39, 43, 45, 46, 17, 18, 20, 24, 19, 21, 26, + 28, 23, 27, 29, 30, 22, 25, 38, 41, +]; + +const fn invert_cbp(t: &[u8; 48]) -> [u8; 48] { + let mut out = [0u8; 48]; + let mut i = 0; + while i < 48 { + out[t[i] as usize] = i as u8; + i += 1; + } + out +} + +// ---------------------------------------------------------------- quantisation + +/// Forward quantisation multipliers for 4x4 (position classes: even/even, odd/odd, other). +pub const QUANT4: [[i32; 3]; 6] = + [[13107, 5243, 8066], [11916, 4660, 7490], [10082, 4194, 6554], [9362, 3647, 5825], [8192, 3355, 5243], [7282, 2893, 4559]]; + +/// normAdjust4x4 (v) values. +pub const DEQUANT4: [[i32; 3]; 6] = [[10, 16, 13], [11, 18, 14], [13, 20, 16], [14, 23, 18], [16, 25, 20], [18, 29, 23]]; + +/// normAdjust8x8 (v) values. +pub const DEQUANT8: [[i32; 6]; 6] = [ + [20, 18, 32, 19, 25, 24], + [22, 19, 35, 21, 28, 26], + [26, 23, 42, 24, 33, 31], + [28, 25, 45, 26, 35, 33], + [32, 28, 51, 30, 40, 38], + [36, 32, 58, 34, 46, 43], +]; + +#[inline] +pub fn pos_class4(i: usize) -> usize { + let (x, y) = (i & 3, i >> 2); + if x % 2 == 0 && y % 2 == 0 { + 0 + } else if x % 2 == 1 && y % 2 == 1 { + 1 + } else { + 2 + } +} + +#[inline] +pub fn pos_class8(i: usize) -> usize { + let (x, y) = (i & 7, i >> 3); + if x % 4 == 0 && y % 4 == 0 { + 0 + } else if x % 2 == 1 && y % 2 == 1 { + 1 + } else if x % 4 == 2 && y % 4 == 2 { + 2 + } else if (x % 4 == 0 && y % 2 == 1) || (x % 2 == 1 && y % 4 == 0) { + 3 + } else if (x % 4 == 0 && y % 4 == 2) || (x % 4 == 2 && y % 4 == 0) { + 4 + } else { + 5 + } +} + +/// QPc as a function of qPI (Table 8-15). +pub fn chroma_qp(qpi: i32) -> u8 { + const T: [u8; 22] = [29, 30, 31, 32, 32, 33, 34, 34, 35, 35, 36, 36, 37, 37, 37, 38, 38, 38, 39, 39, 39, 39]; + let q = qpi.clamp(0, 51); + if q < 30 { q as u8 } else { T[(q - 30) as usize] } +} + +// ---------------------------------------------------------------- deblocking + +pub const ALPHA: [u8; 52] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 17, 20, 22, 25, 28, 32, 36, 40, 45, 50, 56, 63, 71, 80, 90, + 101, 113, 127, 144, 162, 182, 203, 226, 255, 255, +]; +pub const BETA: [u8; 52] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, + 15, 16, 16, 17, 17, 18, 18, +]; +pub const TC0: [[u8; 3]; 52] = [ + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 0], + [0, 0, 1], + [0, 0, 1], + [0, 0, 1], + [0, 0, 1], + [0, 1, 1], + [0, 1, 1], + [1, 1, 1], + [1, 1, 1], + [1, 1, 1], + [1, 1, 1], + [1, 1, 2], + [1, 1, 2], + [1, 1, 2], + [1, 1, 2], + [1, 2, 3], + [1, 2, 3], + [2, 2, 3], + [2, 2, 4], + [2, 3, 4], + [2, 3, 4], + [3, 3, 5], + [3, 4, 6], + [3, 4, 6], + [4, 5, 7], + [4, 5, 8], + [4, 6, 9], + [5, 7, 10], + [6, 8, 11], + [6, 8, 13], + [7, 10, 14], + [8, 11, 16], + [9, 12, 18], + [10, 13, 20], + [11, 15, 23], + [13, 17, 25], +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cbp_tables_are_permutations() { + let mut seen = [false; 48]; + for &c in &CBP_INTRA_TO_CODE { + seen[c as usize] = true; + } + assert!(seen.iter().all(|&s| s)); + let mut seen = [false; 48]; + for &c in &CBP_INTER_TO_CODE { + seen[c as usize] = true; + } + assert!(seen.iter().all(|&s| s)); + } + + #[test] + fn scans_are_permutations() { + let mut s = ZIGZAG8; + s.sort(); + assert!(s.iter().enumerate().all(|(i, &v)| i == v)); + let mut s = ZIGZAG4; + s.sort(); + assert!(s.iter().enumerate().all(|(i, &v)| i == v)); + for (b, &(x, y)) in BLK4_XY.iter().enumerate() { + assert_eq!(RASTER_TO_BLK4[y * 4 + x], b); + } + } + + /// Every VLC table must be prefix-free (a cheap consistency check on transcription). + #[test] + fn vlc_tables_prefix_free() { + fn check(codes: &[(u8, u32)]) { + for (i, &(la, ca)) in codes.iter().enumerate() { + for (j, &(lb, cb)) in codes.iter().enumerate() { + if i == j || la == 0 || lb == 0 || la > lb { + continue; + } + assert!(cb >> (lb - la) != ca, "prefix clash {la}:{ca:b} vs {lb}:{cb:b}"); + } + } + } + for t in 0..3 { + let mut v = vec![]; + for tc in 0..17 { + for t1 in 0..4 { + if t1 <= tc && COEFF_TOKEN_LEN[t][tc][t1] > 0 { + v.push((COEFF_TOKEN_LEN[t][tc][t1], COEFF_TOKEN_CODE[t][tc][t1] as u32)); + } + } + } + assert_eq!(v.len(), 62); + check(&v); + } + let mut v = vec![]; + for tc in 0..5 { + for t1 in 0..4 { + if t1 <= tc && COEFF_TOKEN_CDC_LEN[tc][t1] > 0 { + v.push((COEFF_TOKEN_CDC_LEN[tc][t1], COEFF_TOKEN_CDC_CODE[tc][t1] as u32)); + } + } + } + assert_eq!(v.len(), 14); + check(&v); + for (tc, row) in TOTAL_ZEROS.iter().enumerate() { + let v: Vec<_> = row.iter().take(16 - tc).map(|&(l, c)| (l, c as u32)).collect(); + check(&v); + } + for (i, row) in RUN_BEFORE.iter().enumerate() { + let n = if i < 6 { i + 2 } else { 15 }; + let v: Vec<_> = row.iter().take(n).map(|&(l, c)| (l, c as u32)).collect(); + check(&v); + } + } +} diff --git a/crates/h264enc/src/transform.rs b/crates/h264enc/src/transform.rs new file mode 100644 index 0000000000..943b70187a --- /dev/null +++ b/crates/h264enc/src/transform.rs @@ -0,0 +1,523 @@ +//! Integer transforms, quantisation and the normative inverse transforms/scaling (§8.5.12). + +use crate::tables::{DEQUANT4, DEQUANT8, QUANT4, pos_class4, pos_class8}; + +/// Forward 4x4 core transform of a residual block (raster order). +#[inline] +pub fn fdct4(d: &[i32; 16]) -> [i32; 16] { + let mut t = [0i32; 16]; + for i in 0..4 { + let (a, b, c, e) = (d[i * 4], d[i * 4 + 1], d[i * 4 + 2], d[i * 4 + 3]); + let s03 = a + e; + let d03 = a - e; + let s12 = b + c; + let d12 = b - c; + t[i * 4] = s03 + s12; + t[i * 4 + 1] = 2 * d03 + d12; + t[i * 4 + 2] = s03 - s12; + t[i * 4 + 3] = d03 - 2 * d12; + } + let mut o = [0i32; 16]; + for i in 0..4 { + let (a, b, c, e) = (t[i], t[4 + i], t[8 + i], t[12 + i]); + let s03 = a + e; + let d03 = a - e; + let s12 = b + c; + let d12 = b - c; + o[i] = s03 + s12; + o[4 + i] = 2 * d03 + d12; + o[8 + i] = s03 - s12; + o[12 + i] = d03 - 2 * d12; + } + o +} + +/// Normative inverse 4x4 transform of scaled coefficients `d` (raster), returns residual r (after the >>6). +#[inline] +pub fn idct4(d: &[i32; 16]) -> [i32; 16] { + let mut f = [0i32; 16]; + for i in 0..4 { + let (d0, d1, d2, d3) = (d[i * 4], d[i * 4 + 1], d[i * 4 + 2], d[i * 4 + 3]); + let e = d0 + d2; + let ff = d0 - d2; + let g = (d1 >> 1) - d3; + let h = d1 + (d3 >> 1); + f[i * 4] = e + h; + f[i * 4 + 1] = ff + g; + f[i * 4 + 2] = ff - g; + f[i * 4 + 3] = e - h; + } + let mut r = [0i32; 16]; + for j in 0..4 { + let (f0, f1, f2, f3) = (f[j], f[4 + j], f[8 + j], f[12 + j]); + let e = f0 + f2; + let ff = f0 - f2; + let g = (f1 >> 1) - f3; + let h = f1 + (f3 >> 1); + r[j] = (e + h + 32) >> 6; + r[4 + j] = (ff + g + 32) >> 6; + r[8 + j] = (ff - g + 32) >> 6; + r[12 + j] = (e - h + 32) >> 6; + } + r +} + +#[inline(always)] +fn fdct8_1d(s: [i32; 8]) -> [i32; 8] { + let a0 = s[0] + s[7]; + let a1 = s[1] + s[6]; + let a2 = s[2] + s[5]; + let a3 = s[3] + s[4]; + let b0 = s[0] - s[7]; + let b1 = s[1] - s[6]; + let b2 = s[2] - s[5]; + let b3 = s[3] - s[4]; + let e0 = a0 + a3; + let e1 = a1 + a2; + let e2 = a0 - a3; + let e3 = a1 - a2; + [ + 8 * (e0 + e1), + 12 * b0 + 10 * b1 + 6 * b2 + 3 * b3, + 8 * e2 + 4 * e3, + 10 * b0 - 3 * b1 - 12 * b2 - 6 * b3, + 8 * (e0 - e1), + 6 * b0 - 12 * b1 + 3 * b2 + 10 * b3, + 4 * e2 - 8 * e3, + 3 * b0 - 6 * b1 + 10 * b2 - 12 * b3, + ] +} + +/// Forward 8x8 transform X = A x A^T (unnormalised; rows of A are the H.264 8x8 basis scaled by 8). +pub fn fdct8(d: &[i32; 64]) -> [i32; 64] { + let mut t = [0i32; 64]; + for (y, row) in d.as_chunks::<8>().0.iter().enumerate() { + let o = fdct8_1d(*row); + t[y * 8..y * 8 + 8].copy_from_slice(&o); + } + let mut out = [0i32; 64]; + for x in 0..8 { + let col = [t[x], t[8 + x], t[16 + x], t[24 + x], t[32 + x], t[40 + x], t[48 + x], t[56 + x]]; + let o = fdct8_1d(col); + for k in 0..8 { + out[k * 8 + x] = o[k]; + } + } + out +} + +#[cfg(test)] +const A8: [[i32; 8]; 8] = [ + [8, 8, 8, 8, 8, 8, 8, 8], + [12, 10, 6, 3, -3, -6, -10, -12], + [8, 4, -4, -8, -8, -4, 4, 8], + [10, -3, -12, -6, 6, 12, 3, -10], + [8, -8, -8, 8, 8, -8, -8, 8], + [6, -12, 3, 10, -10, -3, 12, -6], + [4, -8, 8, -4, -4, 8, -8, 4], + [3, -6, 10, -12, 12, -10, 6, -3], +]; + +/// Normative inverse 8x8 transform (§8.5.13), returns residual after (+32)>>6. +pub fn idct8(d: &[i32; 64]) -> [i32; 64] { + fn one(s: [i32; 8]) -> [i32; 8] { + let a0 = s[0] + s[4]; + let a4 = s[0] - s[4]; + let a2 = (s[2] >> 1) - s[6]; + let a6 = s[2] + (s[6] >> 1); + let b0 = a0 + a6; + let b2 = a4 + a2; + let b4 = a4 - a2; + let b6 = a0 - a6; + let a1 = -s[3] + s[5] - s[7] - (s[7] >> 1); + let a3 = s[1] + s[7] - s[3] - (s[3] >> 1); + let a5 = -s[1] + s[7] + s[5] + (s[5] >> 1); + let a7 = s[3] + s[5] + s[1] + (s[1] >> 1); + let b1 = a1 + (a7 >> 2); + let b7 = a7 - (a1 >> 2); + let b3 = a3 + (a5 >> 2); + let b5 = (a3 >> 2) - a5; + [b0 + b7, b2 + b5, b4 + b3, b6 + b1, b6 - b1, b4 - b3, b2 - b5, b0 - b7] + } + let mut g = [0i32; 64]; + for (i, row) in d.as_chunks::<8>().0.iter().enumerate() { + let o = one(*row); + g[i * 8..i * 8 + 8].copy_from_slice(&o); + } + let mut r = [0i32; 64]; + for j in 0..8 { + let col = [g[j], g[8 + j], g[16 + j], g[24 + j], g[32 + j], g[40 + j], g[48 + j], g[56 + j]]; + let o = one(col); + for i in 0..8 { + r[i * 8 + j] = (o[i] + 32) >> 6; + } + } + r +} + +/// Per-QP quantiser tables. +pub struct QuantTables { + /// mf4[qp%6][pos] + pub mf4: [[u32; 16]; 6], + pub v4: [[i32; 16]; 6], + pub mf8: [[u32; 64]; 6], + pub v8: [[i32; 64]; 6], +} + +impl QuantTables { + pub fn new() -> Self { + let mut mf4 = [[0; 16]; 6]; + let mut v4 = [[0; 16]; 6]; + let mut mf8 = [[0; 64]; 6]; + let mut v8 = [[0; 64]; 6]; + let norm8 = [512.0, 578.0, 320.0, 578.0, 512.0, 578.0, 320.0, 578.0f64]; + for q in 0..6 { + for i in 0..16 { + mf4[q][i] = QUANT4[q][pos_class4(i)] as u32; + v4[q][i] = DEQUANT4[q][pos_class4(i)]; + } + for i in 0..64 { + let v = DEQUANT8[q][pos_class8(i)]; + v8[q][i] = v; + let (x, y) = (i & 7, i >> 3); + mf8[q][i] = ((1u64 << 36) as f64 / (norm8[x] * norm8[y] * v as f64)).round() as u32; + } + } + QuantTables { mf4, v4, mf8, v8 } + } +} + +impl Default for QuantTables { + fn default() -> Self { + Self::new() + } +} + +/// Deadzone rounding fractions (of 2^qbits), as numerator over 6*... we express as shift-based fractions. +#[derive(Clone, Copy)] +pub struct Deadzone { + /// rounding offset as a fraction of one quantisation step, in 1/256 units. + pub f256: i64, +} + +impl Deadzone { + pub const INTRA: Deadzone = Deadzone { f256: 85 }; // ~1/3 + pub const INTER: Deadzone = Deadzone { f256: 43 }; // ~1/6 +} + +/// Quantise a 4x4 block of transform coefficients (raster) in place into levels; returns number of non-zeros. +/// `skip_dc` leaves coefficient 0 untouched (for Intra16x16/chroma AC where DC is coded separately). +#[inline] +pub fn quant4(c: &mut [i32; 16], qt: &QuantTables, qp: u8, dz: Deadzone, skip_dc: bool) -> u32 { + let qbits = 15 + (qp / 6) as u32; + let f = (((1i64 << qbits) * dz.f256) >> 8) as u32; + let mf = &qt.mf4[(qp % 6) as usize]; + let dc = c[0]; + let mut nz = 0; + for (v, &m) in c.iter_mut().zip(mf) { + // Residual coefficients stay below 2^14; the clamp keeps the u32 product from overflowing on any input. + let a = ((v.unsigned_abs().min(0xffff) * m + f) >> qbits) as i32; + nz += (a != 0) as u32; + *v = if *v < 0 { -a } else { a }; + } + if skip_dc { + nz -= (c[0] != 0) as u32; + c[0] = dc; + } + nz +} + +/// Dequantise 4x4 levels (raster) for flat scaling matrices (§8.5.12.1). `skip_dc` leaves c[0] unchanged. +#[inline] +pub fn dequant4(c: &mut [i32; 16], qt: &QuantTables, qp: u8, skip_dc: bool) { + let v = &qt.v4[(qp % 6) as usize]; + let sh = (qp / 6) as u32; + let start = if skip_dc { 1 } else { 0 }; + for i in start..16 { + // LevelScale = 16*v; qP>=24: (c*LS) << (qP/6-4); else (c*LS + 2^(3-qP/6)) >> (4-qP/6) == c*v << qP/6. + c[i] = (c[i] * v[i]) << sh; + } +} + +/// Quantise an 8x8 block (raster), returns non-zero count. +pub fn quant8(c: &mut [i32; 64], qt: &QuantTables, qp: u8, dz: Deadzone) -> u32 { + let qbits = 22 + (qp / 6) as u32; + let f = (((1i64 << qbits) * dz.f256) >> 8) as u32; + let mf = &qt.mf8[(qp % 6) as usize]; + let mut nz = 0; + for (v, &m) in c.iter_mut().zip(mf) { + // |v| * m needs up to 36 bits. Splitting |v| at bit 8 keeps both partial products in u32, and + // (256 * hi + lo) >> qbits == (hi + (lo >> 8)) >> (qbits - 8) keeps the result exact. + // Residual coefficients stay below 2^21; the clamp keeps the sums from overflowing on any input. + let x = v.unsigned_abs().min(0xff_ffff); + let a = (((x >> 8) * m + (((x & 0xff) * m + f) >> 8)) >> (qbits - 8)) as i32; + nz += (a != 0) as u32; + *v = if *v < 0 { -a } else { a }; + } + nz +} + +/// Dequantise 8x8 levels (raster), flat scaling. +pub fn dequant8(c: &mut [i32; 64], qt: &QuantTables, qp: u8) { + let v = &qt.v8[(qp % 6) as usize]; + let q6 = (qp / 6) as i32; + for i in 0..64 { + let ls = 16 * v[i]; + c[i] = if q6 >= 6 { (c[i] * ls) << (q6 - 6) } else { (c[i] * ls + (1 << (5 - q6))) >> (6 - q6) }; + } +} + +/// 4x4 Hadamard used for the Intra16x16 luma DC (unnormalised). +#[inline] +pub fn hadamard4(d: &[i32; 16]) -> [i32; 16] { + let mut t = [0i32; 16]; + for i in 0..4 { + let (a, b, c, e) = (d[i * 4], d[i * 4 + 1], d[i * 4 + 2], d[i * 4 + 3]); + t[i * 4] = a + b + c + e; + t[i * 4 + 1] = a + b - c - e; + t[i * 4 + 2] = a - b - c + e; + t[i * 4 + 3] = a - b + c - e; + } + let mut o = [0i32; 16]; + for i in 0..4 { + let (a, b, c, e) = (t[i], t[4 + i], t[8 + i], t[12 + i]); + o[i] = a + b + c + e; + o[4 + i] = a + b - c - e; + o[8 + i] = a - b - c + e; + o[12 + i] = a - b + c - e; + } + o +} + +/// Forward quantisation of the Intra16x16 DC (input: raster 4x4 of block DC coefficients). +pub fn quant_luma_dc(dc: &[i32; 16], qt: &QuantTables, qp: u8, dz: Deadzone) -> ([i32; 16], u32) { + let h = hadamard4(dc); + let qbits = 16 + (qp / 6) as u32; + let f = ((1i64 << qbits) * dz.f256) >> 8; + let mf = qt.mf4[(qp % 6) as usize][0] as i64; + let mut out = [0i32; 16]; + let mut nz = 0; + for i in 0..16 { + let v = h[i] >> 1; + let a = ((v.unsigned_abs() as i64 * mf + f) >> qbits) as i32; + out[i] = if v < 0 { -a } else { a }; + nz += (a != 0) as u32; + } + (out, nz) +} + +/// Normative Intra16x16 DC inverse (§8.5.10): levels (raster 4x4) -> dcY (raster 4x4). +pub fn dequant_luma_dc(c: &[i32; 16], qt: &QuantTables, qp: u8) -> [i32; 16] { + let f = hadamard4(c); + let ls = 16 * qt.v4[(qp % 6) as usize][0]; + let q6 = (qp / 6) as i32; + let mut o = [0i32; 16]; + for i in 0..16 { + o[i] = if q6 >= 6 { (f[i] * ls) << (q6 - 6) } else { (f[i] * ls + (1 << (5 - q6))) >> (6 - q6) }; + } + o +} + +/// Forward quantisation of chroma DC (2x2 raster of block DC coefficients). +pub fn quant_chroma_dc(dc: &[i32; 4], qt: &QuantTables, qp: u8, dz: Deadzone) -> ([i32; 4], u32) { + let h = [dc[0] + dc[1] + dc[2] + dc[3], dc[0] - dc[1] + dc[2] - dc[3], dc[0] + dc[1] - dc[2] - dc[3], dc[0] - dc[1] - dc[2] + dc[3]]; + let qbits = 16 + (qp / 6) as u32; + let f = ((1i64 << qbits) * dz.f256) >> 8; + let mf = qt.mf4[(qp % 6) as usize][0] as i64; + let mut out = [0i32; 4]; + let mut nz = 0; + for i in 0..4 { + let v = h[i]; + let a = ((v.unsigned_abs() as i64 * mf + f) >> qbits) as i32; + out[i] = if v < 0 { -a } else { a }; + nz += (a != 0) as u32; + } + (out, nz) +} + +/// Normative chroma DC inverse (§8.5.11.2) for 4:2:0: levels (raster 2x2) -> dcC. +pub fn dequant_chroma_dc(c: &[i32; 4], qt: &QuantTables, qpc: u8) -> [i32; 4] { + let f = [c[0] + c[1] + c[2] + c[3], c[0] - c[1] + c[2] - c[3], c[0] + c[1] - c[2] - c[3], c[0] - c[1] - c[2] + c[3]]; + let ls = 16 * qt.v4[(qpc % 6) as usize][0]; + let q6 = (qpc / 6) as u32; + let mut o = [0i32; 4]; + for i in 0..4 { + o[i] = ((f[i] * ls) << q6) >> 5; + } + o +} + +#[cfg(test)] +mod tests { + use super::*; + + fn quant4_i64(c: &mut [i32; 16], qp: u8, dz: Deadzone, skip_dc: bool) -> u32 { + let qbits = 15 + (qp / 6) as u32; + let f = ((1i64 << qbits) * dz.f256) >> 8; + let mut nz = 0; + for i in (skip_dc as usize)..16 { + let v = c[i]; + let a = ((v.unsigned_abs() as i64 * QUANT4[(qp % 6) as usize][pos_class4(i)] as i64 + f) >> qbits) as i32; + c[i] = if v < 0 { -a } else { a }; + nz += (a != 0) as u32; + } + nz + } + + fn mf8_i64() -> [[i64; 64]; 6] { + let norm8 = [512.0, 578.0, 320.0, 578.0, 512.0, 578.0, 320.0, 578.0f64]; + std::array::from_fn(|q| { + std::array::from_fn(|i| { + let v = DEQUANT8[q][pos_class8(i)] as f64; + ((1u64 << 36) as f64 / (norm8[i & 7] * norm8[i >> 3] * v)).round() as i64 + }) + }) + } + + fn quant8_i64(c: &mut [i32; 64], mf8: &[[i64; 64]; 6], qp: u8, dz: Deadzone) -> u32 { + let qbits = 22 + (qp / 6) as u32; + let f = ((1i64 << qbits) * dz.f256) >> 8; + let mut nz = 0; + for i in 0..64 { + let v = c[i]; + let a = ((v.unsigned_abs() as i64 * mf8[(qp % 6) as usize][i] + f) >> qbits) as i32; + c[i] = if v < 0 { -a } else { a }; + nz += (a != 0) as u32; + } + nz + } + + const DEADZONES: [Deadzone; 3] = [Deadzone::INTRA, Deadzone::INTER, Deadzone { f256: 128 }]; + + #[test] + fn quant4_matches_i64() { + let qt = QuantTables::new(); + for qp in 0..52u8 { + for dz in DEADZONES { + // Every magnitude below 2^16 (4x4 residual coefficients stay below 2^14) at every position. + for base in (0..1 << 16).step_by(16) { + for rot in [0, 5, 10] { + let mut c: [i32; 16] = std::array::from_fn(|i| { + let v = base + (i + rot) as i32 % 16; + if (i ^ (base as usize / 16)) & 1 == 0 { v } else { -v } + }); + let mut r = c; + let skip_dc = base & 16 != 0; + let nz = quant4(&mut c, &qt, qp, dz, skip_dc); + assert_eq!(nz, quant4_i64(&mut r, qp, dz, skip_dc), "qp {qp} base {base}"); + assert_eq!(c, r, "qp {qp} base {base}"); + } + } + } + } + } + + #[test] + fn quant8_matches_i64() { + let qt = QuantTables::new(); + let mf8 = mf8_i64(); + assert!(qt.mf8.iter().flatten().all(|&m| m <= 0xffff)); + let mut seed = 1u32; + let mut rnd = || { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + seed + }; + for qp in 0..52u8 { + for dz in DEADZONES { + // Every magnitude below 2^21 (8x8 residual coefficients stay below 2^20), rotated over positions. + for (n, base) in (0..1 << 21).step_by(64).enumerate() { + let mut c: [i32; 64] = std::array::from_fn(|i| { + let v = base + ((i + n * 7) % 64) as i32; + if (i + n) & 1 == 0 { v } else { -v } + }); + let mut r = c; + assert_eq!(quant8(&mut c, &qt, qp, dz), quant8_i64(&mut r, &mf8, qp, dz), "qp {qp} base {base}"); + assert_eq!(c, r, "qp {qp} base {base}"); + } + for _ in 0..2000 { + let mut c: [i32; 64] = std::array::from_fn(|_| (rnd() >> 10) as i32 - (1 << 21)); + let mut r = c; + assert_eq!(quant8(&mut c, &qt, qp, dz), quant8_i64(&mut r, &mf8, qp, dz), "qp {qp}"); + assert_eq!(c, r, "qp {qp}"); + } + } + } + } + + #[test] + fn roundtrip_4x4_low_qp() { + let qt = QuantTables::new(); + let mut d = [0i32; 16]; + for (i, v) in d.iter_mut().enumerate() { + *v = ((i * 37) % 51) as i32 - 25; + } + let mut c = fdct4(&d); + quant4(&mut c, &qt, 4, Deadzone { f256: 128 }, false); + dequant4(&mut c, &qt, 4, false); + let r = idct4(&c); + for i in 0..16 { + assert!((r[i] - d[i]).abs() <= 1, "{i}: {} vs {}", r[i], d[i]); + } + } + + #[test] + fn roundtrip_8x8_low_qp() { + let qt = QuantTables::new(); + let mut d = [0i32; 64]; + for (i, v) in d.iter_mut().enumerate() { + *v = ((i * 53) % 71) as i32 - 35; + } + let mut c = fdct8(&d); + quant8(&mut c, &qt, 4, Deadzone { f256: 128 }); + dequant8(&mut c, &qt, 4); + let r = idct8(&c); + for i in 0..64 { + assert!((r[i] - d[i]).abs() <= 1, "{i}: {} vs {}", r[i], d[i]); + } + } + + #[test] + fn fdct8_matches_matrix() { + let mut d = [0i32; 64]; + for (i, v) in d.iter_mut().enumerate() { + *v = ((i * 97) % 211) as i32 - 105; + } + let f = fdct8(&d); + for k in 0..8 { + for l in 0..8 { + let mut s = 0; + for y in 0..8 { + for x in 0..8 { + s += A8[k][y] * d[y * 8 + x] * A8[l][x]; + } + } + assert_eq!(f[k * 8 + l], s); + } + } + } + + #[test] + fn luma_dc_roundtrip() { + let qt = QuantTables::new(); + // DC of a flat block of value 10 is 160 per 4x4 block. + let dc = [160i32; 16]; + let (lv, _) = quant_luma_dc(&dc, &qt, 10, Deadzone { f256: 128 }); + let o = dequant_luma_dc(&lv, &qt, 10); + let mut blk = [0i32; 16]; + blk[0] = o[0]; + let r = idct4(&blk); + assert!(r.iter().all(|&v| (v - 10).abs() <= 1), "{r:?}"); + } + + #[test] + fn chroma_dc_roundtrip() { + let qt = QuantTables::new(); + let dc = [16 * 7, 16 * 7, 16 * 7, 16 * 7]; + let (lv, _) = quant_chroma_dc(&dc, &qt, 10, Deadzone { f256: 128 }); + let o = dequant_chroma_dc(&lv, &qt, 10); + let mut blk = [0i32; 16]; + blk[0] = o[0]; + let r = idct4(&blk); + assert!(r.iter().all(|&v| (v - 7).abs() <= 1), "{r:?}"); + } +} diff --git a/crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/avcc.h264 b/crates/h264enc/target/agent-h264enc/fixtures/h264enc/oracle/avcc.h264 new file mode 100644 index 0000000000000000000000000000000000000000..f62d39205db7207980b68721759351c57d896102 GIT binary patch literal 15264 zcmV;RJ72^A0004JWB?1SEUYX}z*>NSfS>>X0{|cZ00)ud1}6Xj00C(3Jg@)&009OC zLV7$OQq&7hj-6pI#FGWL8fIy1ZDVp_W^^rRGBz}2Zet)YE-@}JAT1zcY-M3?Epl&f zZ6HW4GBz|IWo~0{WMy)I00001Ww-!d0&FQoXmAYybzZ-wid({3uzxcJ@%>h`*;1h*c7z2}949QEraBHg6uOt( zP-kqq;ml;A^Z#T<#{Y*2Q%(a~MXY8J;*j2eXMhQ0psuMg1;HMe)fXBLHj8k(m(UVc4#^ z2L(*6f510Z-|Qm1;R%6-WCr|Hl!VTLcIb;M-wm}%VlLcH!MD7Ft|Db&z@CNiSQkp2o;X(Zq7?N(ZUGN6H!QaHOV zBA4Ckrv}grxL3oTbjo}4W{ly(*X8kOYK3;J>D@&umPq{On8_I+pXuWQ+yb>oHzk(T z#MYh3@CRoZFeT>;_Kl{sE@|Gx8gm)Qx^s;r3kb}>u5KHtHDf;q~*4^#hTIIOO< zn6^C{s@@A5f#>Ba4yXdR_(f+NR6{LsR0zhgq!O>nySJpirm{0vjDMNfNc!baARJQW z@$Z%tzn2S1h+yEAMrI>_(*;8pqXa`eMupaz zF@oEa#DT)RQm{m2n2UjujX3I)gK7`6wv}Xq$?3-T8z~r)wlsNCx;b6uk%cJ2r)Bh1 z)38v0p|#s)sJ9kIy;%>G1x8PwULN|Pa(Nlx>-yZbo0WYLv+^&;=n`qu?GraaJj}(d ze@`l!;nXN|p?p>=Rbkm^u8|hi+&QMv_7Ff>NH*sIZ*B$ALc=BU7nr5u1SbNgtX2o< zNBrb;__VDGuc#nlcEEI(2<`5=`raHHT4O+TL^#fOR5DSea9~8Hn&@gy=uP+TcOra$ zx@qYV-iZ#e3|=SU8ds^LAmF!~5G62v@vit!SGbcSY95q~Ag51G8glXNM!#$>i4c8F z_Z{A?C3Fz(E6G{V?B!L0NLV{c0xc&2vmJiuc#`9zFF((zUu8#-9^1c6GDpvE z>{OC*3y^VgaE{IvorsG)*S{JM*Ld4k&gwf|ba-UaDox=5uELDhMq_OQkp63Jox%)R zd63HsL=&#DD-)oo^vmt)5cW~fYHg^NsKL%29rSQ@11whPEzw1;er7Lu`}tNX9mtpC zOkq^qGR6lH;)@{eV#iaYyL!EOndJ*OQWE*nq+L$mni-4m%Z0>ly~ht3yVGENqrchs z)>ofB4h|;qhGzQW)ot6QpByvyJpKUWpc%Zl;xx}rKg3^>f!#hI_8_uwvQ)eLAOHRU zSRz1pdq^Ral+}wDUV8bpvUX%@N+{(VMY_Pqtw0o** z2I~HuwQd&hAsCvkVDNSzF`=yKh209-TY_~wZtbO=Dze|a)))Q%fq%TIl1(R9BS`lk zyPEa>i(F1;vMVRbQ_emCok8<(<1G~vn%)1&;ZLJ*TIFd3**8lMS!x}6oJ)=76DvQ+Wx zfgVV!Y@p4#nEW8?Om8N;1HxZ_!eFc}DvUvQtp0=gyTz>t{w2^Cp=5$wmNK!&prpjV zE;r$x6~A-Jt9<|nbHC(HI@6v=qxjArbnIfDD=tUVI9`{c`e>zDCnYkR!PW7REtQtOe8U{hxasqsxB*J=7X@nR?#WJ*Wa_E?1dy1HE_SzPr@9VubQxZ4-#M#cxjO+ zQFlqNkHQDiGl^@koG+W1Dd4r)#FK!Psx@Bxv8=@97T(kr3v3lNh~p0UK+;9LzIq;t zw=!7M224~)TrR8)#+Kmql0Pdw#Cs|mwm3e#rn(*bbY^dp5q`4%KRwU2;E1j9rkN}{ z*%>>_e4K{0ycxAV1M3WnORc)214K9Fsjj2aMwfJu=VKl zKmmz24sZbAy7ctJY(SF!s5>|1{Td>rn&R;U|9c_#dFaoIio`J@r#oj=MvF0$`o9)) zxpLX9xsJ}A4*gI`a9KzNpS|OiWw76A8Wnc2W}dVZ6AH5fuX5%EL-T<0t`IKG&M+6U zeKw=MHnr^1vb)3^Dh8t5JG8)H9ZuMIkT1cL1*`Eck*Sy#7(%@MlemBmpvN`A?Rqyu z`!NOBX{e~WkrM|4<9oJX#ROtUxD#uE!Iti3@=q9yL)Wi2`MwZc$`6s_)vH_nZGF`q zIP<|?Pg_<~b?O0ndM=|Km^;4s0=yeoJ&@o6;y+_t9X@Uj&sTfCJ;cv8Q=&T~z+ht9 z!{r%=M46lBJR_$i{w!TxLXDQ9?Q1&Mz|?sQDm{b8Jv(ppZRjXQW={rzgzw4;Q)hYo zE2A4u-uO(ye5DCIhC84;iqf2?|If1K(Rd*xuh8Ik4*uHhbvi z=U6`pzh`hQ;LgPxWq;C2wZE>q3W1eHTWG@813f`=a6V*kA#3X&Rj5}yv!1b8&Ma}& zy8UiMmrnk$BhN{^t)z&O{pV$XH~N#farq~3HWED*A@$er63QTxrcHb zo(L&QSyBpY(;cVepX`QlBQePa?{!$g_?g5xd*)T{lw&n~+?2rbmp)z>gJK!#4m44B zXZMFQRwN15+UE0>jYNSye*hpi-3xkE?S^d;uDEx)$Du3R&$W$ymkZ1R94pI)Um!|0 z(W_tb9oP+IDEsO;eB=x=oq&|P^3@NWp@Yf=Ph+UHVSi$YJoUK}T|7z{BI?@vW?#V^ zI9)8-+I~9tf#tZ4WlUDgr?CZZOm|n`(EhOd-!A$}&W6R=K4ZlMVSIX!QmTYYI^U0M zmc3(GIZ`eI;^1w%#03BgGHzMbGs`XTmb;%O+TEDF#1=(!H&ZI2h&9N&KCqt#&|g!m ztZYPeSXOyi!}B^J!#Z(>uvNL4p)DJHi~B-vUhsaguF#f#VK!=!mG={2aRn^U;oPeb}F zik}ZIWfrjrxkf}-tM}YQ^us)~l7z!#XFxXfZjnQ6yO)>})ks`g=!|9Ii3JS>*EN~G zQr2+O)Xgxt`{8nQnTN|v?Y+nsXGtvFVpLxnplGi!x_C2i-Idc_ z_7wSmT6o4h+e}C`3JZ2LCR7Vkve4830003&;2?wlsV}N%V6LMvr_ycypZ}?$u=5HK zW39y_ar3Yr|J4J(`uZ_LV$7EBhBzK&P|pc|d<(E39)ae_Cy775q*)4+LF2<78y49H z+{!0#QNH@7tQ|JDh$J6jA(>vA9q~<>`UCUl-6Ty~w6p;>O6+x^G<4&82!JhN`?I%v zfY1u{sQmbGN3O}Q$CFGB^ZZTvB#h^K1WySX>-9mX0vj!LaA-BJiumQi< z)$LgyL=x3nepMr@k8Dj;`{iaPD-B&O?#~d$-fz$1Q(Je+qaCpftfoaZCvvAZlU}XV zrW7{Jp#H=Xd)|#5DG&M_r-0SQA;a2JB5taA3nXgFlh;33sq^>3eEsRKlC(TPOg~>H?h%=nXjmn~${Du>hyXph=iL0;(!OzzS;GRl^}B7a3R8k;Is|Kw zLU-X_o%8xBvZ)`Fp$qMwjSb(CEoQ1io7xnNDWAlIc+QatoxOap7(Pgo^KOC$_F}fz zVGS|VTO4u#_))aElkeI?$Fnh(Qb4;~?!BgH5QBR)Vyy}f`#`& zDP57m{ zLsl-HuFKQ?yJlT<-Lz>HST9yu-y%`*CDmf!`H^yYnzS7koE+x)WY$#>4-XL%i*=+^ zT)UMT;DxO&+-6}~h8);O@Eb2&^YjF*Ngr9Ml~&h{mKs3LB?tYfSzC@!xceq=x{C6P zv(*kPu1K+-kn@FWnhgCZpMI7}Dtpa0AyT$l0=NNIR|T=xP;>?<7Bh1oY0i6j(RFf` z%ZKmdb6LzC&(ys4XuT-&}`lIzW9b{Trzg_ZYtmmfjR}a&hYt>qw)D|FuBBT!CkcPNlOFtk-cC> zt)bWENVcR|*DrsLNrh$XTxo(a$rL(ytHEwa2&6CnaNEizS0!?lR8c$M!>_atOD$LE zp0%FTGZL#qrut#=>@_b~q6zFI@ zAObLcbh%Gz;e!VgyQuTtfG|v)nxD^AIFeCx4p)2!vYxpV_3&0=AUPBk&Wtz%xnW@7 z%Zyy?=-!S<%m6=1Zp;zx`qz3*MG+A0Fyri}KZkLC4Htfn53zJg6|AI1|l!+-W*m=zzZi8VsWj?Rl=8z<-7*lgZCt(-{RC!NSi=p3~Q<98i$-1e0jB^KfT50PS;^ zLp*AETWVVrxlkrEZ@=OUfQgxZ2+wn;rb{sTv|5^DHHeAa(bz-5OD4eFteT>6sbU#w zbcX-}5?JR64!jHcfet_W7u@(9b}`y=a{SFCw+e)kTBx&l;>vl-`$mytQBsVp9w1@d z;>Sk~wJwt;$8}=2>4KFCoP8Nw)ddcHIyoMsg|(Uk=-7!qZ-vhWh$Cy(G|tiXOfBpXSMS{1nD!Uz$;;r5;Pr{sjfxPSQUhU<+ zF53%;f9EsfozcTfJ+;D~>8$GbgR}X=snAHF1Nci|!Ci-^B^vMJ)UuCc0!BZ--@qoI zpsbt>9InieEHxfZ(uCnE_HA#6vXB740*92moeWF^&1!=a-{Thn@(REwwimGSXHVKO zFMDpqS+tV=sGlB73;h}F!${<=xUB}1qqmZ}xxPTRHnA#vm2d|rcS{OBP4H|_DBNd` zIMioE0RgBG6iNS^;AgI;&vSSAlp6DfK^iRI6;WNKWC??v6DRMpyKes4Mz)P=!P|92#~diwB9U1 zM31XAEvehhg6`6eOOz-G1_9l`u6dB;4Bv>7=d=QkZqq)&;9Pkj&Imxzt$adWNv#)4 z0QLBB+@$`m(W$1mYd3XUk_ZBoZe5OKxBxYskgeFIs4>&hdJLOs3tix z)pICZV#}5O*Y;>4BSQs|Y?uL|g#1)Q0ZFbo7RUZ|-Mi>F*#Gay%lceX6|hQn-4EO= zd^ zXN0`X%I}GJgZuBH1LdrZ`tg>2*b4%;2k@@pf-MWBb9gh`n=qVJ&Xz1%8J*X&?fIRm zb}lBixg9~hvt)Js8`PD#r+XNpXXE(Cl+%+D@6xo0%V20P&bs_$Jo-Qw<>vb5RwD7s z_pE4rbo0EQ$wc5K(nVKiaAeIw>J3R^MY5>LhJZis^4M6qrfc!W$zOE67-f~Axk)v( zQy@{ixe`n(Wp3xWq{+G@20;2HmO$!jYnf5eY)MP>s;tPezJC%}1dajt>-;MZ!mXt?-#}NLh5vQnjByG)wv-%Kh!Bb9MKi|-qt3Mt)^~oL6PmuPR zrEB`?zNj15oKWXKey!j}o=l3d2vaE`$K*r75|&8gJHFq6TNBPxn%b+z;l{bigEBx> zGAMPXmGY;O-_jQd$TN^*IuCR^){VD5=`$TbLqKj4^w*6%85IA@MUDITo1gSXe|5hD z2b>AFu(AZL*}(svIx#~#0~h3{1Jkkj(~=Qi<@a+iMbZhfMqHUCL-ShaivIAnv3@ar zns0#U0Y6fi-6q2>4CB~IgN{7`NF5@ZgCGxW;}mV&c9%vGj8|cIbO6IK+GJ(gVbB{9 zNcme#WmHhxnT3LupbOmNB6_AB$Q`fg!cEO!#k9B zeqlE9_Jg!-u#E|=%>L*8_6Zb6XQ{amnELbr1mw6+Rp-oJ?!bUc#Jt-jRFQQtZCDoi zN!+wMkR@7SM0nKBBDL4b=m&|kX)MlK>dWHBlkkHo&MCQ4F`!t;t~Jo}A^_!&h9Sl- zpYzjgGi-%9h9jRRZNe3_bLG{RhdOG{F*Xrizs5roU@yZ9<&WTqLrF3xd+<6;0hY1) zZLi^iD!c9yZ68N`00001LEu3H|EW1gHPv#7burK@P6!ZVmQ?@jpF|s_iPyx;AHi}y zqTCD@>PsrcI0rMW2%T8669p$8>W1#JDehU+!!?k{4`w{H5l&Dcs`tPICzh4OF^z9HnGxA{%+r>t zb-D&P0Q5l^p!_kj=9f__kx_UToK6j+{+lO-`A3h4V`o|+Ymtx)&MwqSeHw=8aJK8anjtds-pv}dIy)1wu zH^4wlsD_lu)BmWVDgAte?@%Pj>GT0l8ubM0h=!g zv>gvGo`U(vcq3LvcM)TW?f{)~4Ws+y{^#lIk_n%2V;(nDJ!qj~4;VXsxBb0hITqQ> zSiKSxrQ!$eFF z!v^tm^~P6XXt8SY6wNT_v_^iE_}s5djsA=&Iv%f=&SjfG{YEl(Hh)#I=b*7-B(7L#~0{?Lr~-UhwsZvIgrl|cdlD(AQ6X%Yb;B5MGf zin>OT78})|%3}~vC+DDYK z+%!jsQKG#blAq@;6ZKN|fAa5pI&EFQX%%JKE8K(CwJo{^625w)$L8g_D!&Y2^-{v< z(^B#$bkxH5(VM7edOeG$q^j0D?15msXBIJw&w9eRH0ITa5C-AeHiCJ*rj`c;%g`vK zy2oJO`|~v5A~A*uSO>TC^Be*i@T{J!*K!K-5m>Xkr=hRF&S)f84>$XEbHnCOU}4CE zX=7uDv8!hb<7$XCs%uY|^_?8G;mZcTT#`)r0DV zknK=CFjc5hN>(`Xpc(MYkDS(d#I4>SGUFszzyQ`7Pg)OHW+a+8QaJ6xLnw8lwv5B# zO`%aoKbwzZ>9$9dK#Tx14=Oouvy+DpU1uIVRk>;xC*$E?%2wZ(&75XPBAkeAAdd3dis(;5@Ilp;OAN z@ou)cVaGT^egCWJA?~%XrRW^BId!!&&5ckyUk&aRRcg!0@>za9w1CwP{D2hDsrZ-- z_nIusArXhO{SmmhtR1lA$uf4$TD#w;vy(3(*hhR&1DoaW{NNK{)&a~&^7aq1dnW7- zHXE+vPth+LSvtL}UzgMBnFB?X6=xJ0Kd_CpzL>r2RbQex1RhtNZi|tG>ft{B!cwDb zAQ>yjV%IgOyj)T*td9R}EWM%54Fv4eXi(M(KI(~R#`J1K^nQyM4Gcm|hu}_AExZX2 z<>G}=Age0RRhuf#w(|e}a1sUZLM=DHuJ%(g_uL!=I$RqFcQ659!Hi0V#^TnEP#oc$ zrpa#ka>i-yZK%D!rApC7xMsWhNU^HQGXtHro;NX9&+!|O;7U-mu(jBLpE(q9YsJVX zk!AgLrHS=2Z`H*txz}n=48$*$LXE1#*_$W|r*ds@L#;;FQz&Sxcy7^51clUZz7#YY zPN8+S!gjuj6>V?{%_W0+7KWb$Q^Uj!#Sz=u1+9tAIj3)@oCpK&=P|gK_7WWxqJBiF zKVq18c6r|iq$bp^Q-lspHjp(H9DG%u=@q-f3-K z+Y=qBy_n1Ces1T*AMEgkh)~A5PP}^ut{d=mLD6cVLX`n<(+rxRu9MKKKf8gJP0Q7k z(d!ksP{PUr7%!oGC;7VtfXjne@roy;(>AZqpdR8IwVRSSK;lskZRBtS!|~Xgvc?{` zhwEcY#*3Ya31%wQd+Zi{dP}=&*>0+7dN0>%&#OpbN_g|p&)Qbf=pMz=x{+D&UyehF+fqX3!@mJ?Y`mS2| zF=K4sk)zg>%{(=Ja3=$u-~??-&G%^{R#vS554OMd9!E*##+(7o(@uNtCdyR`fMU}x zk!;F*CTb($GFBOcheddi_llXpU>DNXF>*{#*t02|;Q@RU%_z@7?$zT(P7we(;fyl? ziArQI=c(^CnyOhhIW#HDWJL7Zs-jgfB!+co0Z#na4UDmj2tfqqik5LZj}F2O&QM9J z(78Zfnk>b8KS=<})o`Brqh?sYWmbMAe7|n(Zr-xYG^M)YbEo)O^r27y00032s2DU! z|C`_vDYfDaR_$YT$`-? zHJ#{A_iCRuE#tu;>pI>Qk=p>aG|FV(qTM-W1j7M8tsX#;c&w1FuX2&LFO8fRFZBxD zw7NoV2Q%5)&TQQNJr?!ja|@9>%}1n66(p`!{{a7S|Jp24fmf}Kb5XyTmn&|44L{In zx~-)ehhC7uoBlgDLp!?Tim1F;2kXxCzT5k9O=4j9?#~V3n zc}n>vnknhso?UN8xNLO_O07NsP}p2ipbnQ-Q;wgk#f9#ap8we%&(H~ox?%SPA)i}W z(lFywKLCOG;UjE#uz<_L*;Ztz^uEQP=H_FdfDfX2r6i7&?4?#tDafJm{)DarE~Wr}S&LkFBbl?(Pm()#Pw5xYF$% zsPSfNXv?7@EijVWcl2BT!bbQ)Rx1S6CCAxbdXFo9x#$G5a|mk*7VC!e;dSYGk<#=x z6bWFX7a1lzwpeR@bBqI{K_#|z3bY33xb$3Yw~+lx!kL6K?GB6&kInz=Uxp8UPssNu zpfUzlQ8b4Op!~IF7EQRio@iQsX{{EZKg?fJOjWlF+0Mary}{0B?5IKZA|+^lh5ZF> z62=#V#LKdcYP4yQ>YR@ZflTfKp}cPb(w?4QT@J-7HbVtkI>Khn+b;PtF%d%@`#bLo zS|iK^4o!9_gF)0}t*WlI&;x`WYs85+DtLS&HgNFrpyT%Wu|Ns@D$0610av7ou|nKKhJGJ2Hz-mPBZL^*oOLg zxC>j+$avEF^dD|9nqbd+7W)g(TrD4h@OXbLIx29sT-;=490)-%2Y+VtR(s`6oY0Oqx~oSSeyOU;j71@2K)62O9eJz_3Q#|0 z%h!0*XHmT|;sBBdE-4N3aGVNje{jBkVJJP?)%4g0OqqoN#wl=lrMW<$i$soJ{^}!g z_t0}_LrGapDR>h#wZRP@t2ivVya~Q>&twDMKBBLfS}$N&M40*cG*0l1?LBnyNGa0003&;9-OR zsW_T}z(PnO(3axcZ`ho{LNn`3Rs#!A|C3=kMon^&a*R7K^liO9=963(d`N4)5Kn=A z#HuBk$R!Ss6P@I9?P+W;@GzAK*^PwomZoby^o?ycbzftSdfRz; zl_9(oO6@%mUK;vC5DC~duu^Rk2|saw)`6r|Yam@aXQ(E{nSH{=*}>QzAo;D3^{u}5 zVJxke`U%tui@*(ee96u3Fuj=a^*`qQy6FXB-_~j2;jO{m%SH2XMB$27#IRVOg_k%^ zcgvmpqR9}CuPQ8vUd|UPN<)?X4u5{H`<3M8bKQl%s|p}MmY8FK|Ar0Hg+vguu1Z(D z9frDM)s__!8EYVD;s+`NS zyzTV4B*BiBeoqvVZ<0`!GQ4DvjQW@0Nvfxmp|rt26cg}w0k280REf%NC#;)T zl}{%?3BtJIp+ugI7UEafl+APvE!$l9rm*OD5UW9u&%0v1H_K03WQkvm?%-K^ju^Oa z=(yK>!#fKLn;qLTIf3B%3G=3!u5ccLU3=Nrd)iqh5FTF5?MBajCuH*O3n420`NlvC zv9mT_-GcBOT+U&;kG)q<#b1ZMpT0X=A`h||eN-c-YCth#57-~-p}|e=InYBoH?xt` zE*hsuk{h4u+dGo;Go-5Hsch=akszb?|SA?x0oxXYy<$S?$Yg2!@zMR#p?SJy7z+ecxfoR$IreM+s;E zX7f|~{s$BK;0PZvs)@hPDY}lCSZ!|JlpzHtx|+HOnIqGhdfnSzu02{HjS|iRxXFW8 z%yuhz2$RpH>MzD4U^e7vJ9l7x^{l+@pRVSIXcE9qiRJpNf%_%U>asqC3|0wla)islap z@3n6Yy7qawkdPyM-zWAl{b?Wz-Jm;vRe2QVr{$Uk~Xo@tByJ)Ym>g8Gh!=jZiE%yj$?fqW;&9O<8; z)E@V{$ne6GEWPpi)hIETp<*4vf9hqscD$Y%GJ6DQ^_E}T;m7kIITs!loqR_D_(}Hu zVfA?wrX0@l-0}g%>TySb^^Jfom95op57B>WrEO6dRKo`U+(wNZPeM&Qus~ zcR+neG4uvngcyrSlNUd+1isK9te3SMGaHi=%~m58lHT$#5sZ8Y4sx>WWMK#Du=n1L zmh`aZ{(whLR4|m#6af7^C1qyVNU;`PoctBUY1U|Ac>o!yo|5kB1t;nphLXviA>`3i zN}l%G?!0&Vei^$2{H3`^9pSA0i*Mp;HK_YwBDU_gE_YTwP-V#{WOoXaWNJq2(y<<1O>jjjUt#0_XsJaVJi&xd;!UjlT&P!<5%xfsI;T^bVSeGu zmnWd32S?@KSKB*Y0LGPdVVa!JdI21&uO7$61WPx2MENy5*6L$2AsCbTn0k9&kMZa! zM0(iTa;vJxtv9&N)iRyGkXepu000010jMBUN&lOMgl=@IcNHg`M;3a3mbFM@9Bfj( zC_OziHEP*ZJavZ3``vOPFtirqCXz>AlHer4E9_@A3v}#FzBk;H9ypTeQK$Di?v z4*fW@wCxNr%oNUw+T2XrpLlm<-nUd)4xD6*Noo}q-gbn%LSs_%<@?#M;ht-*HYd<~ z4b?y*5VMRiXo&^I)c~N`EcXXGjp!&@=1-Lg&8^8zq7B`2H{R>|0^*AjW22Aj>Y(qKD z47&=GX1}ZNYxt!|qB=gaE#FH4|Me-fWGxo%%|{SzKjstiL8J&JYi($v!^;yqbs6YM zJo#X^p~YD;M!Gz2BlcW~b*YZtu@u9-=LB7L%q4#(2sniUj=T4%Eh1F zV3dLy!q5Mn&KL%YHIeJoW;@z1)$lj|2T-j){uP$QTEKtTkUc?+``sEagZ!aatH@4lJt zWBp&FyG6yHcqsTfIh)UUVa3}K0$067=V@kP-A@;lsXkd~n}DwvDFH>#^QE;;7FNUX zLPZ%kh5*X`72L#0Gq^c^!5l$R@b+K?xlT(w*_OLfC#-K$Bq4WTA$5B~#=ReW>On$E zDJ5YW&5UN334j4}y*k%Rr3zL6W++`6CBGrmJ=DN2Jd{`;Qo&!hiy*WBh%A+cif)^J z_*;>}&VofdU3GtJGgI{(@Qw4j?P{kzQ5=L1@@o{M4|N>$os1fMYrXQ<*LM2w)OBN* zLS7@Ky@Rd}fp-s)&g{jrCOuOPVzb-lFmB)3YD^K}+7#1uXw}5$ zBQx;4a>a9%0~YG#Cv(Z0c_(3Mco?s~D8VlYkDKA*yF(v>^9SFpP(FY$+a8>B|3yeU znpL`+pK2<1JRSZBIj13s=^>3+p;A_gMV4uf>_46^NL$#lL?P)n7YQOV+fKcHEaR?7 zlIpB_1b>|~v+9vHoGmNDX-!svxRT*q2H5kpOa|0zy`!U@KYZ%$#U?LZj=SE|UO5fs zj_OQY|5PKf1P}Xp<56`Wc1IURf-e?MYB74GFV)^!%#`$77cfTEBL4oO5neaJOgwR$ zTn2+0QHNS7RKKa`Dn9FI-01>TS|^*YDj-ggZftPZ%~<+ZVMcPQKy@~&P83`tg~tqF z707v?CxB+GKb^X3A`1O<5SR0~oRBUfw#B;v0EBTyQn^d-ZVkko?NAY7eyStN3S5lxV8Nh4yCyn7TcmMzZ0YTt_!~d(wt*P4YNR)+J00UWF zfv&J>oz4JVzg%Oa?7z4H4N+*r8crweB6}y_^yn2(`Tah?=G+0{RL+oO{dpKyoRJTH zF9@@dTJ*Sb0v6N@0{jvit6 zAqZ>4gZMs8g0yDMAYEpV-OQ~jW^D*kqLlI!fBPVhCoimvrE8*BO^66!$AZ<;HdvX0 zZJ+(fitRfzaqW3A*%s8euTLRml{vt0c#h9Ez}KaSOY$*l#p3Y*IN`f<*fKQy6fLAB z7%aC--|)wq1wG}8j@k_vyqwpG$1)>d&AI%Z`ZZNbRD_WKzU;syTlTu#o%TOu?RB;i zlMJ?s7d%N5Ur($=+u=@-k*DbAGUIr^X%EM~#G5e8Skk=UGXs@ZF$Q@~pymM~RwTeS zEsJg(aYBaW!V%Hp!2hr1I&C^|@HsslOB`%RP1OE`=Hfk=2UrjbDkh z0*PSTx=3(ebMneS7c;>WX#{(_x~{=TcBVI(p9w05F}+3W4=|HNY7i@ypY}|AB4xN zO51pL5aqfV`d;t;ev-$Zt4V8l?piHR1J-(#{O{EOz`wbeUd5pn?~uwu^19iqL`#qv zRT?#!toQm+Z%YpshmZWgIEO*E%YV~DqZKid$JpZvg=_fD;YNhYG3LC5^%5Q`i>p2+ z;7t7n^eb>;qcAQkJ(<94=89rn_?R~@GCI-G@0~XdZwaU_p!XGzgeo+dk$R-Um!KHP z@6BN*zFjjcMpw}&x>-{EUAGJDBR;&~B23SSI66G5zlVYxtr^5&qr|#um~JQvD1H(z zWe}n#Y%A`*_QNJqPKfHY99&1E!M4HoPsj_|KnjGfNJXpJY@Go@GOQNtoA2%O@(dSP z20^nJV?bhW8xvSSoX^6P!=hgHo~jMOUgLr(6lRwqh{pcga+opon;FLDdGo;N)wdD< z854KC)v>I!OE*r%SqvqK0eE<)R`nv-AN^TX@C0_b#FQ?W0+ zZqGrhQB@{7uy|CfLY!(7kgbg%>bQ|Hrla^P4ePq=!MaL3AT4r`O zso+(4zve1IWv9-`+$sr+r%709qEAh7T{x@SVW*alov;7|2C1D?POsYAQtyC@%L}_J zk>e-_7MAqkvnO*u<0m9vtOOJ-54{HKLncrKqP3nIfPeEgmcJ2vol(XAO?jjoVVe zh$b0Pwg~<%XY8cAe5Cjvp*@#M@msZvvth=rvoa`t1t+&7uqbPHn$IN7Q^b9?%g0Mb z25!(9k8n_+PZDs7JzQ#uWO=@EJ84<>fD9l1qg&=UJF`u_cq{>LyfbIVx_7OrHa_Dh z&wQM_X_VEUGF9P**aCpFc;X-jXX=gbYL_Uuq5T_{6$s>|`y(JXl|W1a0K4s<4&2{O mrS_Z2ga`vICue|?4$oyRubxWHhbsWkdJC%_8f5{F&E_nocB2^w4|nSO`T#Km|m)5;`JHETAYEdJz$6B7#(rCQVcnkWd5! z6;xD2y4b0LfKui^o^#Lr?mg%Guiv+_GRc~GXWq8=v!7iC0Dxj90_5mp#9+;`<&&W) zQK?ib05AYKjluLRM$7-c?E7p7^!pc&GdjCN)Mk)%e@dR4N@qImo`CmJ{{W8=H}8{5 zzUtbVUV$DIRb@40Rf-bD!{5s-P$?uRD1c(7j9jB27lS;#LjHJ?S0ff+YbF=s*E9jq z1V=@I9^E8}fT!95A{&%P9TWS|>}6XX&=@=c0SX5|gXh2`5k*_z_tXI%1&m4c+5It_jGEG(eHVqc9dI9UTWHfrCg3=m(60q_|Ra3&};(iRuR@QHfVbY;a~J6yYd@%`GgsE}Bg z^}$?Am?kgUFi~nEv>n>kj7|tkW~l98dvP7<0VQ3P5En0Lw&m zrt1b@_M9so#E$e#7mmO~fj)l<1GIe7BijIJuvLtV-YxZ@PC0_Ix;f z233<9+;$;iE7h`*g@!7XJ;_afY;|+}+Xl)C*1NK6@4f-|x*zd7IaWGDx7x_Ly`8$9 z=_)6vEX}lK?@&BKl(UneI>G} znQ}Gl^nGVqJyF7>vTQF$D;!YNQRJw&T}F9)xJz$2&;89_sy%(48QV}BePT7w={m6D zQxbZyIrBpX9`=RU&(tVSg@!}Io!Ii_AWJS7rtOMX;vN@;j%}_{0CfkID4ERXPrDsq zp#0OcqO;4^>kD;9v8X`oAj|x)QBmm04k-s#_!+?33XVK<$n|q{hJ69y6K^_SeofZS z0{+^k?H9j1thM{akiC`amv^jxu7Nq}iIM1ON^MD zf)6^};W@lj4OA5_p0kI=3k632fY?_X-SbOsy7FCy0EBM56;y8785G9f7;-G5v9TLo z5Y-RpqrQ8V4L5+eZC8!nHr$bWI|%>-@KFd{l-ghiUF3T&WUGGw%J4q+I#k;x`hL1J zJPUEeH1;XwI8V+@0OoHePbWXrhKKQ#k^r7ta$|$5Tk`;a^Go@wf#NqZ5Rh3LtlGx6 zS2J`KU_uFB>z(bdUKs%x?XNZyW#y&g4V=bmVaL|b!L2mVBlR%2af@3CJ`M)od#KoE z?V=Ci1{bnJMOJjRsH};}J%Y%}OUc;>hkDW^X2W=r1NQa$B9ZA~F- z+31tC!8-W7ySF&;Qef-5Ss*Bd(Fy0J7aIiO`QBdGBa#*(g?fO_%6uZh@@<*E&Y-eZ z{3?1Lkwb^)Cx{}wDZO*nMGd7nCkMgoUJ3bM7<+37C3ClU-f9%Fi{WI#0m(ReR zJDOWn653Q#Z+7q&L{>jIiQ8TC%Hwg#Xyz{76EUdV^}hZktEFWTtr;BE_Ipdys*LpX zXNuJO(>pgCc?;`<4%aVT4}MQ7TbwwnC2R=B((9?RXjKZ|`J9n=xA%T*e_MA{wB5}p zygT}_@U9+q$#ufBLAtwNS6dDTZ?=0re#w}8CesuA38S8i4Zh&M1t0Eu0g?uH+^Pub z9h)FjkQKhS|yQiG(`SLfNL7R*MxhGFbEf3#-YiZodsJ4(T_w*W2Tgl|F^*+RU01jR*N}zoAaHhk}Lzs{Y$3DIY8GNQq zA!So4qs7>s&gqK~cKcHn43Y>B@Y;9Nf613WzLk{VuUO z(Ki`D!{6UEU#zcn1=6901iJQCU1=R-3~c}6@+(dHBVIOz<11u_Js|Wv<4$?GSmF*I z6jzaKR8CsZYK;JCaofx@;UA}ghG9a~q~^r?(UxM2T;U#hc%FIVBNNeP@Ij`yef})} zMGEdv`Rc3IhZ9oR$}BiH#-@|LHT9ZC>N&AYI=XJfk2IMNJFo+e*`C|9$iKF7#E4%R(gQ=T>GU5EyVgdt7{P}= zY!V|-khgpOXwjtaD;94{Hs5`u@R zRzfM8O=Hg@YT3$7bDvuEw2qeo_d6Ey&#e{p;90P6sPqg$SpV5q2wT*~WBuAC+)f$@ zEGSBN5a>^UoD15B7@+4A4rq6k%evk@yh$HlX7q~x(hfYI4gbVkTfs8Y`8u$lL0?}F z^)dH_Pve`lW{hmT15vuS3|>*IN2ml9`7Ri}uaIA;Rf(Wvd98A@!vJSsA&fF3lUx=i}Z*LR$zEcZTD z{*+4Xv9im&68gwpNJ>6(n*Er~Irjsiu(Vr|7t4!*qpYS+cN^|L{EEzfU@LpZOIA}r zbcoa1U<`hEfXJy5Br{9lZoyu3cmjW2)aF41YNP*tJpR=lsU?>$Fh`XnE_4qS5HXZ}Z@o{PNZbXl zLa5-2(|%vCJ{WpR&NUH;s~%prZ>=I(?yHn#J0%)49lyuRt5y(I-C*My7_&w@N$77_ zDzLOOZbZrM+H(F#%7~+{vI%jK%8f)SjH}UBr1i$#v0%T zcp^XZ>(omI4Us=>xkZdDU03Hof#gkvs>d!}@C*e+PQIAhbuR;kolSvnu%6zPaaQh) ziF@05_cS{>>aAkzaD_sxN3R|cn4_KYyNqglYRm$!Q)%U2lRe_vx$m{kvClK^8p_^s zVTR|>m>Yw);^>>#^5H%d{?6m+C826+R5iB@)N{JKpedm4z_`l**k^Vga%y=WF(6RD zeaOLNqjP-oJx~{U|NO-D$FCMZ{K?t=Bl6;*@M@$SwMuS(wVcN3Embyt?wM&qMpFcR z(+%Ymr~fKDQD-ZxHr&ejDYr|F!)E~t+~In%GhypK#n+?ELVe!M0G=bGPMi%qoQ?T> zX=?zLbRnWU^`64)2!&AfbJ}dVP&)y?qZ~`J9$aqSS_ERmh*CS-swCTi&%0JspLOwf zU?^gNX6a+q120v1wr5gZV4g2HRjnn38 z@DQ+Kn!V%uSdEju1l)(;SSdG5ufw0fV+pOk&q*Z?m_zKoJ=z1NAEQ6r&X!hCVyAx- zBknqGVsbeT_Q1-_J$w11Y2KIUIg@RmQS;?o>$)W*%qtgLdNi3ngZq3z|J~cYc?}F5 zruQ}`&u_~*fD_qgk4#IR+FmALSJ*#jq z_qft9}PK6#DZ=s=ok+Wk3>&Ibu7LD08ka9NgIUEE|^JYdWZ!u4W z#_;Rjio7#*@R8dL;PQ`SaXy%@GVTH;IjC)q?XV`n>jYG4k9vT9v-9|Q6rjr-5YLuP zs=SqOy|yAlhID=r5C58AN~j_n(+b&R)H=+xNSKX_OSm-O2)OYfTc>p6T)9usKw;eO zw#!mV0ffb+ej2J`AvK#eH!{4xU}7F$&U4pSN68YF&L|D1nn1dSPpt5x>Lp z(+*l~SuVWEz;|`>Ta3SQ&!>?(2DcslLHsa>7Ed2T&%Ic<0K+Ay@{VVH^Wte+Q>d~% z`$wm@3C#`g`0ANoTAMr?+9MMA>+$v-znx((?0ytLDQrybXnoo%ZZqQc%f?EO{L5FRmi7JRsk7pwLZ0G!bk|GAMXv4y>j-hZ)H~E{P+XVSaEl#cQpzQq#;H2u3RgC!ZNIyU^ z2?Ie(oz~{n!1>X3Y3@kzW2>5F@CayHzRo-R>YBEuswg#E*s|tT6M2rl4p6+z18T%0 zT?U&&IQ%DE5&zIhlTrGJ*vS)|eU@V1YhVttP(kPe&*9ZaLwRO77VBVlMg^fN-;OCb zhWf*u``O1B@qQR)uN~Rnsi9D$0k!j8DpAcfRJLhhR2pi{gp#;v^e}B%E|Xt`7c5=Q zU0@VlxzPXPS?ib+FmlnW5+xVW;f*kidS@*D-FJeDbdSdy!-ony?HSGJh>xA!@7P#a(&e0b%{)8NIiE`D`^~yi&U; zW-+_6c3&m3qV4PY(&5n@3PH{)cFelmVZ61OD>--1`c{xK?10{OB(EiD^U~P!E)NcO z>1d@n`s##R&q2aNC8w%-7#_w5Ws8?RbbhvEiJkJjWuz5bEV4BLB6SFlUT4of9dE?q zUwD_BiQoTq03HUN#uDpv!`|y_Q~-sETqP?GwfTGq(hgU>VQ$s} z0G0X6<5y}{nebBNkzLelUKSd<{V>g~aG<(@wWI-tUsAsdiBBrNO3YPp16MB^haK@U z75Py>;17T2_lf#RZzU(DC!)gRgCtg|y__e#ypQgt{tygvDLPX=Q{A|;+a{McW(Ay~j_UE z%SHp__S025dPW*o=4n7|r~dhf@)ykmVz;_qhH`0atArO3fguAde&TuOpWbLRR2!f? z;@Hl%u7eT|fGPk)ULeV?tSy6fIwWH(sDV8!>fnCI^Jbp1uN zUiSck>m(YGK;ny7GgTBrJPM3sJ}rMVI1zGvQzAH67Z7$sAVn1*ZaP#8G!axUz($o8 zXcJI?MMraQO|2U|7*_i@D=E7^<URM4q`)HFz>D+>ux*c8* zZW+FcW`Nx_(GZBSM4rn4a!5E8_#&G^WybBJ`V-NBCy+J)#CRYe5e38m!dx#Pe=h^J z5)puB03X%z#hE zRRu3tL>!FQyiWMUI&x;-y#)@zgrIWo!pg!|AtHf9!EERo+?0w4PtT2??J&s&7%HI1 z12cI)3=JZJ;uGsqBF~XicqH5xec(K>Oj9d(^wHsC%2=wBEedD_lwhPw1H`eKS)LZ>=PR{^uZ?3?xPini%B__yq zm&!*kHS4oXG}(%0d3(aE=^Fza>4tCHG=F>v+lun+~3r3(E>dVaF&x#g(nbqAhZZU%cm$ z6yf-q6Zo|}ZQn@kPGVl9xunX_*#JCz9$FwgvV1bbWr0{ie*<$Zg=m=y*kQk^FyW;7 zvwLtA_V_nS|LvVOS9<~0{d*g?b~pTk%;t2|%&%qtJNkMchI~8DusyH++1D@TWSE9qF4*Kfn?rXR087NfC1uMO;Uxy~h?i`V z@Zn<4AOSA^=cwU;!4LE65L3t31N9mQ<a`DQ(Y{KB1efIkVn0u&6}Q`rtI-RAFbZ*$FZSPk1SWo>RD= z>uhml+YV7U416hJU-+f`@t_D-;gLkMT?)T~_JI%N{?O{zLF+JRaC?vy*dORG!=M6V z|BhNf_2h5Ph-#dGU&-zK{m4*?{6lM($w{ z@KN~UO}sei)<#U=APztTP?Pdd!SCmVgXl{`hY>vg+ZD?E`^CG=V4ZyxCm0hs?!`DfeEx2`4^=C@3N*@aa#? z@GV_hLOB@)M*zGPI7Em)m9UolXDq}jPMcH@KFU=5r+j$PJjmJ`eI1y zki!!)-?yzS_tVYSZi_MRd5{kAxXm##vl5zTz0ACxRYN--#BqFyFv~VYO_`IIJqnB3 zopbM7C&&WPS|V#CXu2MSx~%Hoc0FOgzAvZ>!QW&IC}XJReFY&IR5l<$oE9e>FxKVL zEpvU6z65|7sEXg+yEZDlp#?MR7+&S zqM5t@BmsSYneUX}(z%NX5l@7MbY)SQ{!L!Aev=oJkoK#Me{QAQzj@PNa^#Ne74mqD zAtS02stsdh`h7ruRc>9pV6rC*pdr=&?0k{|jAQ}!I0KkEqG&)C^B1o;>h%Lz&)rzm zL#F;OPa-~k*!j!!^S`xR_n!uz0thGwKqZ-9zqP5d|aLFmiBM-c&_ zzw;-xKNGdaM(Dr1-}c0x`S6si-e0;7Kyp3!*KSMz3B~9IMEC!n#4+Kwt6YTQ+~M%& zC(SMSF#H}B7vBd^RPw(@gpggY)V~OxX8r%NU9|t%)judHqI$0;I{}fI35{!sM-slB zmB}t1>*+nmy&eAZq319YfQZHe6gjDj;lLdI6h~!2ec@t7Q+8MZDpM^WFmr*4@rU`A zxW!1?%i8}%Q1XaB&Cq}Y>gw_8c<*i1Y$- z_Cx2aNEt{L-OP|`*;R}%NItF3Ulx7r)s4OaLSV>?uAI>CNnvZH4X-1#2uB)e5I*-bB2M_^8ghEvOBkh@f zCj0*O#QTm_j5;c!!(zeE*l0o4aRdzoifSCjcJo0g1S&0t1ibBX&HlxV)>Bz|L zbs|3QF_sa>nviPA4At!g3?=Js6p_`?6ZJhxo^$uh(sY6b>HDF(hIVPcofv>bJ|+t> zeHb3LS~cpl%jlRmUi|1j{ih6&4G6JzoDN7OK>{a)qk`|JID>&IAYx5+GJbo~+qq-05{iOfpcl_CTV~)3I*|#{08ua+#gu?sykPTd{$iVz`p zIv7s_5Mu=h?;%8HIl$kDJSjTB75ukyAg7M}Qzr}%qd@&{=KMD^K`lOE=(D{29p|x%shSIU-mZXPkU=_ z6boU1ssKUY(lR;0w4Qab>B!&huMnVOfql0t@9lGs$|LQ+l_CB8cFB%<{sXh}4?_P3 zBaHAe2q0xHskVdoHe^7ipZ>YO#%263Uz|Qd$F+9tAQ;^EizfVg*bwC(JV9jQ@6;|I z@r?flc^iuVTwdy~ZjX_O)T42!#5`tXt-(1!4TwZ9l)AE%Op#>rg8&f)co*M13UF2P zqgk&r%aY?P@wSN&iiq{hK5i5DPgzv01=@(mi$Fvvq(eYqCGf*Z5XuUe9?&fVswhb< z^ULDuUS0)4s0@mz9V*QDaS1=!VW67KN^qtzCBx8#|;@c_^VXpp8iy%T|00P(~xWrsDOi54IRZ-+18N5xhY z85`FW{qnAa=pm<|V6V(BqXxDcx{j=h=$~KRuShY_Y*$}5U`JoeGnb#r1Vm5tEeR_t zfyW6!n<$2S|8%Efr=<0LJuO+h7NeoQpsg_+>4pFZXbMAVvf1i%HYqmj8x1MqL286v zNkJT_Y$-^LFN2ICUqZwZ1yI05thf*y!77^R(GqUOCzU|70uwXx;TarVmLh}}qL45A zR?2^K$)pz{A@Hw*96L%SEDpP1l3m4gityK#U*9i<^Am=7UD z3?MEE{Wf(FFh+=o4g}0fojd;Yc-~v6Fo=KGiI0)8Qq_?UMRtj+a(N>7=z!qkZ<~(( z&+-1&Zsfvm?OyxsGX0Jc(SQWPF~rSK2pd6qQb#%r1K1?5Rs`VxpRUhuEcrvXz1EP9 z2L3xd^qhF5_SB0Gwnc&}_kVf;d_Tcolax?IA{Jy67@!W)3p$OoT2n6$Gpq;zVp;n( zb+;z_j;ictETWaQ{~fRTVGOV!iGVrBng23-^J#DR^2LFFlJ{srx^T*v)2U`oXi{bn zq3Tq0Wd1IB>!NG@PjGT980YaPcT2A`JOb{0s)%#RSHCOlMC8 zqYx=^6+UP8bcwGN=UN?!iJ$mCVK7IWuCwr^!{5&F$8m9;`X?U!1CCHP0g7bejz}sK zHT7c2aWx}^O+x>Tdt3jDNib2(;s(G9=;M%d8mD2DHR_$^`e?O6#x?y8_Q^{=$Om|nu>Xz$%sePKoa>nDb-)Sdilg-2U@)t8wtIL zKUC>GYd^5-%&ook0L#D{x&RhPpniy{&*=LTh=8TqQZtYGK7ns6ZiWaZne9v)bmF{z z9;3*&KHKq=XK&);8VDpvMimfGs&9P}7(n7+k`y9l@Z8QJQ4)|B!tT#60caPne3ki? zE~y_Bw$WrVj=NLN$_C0;RI$X^u(K8g>okAe2cc0`>QpU-26u846j9bfuw@N z(?guhRNbADd@B3hT<(iid$QUan+CPq-w@sQMbk282SU7K^DGZpF$zo$O>n6|D6HEY z_@lg02+A}?_y{!*^@)ZfL1>oVioMYc0#J~V;b^Bs@!b*|sNKYHpw0))&oBITT9{RT z9}OocRIXM6&Ni4lpUUO8wS8gl=-Pby3)xj!n(H2>?Vsr1W(wPuRid;sI; z00grD&W#4N3s5xY^lab42dgJG{qW6d(n&d)rV0^rSZdOGwj*>=VL4*JC|%O;m(bz6 z@Ukr82mqWrRmcS3?0jElXUv_*=ym`JWjbE>q$j_W0mohy5#zqBJ!1>%hgeGza_i>4 zE}CGyW}5;!S|NNk`p5p#&J){HR2?Cz!qLm{wCgtjs2Z7sn_?`L>!WSWsvlCF_g>sQ z6!Rj8q2_-jbTc|~Ow{Jt;O*812;t70aR+KzByYfT4unF9S60$*ZmFsrtM<1Ws6Mtj zOEcQ{X))PoZakPFZKvsG9a=3x_2B(7Zf$wHRS<7PJ@Q(4;T7xzj=xCtZ`jstyrQ~x z@99yjip&p)SB&jeuuk{pW-(^U;NS;JQ@!g#q#_y1RpRE*(P;1<)>^%vEtl~295t>8 zK9b<-Br+h+dAqae_Hvra9LCVVK>hTj9eus4D@n@7+A4rwUlnCjPG~#edpH69QA{ms zn37J+#k8(4_{otf^w;0iKCLr$cnQu}2DaYV?#e*7u+gNhzX+MqiOftMP~X>O{F*dR zyGEx~ z*$_Cdir@D=G@48V1Q4c55I7=tCCU9>gn{&9O}85(htH{Q+xc`iy)4>jJsB*pvh=vy zlpH|v?8k(rE7Ts=x2~sw1eRFq;Aqbrq+&~i*4S{(;;VS8?*N`*5&vMfRA)GRoZW2a zH@Bs#_|3|;BHb~=0k75dHtwYQMekmzg-c(=uEVCqW>UVhAc#eu=f3$~IRiK>G4giY zUH6mb>zj;o!rL>6@prBxg*do8Li=~F+SMP$+HP^7Bet6K`8$!3N2L;~d|cmz!!ZD% zmB<`44QaK{6#2;UqbV`!_?+U17jY&ZRa4oLW>cnR=7-xJer3z( z<|QIUkDoJ|8}>}CxQ>W3)JpFPT?il*P`#c!?|WzT@p>&)U+-nlQ$K_K6l3u|8GEzA zx^tMEpR%k?l~n`wDZP8@x31J|mUz>sz8;6TU$2nWhav9yeI?_yM6)uf(`o14gu7ux z1*CPdwr|fX0!!`I5s%;A9l)Cwdd}=QDUm;kNHyY!U?w}zyAemkN{#v`2+7PbuDItc zGfoDqsy0-<8wlR;Bxyny6}%-VB}4@Sa9JhZ$3SQNHV>Ax+mb_ zc6bXaYrNtt;#t~uBv^-YxRB43G)gk2q16|^w9EC`sjQAe7-5$FmObyd%GMzq+a8wT zpKlX2)~RJ&;#^5e21~0jRW)aXbS}uwQ5J>^d`RprM~!@b-CAuXQR6M=3tFUo4t3yX z4_$N@ygf^SN^fXCD37Xu*C0S)Ly{H80rfKY)8Uy=+&0=;HlcRuhCsK-fb76PkR0w^ z!uw%tefHZR-k$e8R)<6PuY8`+i@yBw*mjZ{haDw&@+p>!|B&u(CEV5V6%_iE(!pNy z^D6Mxn(8)|i<>gaM)}`hqS$ql5>pvx$@tc`L(YXG8CNSWMrjC5Q<=*)9!_{Ay0|!t zGW)C>3F!Ohj$IJSpY2q0^>f_R9p48tTOCsJ_Ycv~NFx5(S?olUx+N57M z<$s_$WGFRK)TkeSJ~m+2uW)!(qOIpd{mgI9Z$hU9V&3TQ@sE>awsiT{}q!J`22$e-rnO?~QT3 zSolVrnxhWatg6}s+g-D{(;zPLoh{U7>BrIK({VJk9i{ToCh=O}Fp+C(r?XO&IgGZ8d)H!kD<&M#u-FQoz=GnhGF!CUT0r^TsGTEb z8ja7QTxS7h`;$|7F|VcInF4VX-|YGNn{d7Hv);ba+(8FD6j?%H@47g?)P$FPZu|&D ze&A^!?M@qm=Zko@DX>4_#ZJM#-{PaGk6UY-Vfk{r;Mryj zwMzMy|JJZf{X1i(&~g!H&Zc=(Y=m?G_HcH7ps%VQ(|{8_P_t?YXfX)C$Gf|L8U4|}IKNt0+RIs#i#QC~|eKlM}vtBr}F!@WDO z`l+;yZ+fVm{d0`ob|q!++q`*^IZ}?3U0*hG`LJfFYUaTQqAgrdP{fuIbsO0xwL^?*Hy4nHf@w&GGYPyP4|@c(8q=8`(sV_2EQTsh1AuWoM6g5ySS1e%un3gO9xo%qM> zX+nv6-%h>=riu7c9WPwaTKxh*g=Il{8u3RJI+$g+tMd7Bc?ngA{WxcA!R-|4Bq{F8 z@sn;k5Qor7(YaS`nc*Pb+{Pg+;KKz`-Xa$8;^%*Nt`B-Lj4JhL6Fs?3U(_&?cWMT;Z=Kvc*Fgn%C>$ zVCFF4;JmYU?9~{Inj@Ajp+?dtwfGYAOxqkEB9${ne(5bCE%m21Wequ$^uC^MT5G4W zW7-4FGF4U^(Vs~kN@R(3CxLDdM;E+<6h!C}8+xU~g*0@HP7|b0QC#H&`r)<6&-C&g zFHfrby|&^y2}lxA6|P_6$3GN+;Df~*YB2Ql@%?L;2{<~uIfU)geS5YH53jozs%EbU zZyn=ig125LFbrjTU~P|TQ?0bpHkSE2_|QnA$^A-`p%St6H<4L31(ZrdxnA~Ki3{>a zL&8_j!<)lm4Y&%UtI2{)a1TN8^mdMC2Q-rp6F$55m>d9{$&anFl;oeTEJaCBF2CFv zzw135Oy-Gkx$$_rHMV;-P1R_>pUUVvv0t*B>D?kL3ERRMiL%>kld}7dc~CL8)BxuN z!3&0>r%{&L0Q8p)OA)>mMh)CZ%Nx3H zPbcJWEtp&1R=83l&Xe)^aRJ&vD9NO>HN1yAm8D_j#El<4i`Y}%{%h_1Tn+kLcTZ0I z;oe{!2t;g6JBqmbz;Q;P6#9Pl%@&#Cj>i|95~C7BoOq%_*w++U{N6f8bAxxW3IA||YCGW%z-CJ*Jm(g=R%MiOM5DpH?fy_L) zKGKK=@}h&drf$;Q%kS6cj7m!%G_8nxI@VytI$xZ6I_omJwdDuuOq5u}!)c0!chSm+tdr?9tcZnN zxMt_5!OALN$3Fa7N%G&b6CUPwARMSZ8lxSw!ccp6XdvxI)xGuFNJT8pti3MgQC%X> zUXA-%Wv1Reb1jtJZPro8W$h@f@2HYGpT%z!DOpC00IZct#T^|V|HC7wC;V?h`STKM zww6&Tu6OTE3i8~XP{-d|+Txb&-wfWZ%XIb)+Y#qGt*gIyApZ#n$fn@!8`sRcDme4xqG+^hD44LZN!i_$ z?Sxcx(wZ{7AA}6E806oF(7NURN9vg!SBLQ2i3nl6cFZYxR|OxW&Uj+Zm9?T z-c9w1=Zw_Xfn;OmjTR-Ln)Nv>wv}Kpi6H5;Bh&#ig(+k?Sv;=@ z_-m7zHRQv5R)~Ndpe70oaZw$*wGRFg{!jWPFMeH;sYwt~L+R5V=SdiBE#QqWWM7DN z;wk+^HlH9}4EeMrW&q!3@Q?c5|8jwNWs5;9Jv`Srx#h8$0bwnlbbbuIHw^p&MI_YJ zf|}r8)ETR8x#apA@Wwz_y+go}Qh_{gx5F3u!=rd_Y$e8qyV$$J>!!qoJHPe~8r)kS=3-TUS(MzDx4D_1Lj68+ z;|<^Vx5&IGZ^dNm#|>}l2-p)EEvbiGUq_MH*3~cPR2jo#ti*Udm%Vy>M?#?ks8b4q z)%ov2*$h=*J~5*=R|DpliUy%m)i)~U!RW;2qqj|tq#ph~2L_<%aB}?njjWiLX=mHT zpxx)YQECOzZ9;E4fxs}16Wjcjm>SS$2tdAF)V$SDacPta_M!6Q82_>p!s_87EHUjy zICNR{q?u(Gl9t%Kz2d1zk66%TiGMw>@7lSwgx$^QLh^rIdx;JYF)F5_N|Fkh=6>0j z8FlpaVT`#2)7y@s#A3zV(Rsi&E_KazMQ|NH7KR27mCKIsT%orVi2ahbGy8dCq;F$V zogu;7K5G;+zcqoij1AW;!VfW4)NC>SK7%{g z=sYG^8j+d)^26=UPCjgUcSBw6O+)!6iWc)7G3Kq6YOEbE=DO%Sz6P3gbXjYSiHR!1JJ4$6iGA1P^0#IHYT+7l%Kn4r z=T{4qv~{TLvl)BnaVK}UcQx=+=179FCj49n+^l+NJIJg!l)agy?6ouJyzf0cKZC7@ z*iz2Dpu}%EQX*PsW~`lU@xCX|sd28Zj_1^ZN5>BLDyg+cz!RFf>S1>0ef#?Rhrx?b z2OG6oaf6>UwQNf}CCdQKv+uMr2O@q>QU%d5gq@`an9$||x{>Zt*J-?yl2fEszG`>| z{E{W^uW_yvFz^9>+k8KMtGpU~0B#&2{;VmzBHKHUFdn=DR|iM0;_1XygBj3M`0{na zO_6&DBB7Nz zQRU!?i&H}~2~$=*l4tKQK?~3E!HNJkPz|A+$a6fos5aD0+X`0LmhO6Ta7HEMsxEGoynxU5TLx}D8uM`h(Xo_-uD19=UYE;zX z0|*r6iJfGYPSq0O01H;juhAVUV%mfHPs?F>xi-hVp{e~mS*j@Q_B6mWZW}6LoS#{jPc$o0I%M{)|EuJnuNw#{TXBZQ6y7*fPTJ-|(sAGi3 zs>#H0$$6Pf2=D=HaT?=J03i2OBaa#t?ixQ+74y8FuQ zA0@hKjB}IwI@eqn)~lA>5DIq}F!9I~NP(igCS2D6_P#CT{un7}s^Xa*KosgkrrkE*9$lth!7j3KhDXwVy z0gDA-qoO(>UB_Hx54-tqWH;A)r|%>$y~pcj1H9z5<)$}-}mvz>|)0w0%7k+jm z#5xV}`%Bler~*4_o`zUiE6%P5v#Fp!AKc$^z?*}zFRK(#KC$D|fgP(o?}Y&K64{@9 zx-}tbUw8ylnjd;G1eL&VjoNM#Npp9pJtTi`8z)8}DoOq(t6|z@4+xbzki&nn>9@lD zm1>|uNUDJyyZ!WY#<{QCg_s6TgjJo^Wqv|gfV~f35ZxaE`cQ#Vna!cwx*cO=Ex+nxoQ~FQJOfQhpDg#ZCxo`bC)k6)-F5YW zs7C;e-0%cJuE7bQaUE5x^~1R9!<7%80nT?io!KHnPh{PS%W?OA?Vmk- zH6eVnpea6rED*+j&#y1Wb(X^Vnd*bpkKOnjg@NJ}(wTpG8Zn z-HtnUrG*o&W$vSn=V_kmr#BLxCblX(ZhX+pqN_BzPRU{!UuTA-{9{e}e ziQiHPRl(e2VQWn30r8rma~|IZ!yw;XyLHWF9w=_mtAK}gSeCSc0sOXzn^Sq?j>$8F z5rB^6^5OgW_UvjlMn*PZTHB?oj4IwC7FSWxbmh{<+p=!U4BTmK+x;!eYG%Ihxd!mE z&GO7I3B4G2X@r5Ej;^y>d=)i9h(2)4-X{6o>9*QP1MH&kFMosaehtQH!96CJx~mf`c!L)|IUr1X$GM3oK(J95o$s|3*1f(r(L&`&%8X*CihO(tfx@c zi2Hfnhgx2(<|Md@A!HBBHf(+=uLW*D^vgO2pSRv-UFe-l=-mcCHAau$4%9Kb3kM^) zU$BUj9F@u?xnCW+_x`*8==|!_-m2Vb?fTw2{q7IWHdd-msrz*$s`rH=6VPDPm=U7*o%tl9ED=jwc9a@EaV@pA#l z8m_6C^A#Q{F1|;frxBHIn4hHH@VNMLAHKUgLW1)O{0$mX&^tSLNStl^h${b)*h(=y zY4;Y!Hr^#?4T2wm?klCLq;7xq=7X(uz@6Kxd?qERZ+(mMJC6)>A&Oa4TN`bvtF;5kG#f$cGo1o37+|bUA4*ecg#l%+1R-MMoS+&!LgT^?&c7zr_hk2HxTrJ^9Mkm^(J$AS!)d+Ml{K8NHnK;Q(+YqP6g*dBdd z$>qBD+2CbrH%#Nw$LDpAUaNtdfSYg52bSa$Pqo&eXAND-yryDLNwdIhivG>Q&W{ee zfbZ6PPEK2|^0x-q(&FU57P5E`G1p z4$lmDw2MdmFbglUv&@;vCPUDp!Ps-on_F;wdO43yD?tj(vsruz=f z4QHj31Y@4Vli_xJaQKm6f#JIA@M>pY*2aetk3k-FzL8RB&*QaTsZR4m!`X0tbbCeTUK1W&Y;+BGr?Nx3Pu#8%@`{_P~X&@RmayEA{9OTt8Qdl?CRdMT{2;+~@dM z`tF3F`|o|ms6#(}$p!Pehlj^BM=VVsV+?hL2y+Rd40~&pJLzoav%%G!R z;LoT|Nyb>~bPsc4CuAn<{p~tF&PGzxQr?dGv6#6)NE<=d26=hnreNZ*>%~W$$&qNP z{usd#^usZPs-+!fchqu#7dQ&+Pcv(IsuoXIa0d^p8)&;d-hw#6=hdEXy0AJ98W``@1Li3ML)TA zacWxrr>1Ncg?9Vs57oazN3k>92Y$lAWp3sq6jA@`nuiC(r##5?Yy&RWsZ7&S-Qu|sD0Lf*=)cY9wDId_up$`hOX207IA+E$wut5&UPL9C zj`v|AA$-cbC54aj_R}E+%t^7&Ug!AofMF2w>%})`(w(K`^x(%Tb%&;gg?)YHB@w!Pc*+=nZ5bk`+H<$>TIJfLc&xgeO(L#!sTU4-6BG&0d| zVwq@iyd0U^M?i3Z;|f#!%5hEv`*q;{6#jYQ-y*)fBgmX1i*ukNT4=AcCA~!cRq