| // Copyright 2026 The PDFium Authors | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| #include "core/fxge/dib/cfx_scanlinecompositor_hwy.h" | |
| #include <stddef.h> | |
| #include <stdint.h> | |
| #include "core/fxcrt/check_op.h" | |
| #include "core/fxcrt/compiler_specific.h" | |
| #include "core/fxcrt/span.h" | |
| #include "core/fxcrt/zip.h" | |
| #undef HWY_TARGET_INCLUDE | |
| #define HWY_TARGET_INCLUDE "core/fxge/dib/cfx_scanlinecompositor_hwy.cpp" | |
| #include "third_party/highway/src/hwy/foreach_target.h" // IWYU pragma: keep | |
| #include "third_party/highway/src/hwy/highway.h" | |
| HWY_BEFORE_NAMESPACE(); | |
| namespace fxge_composite_simd { | |
| namespace HWY_NAMESPACE { | |
| namespace { | |
| namespace hn = hwy::HWY_NAMESPACE; | |
| // exact floor(x/255), x in [0, 65025] | |
| template <class V> | |
| HWY_INLINE V Div255(V x) { | |
| const hn::DFromV<V> d; | |
| return hn::ShiftRight<8>( | |
| hn::Add(hn::Add(x, hn::Set(d, 1)), hn::ShiftRight<8>(x))); | |
| } | |
| // (back*(255-a) + src*a)/255; operands <= 255 -> sum <= 65025, Div255 exact | |
| template <class V> | |
| HWY_INLINE V AlphaMergeLanes(V back, V src, V alpha, V v255) { | |
| return Div255( | |
| hn::Add(hn::Mul(back, hn::Sub(v255, alpha)), hn::Mul(src, alpha))); | |
| } | |
| // scalar tail; bit-exact with CompositeRowBgra2Bgra() normal/no-clip | |
| inline void CompositePixelBgra2Bgra(const FX_BGRA_STRUCT<uint8_t>& src, | |
| FX_BGRA_STRUCT<uint8_t>& dest) { | |
| if (dest.alpha == 0) { | |
| dest = src; | |
| return; | |
| } | |
| if (src.alpha == 0) { | |
| return; | |
| } | |
| if (src.alpha == 255) { | |
| dest = src; | |
| return; | |
| } | |
| const uint32_t dest_alpha = | |
| dest.alpha + src.alpha - (dest.alpha * src.alpha / 255); | |
| const uint32_t ratio = src.alpha * 255 / dest_alpha; | |
| dest.blue = AlphaMerge(dest.blue, src.blue, ratio); | |
| dest.green = AlphaMerge(dest.green, src.green, ratio); | |
| dest.red = AlphaMerge(dest.red, src.red, ratio); | |
| dest.alpha = dest_alpha; | |
| } | |
| // Porter-Duff over, packed BGRA, normal, no clip. Per pixel: | |
| // dest a == 0 -> copy src src a == 255 -> copy src | |
| // src a == 0 -> keep dest else AlphaMerge/AlphaUnion over | |
| // Byte loads bitcast to u32 lanes -> no 4-byte scanline alignment needed. | |
| void RunBgra2Bgra(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src, | |
| pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest) { | |
| // Make sure the compiler can see equal span sizes after the dispatch. | |
| dest = dest.first(src.size()); | |
| #if HWY_TARGET != HWY_SCALAR | |
| const hn::ScalableTag<uint32_t> d; | |
| const hn::ScalableTag<uint8_t> d8; | |
| const size_t n = hn::Lanes(d); // pixels/vector | |
| const auto m255 = hn::Set(d, 0xFF); // byte mask and the value 255 | |
| const auto one = hn::Set(d, 1); | |
| const auto zero = hn::Zero(d); | |
| while (src.size() >= n) { | |
| const auto s = hn::BitCast( | |
| d, hn::LoadU(d8, reinterpret_cast<const uint8_t*>(src.data()))); | |
| const auto back = hn::BitCast( | |
| d, hn::LoadU(d8, reinterpret_cast<const uint8_t*>(dest.data()))); | |
| const auto sa = hn::ShiftRight<24>(s); | |
| const auto da = hn::ShiftRight<24>(back); | |
| const auto copy_src = hn::Or(hn::Eq(da, zero), hn::Eq(sa, m255)); | |
| const auto keep_dest = hn::Eq(sa, zero); | |
| // all copy/keep -> skip divide; copy_src ? src : dest | |
| if (hn::AllTrue(d, hn::Or(copy_src, keep_dest))) { | |
| const auto result = hn::IfThenElse(copy_src, s, back); | |
| hn::StoreU(hn::BitCast(d8, result), d8, | |
| reinterpret_cast<uint8_t*>(dest.data())); | |
| src = src.subspan(n); | |
| dest = dest.subspan(n); | |
| continue; | |
| } | |
| // divisor forced >= 1: copy/keep lanes may yield da2 == 0 (unused) | |
| const auto da2 = hn::Sub(hn::Add(da, sa), Div255(hn::Mul(da, sa))); | |
| const auto ratio = hn::Div(hn::Mul(sa, m255), hn::Max(da2, one)); | |
| const auto db = hn::And(back, m255); | |
| const auto dg = hn::And(hn::ShiftRight<8>(back), m255); | |
| const auto dr = hn::And(hn::ShiftRight<16>(back), m255); | |
| const auto sb = hn::And(s, m255); | |
| const auto sg = hn::And(hn::ShiftRight<8>(s), m255); | |
| const auto sr = hn::And(hn::ShiftRight<16>(s), m255); | |
| const auto nb = AlphaMergeLanes(db, sb, ratio, m255); | |
| const auto ng = AlphaMergeLanes(dg, sg, ratio, m255); | |
| const auto nr = AlphaMergeLanes(dr, sr, ratio, m255); | |
| const auto gen = | |
| hn::Or(hn::Or(nb, hn::ShiftLeft<8>(ng)), | |
| hn::Or(hn::ShiftLeft<16>(nr), hn::ShiftLeft<24>(da2))); | |
| auto result = hn::IfThenElse(keep_dest, back, gen); | |
| result = hn::IfThenElse(copy_src, s, result); | |
| hn::StoreU(hn::BitCast(d8, result), d8, | |
| reinterpret_cast<uint8_t*>(dest.data())); | |
| src = src.subspan(n); | |
| dest = dest.subspan(n); | |
| } | |
| #endif // HWY_TARGET != HWY_SCALAR | |
| for (auto [src_elem, dest_elem] : fxcrt::Zip(src, dest)) { | |
| CompositePixelBgra2Bgra(src_elem, dest_elem); | |
| } | |
| } | |
| // Opaque BGRA(4) -> BGRA, alpha forced 0xFF (src alpha ignored). Same byte | |
| // order required: src and dest both packed BGRA. | |
| void RunBgra2BgrxForceOpaque(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src, | |
| pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest) { | |
| // Make sure the compiler can see equal span sizes after the dispatch. | |
| dest = dest.first(src.size()); | |
| #if HWY_TARGET != HWY_SCALAR | |
| const hn::ScalableTag<uint32_t> d; | |
| const hn::ScalableTag<uint8_t> d8; | |
| const size_t n = hn::Lanes(d); | |
| const auto alpha_ff = hn::Set(d, 0xFF000000u); | |
| while (src.size() >= n) { | |
| const auto v = hn::BitCast( | |
| d, hn::LoadU(d8, reinterpret_cast<const uint8_t*>(src.data()))); | |
| hn::StoreU(hn::BitCast(d8, hn::Or(v, alpha_ff)), d8, | |
| reinterpret_cast<uint8_t*>(dest.data())); | |
| src = src.subspan(n); | |
| dest = dest.subspan(n); | |
| } | |
| #endif // HWY_TARGET != HWY_SCALAR | |
| for (auto [src_elem, dest_elem] : fxcrt::Zip(src, dest)) { | |
| dest_elem.blue = src_elem.blue; | |
| dest_elem.green = src_elem.green; | |
| dest_elem.red = src_elem.red; | |
| dest_elem.alpha = 0xFF; | |
| } | |
| } | |
| #if HWY_TARGET != HWY_SCALAR | |
| // Exact floor(x / 255) for uint16_t lanes and x in [0, 65025]. | |
| // x + 1 <= 65026, so the addition does not overflow. | |
| template <class V> | |
| HWY_INLINE V Div255U16(V x) { | |
| const hn::DFromV<V> d; | |
| return hn::MulHigh(hn::Add(x, hn::Set(d, 1)), hn::Set(d, 257)); | |
| } | |
| #endif | |
| // Keep the clip decision outside the loop. Both products are at most | |
| // 255 * 255, so the exact, truncating division matches AlphaUnion(). | |
| template <bool kHasClip> | |
| void Bgra2Mask(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src, | |
| pdfium::span<const uint8_t> clip, | |
| pdfium::span<uint8_t> dest) { | |
| // Make sure the compiler can see equal span sizes after the dispatch. | |
| dest = dest.first(src.size()); | |
| if constexpr (kHasClip) { | |
| clip = clip.first(src.size()); | |
| } | |
| #if HWY_TARGET != HWY_SCALAR | |
| // Use 16-bit products to process twice as many pixels per iteration as | |
| // 32-bit lanes. This also avoids emulated multiplication on SSE2 and SSSE3. | |
| const hn::ScalableTag<uint32_t> d32; | |
| const hn::ScalableTag<uint8_t> d8; | |
| const hn::ScalableTag<uint16_t> d; | |
| const hn::Half<decltype(d)> dh; | |
| const hn::Rebind<uint8_t, decltype(d)> dn; | |
| const size_t n = hn::Lanes(d); | |
| while (src.size() >= n) { | |
| // Each load covers n/2 BGRA pixels, together supplying n alpha values. | |
| const auto lo = hn::ShiftRight<24>(hn::BitCast( | |
| d32, hn::LoadU(d8, reinterpret_cast<const uint8_t*>(src.data())))); | |
| const auto hi = hn::ShiftRight<24>( | |
| hn::BitCast(d32, hn::LoadU(d8, reinterpret_cast<const uint8_t*>( | |
| src.subspan(n / 2).data())))); | |
| auto alpha = hn::Combine(d, hn::DemoteTo(dh, hi), hn::DemoteTo(dh, lo)); | |
| if constexpr (kHasClip) { | |
| alpha = Div255U16( | |
| hn::Mul(alpha, hn::PromoteTo(d, hn::LoadU(dn, clip.data())))); | |
| clip = clip.subspan(n); | |
| } | |
| const auto back = hn::PromoteTo(d, hn::LoadU(dn, dest.data())); | |
| const auto result = | |
| hn::Sub(hn::Add(back, alpha), Div255U16(hn::Mul(back, alpha))); | |
| hn::StoreU(hn::DemoteTo(dn, result), dn, dest.data()); | |
| src = src.subspan(n); | |
| dest = dest.subspan(n); | |
| } | |
| #endif // HWY_TARGET != HWY_SCALAR | |
| while (!src.empty()) { | |
| uint32_t alpha = src[0].alpha; | |
| if constexpr (kHasClip) { | |
| alpha = alpha * clip[0] / 255; | |
| clip = clip.subspan<1u>(); | |
| } | |
| dest[0] = static_cast<uint8_t>(dest[0] + alpha - dest[0] * alpha / 255); | |
| src = src.subspan<1u>(); | |
| dest = dest.subspan<1u>(); | |
| } | |
| } | |
| void RunBgra2Mask(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src, | |
| pdfium::span<const uint8_t> clip, | |
| pdfium::span<uint8_t> dest) { | |
| if (clip.empty()) { | |
| Bgra2Mask<false>(src, clip, dest); | |
| return; | |
| } | |
| Bgra2Mask<true>(src, clip, dest); | |
| } | |
| } // namespace | |
| } // namespace HWY_NAMESPACE | |
| } // namespace fxge_composite_simd | |
| HWY_AFTER_NAMESPACE(); | |
| #if HWY_ONCE | |
| namespace fxge_composite_simd { | |
| HWY_EXPORT(RunBgra2Bgra); | |
| HWY_EXPORT(RunBgra2BgrxForceOpaque); | |
| HWY_EXPORT(RunBgra2Mask); | |
| // Unqualified name, in-namespace: a qualified name breaks single-target | |
| // HWY_STATIC_DISPATCH. | |
| void DispatchBgra2Bgra(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src, | |
| pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest) { | |
| // SAFETY: dispatch indexes Highway's per-target table; no span macro form. | |
| UNSAFE_BUFFERS(HWY_DYNAMIC_DISPATCH(RunBgra2Bgra)(src, dest)); | |
| } | |
| void DispatchBgra2BgrxForceOpaque( | |
| pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src, | |
| pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest) { | |
| // SAFETY: see DispatchBgra2Bgra(). | |
| UNSAFE_BUFFERS(HWY_DYNAMIC_DISPATCH(RunBgra2BgrxForceOpaque)(src, dest)); | |
| } | |
| void DispatchBgra2Mask(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src, | |
| pdfium::span<const uint8_t> clip, | |
| pdfium::span<uint8_t> dest) { | |
| // SAFETY: see DispatchBgra2Bgra(). | |
| UNSAFE_BUFFERS(HWY_DYNAMIC_DISPATCH(RunBgra2Mask)(src, clip, dest)); | |
| } | |
| } // namespace fxge_composite_simd | |
| void CompositeRowBgra2BgraSimd( | |
| pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src_span, | |
| pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest_span) { | |
| // `src_span.size()` pixels, like `fxcrt::Zip(src_span, dest_span)`. | |
| CHECK_LE(src_span.size(), dest_span.size()); | |
| fxge_composite_simd::DispatchBgra2Bgra(src_span, dest_span); | |
| } | |
| void CompositeRowBgra2BgrxForceOpaqueSimd( | |
| pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src_span, | |
| pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest_span) { | |
| // `src_span.size()` pixels, like `fxcrt::Zip(src_span, dest_span)`. | |
| CHECK_LE(src_span.size(), dest_span.size()); | |
| fxge_composite_simd::DispatchBgra2BgrxForceOpaque(src_span, dest_span); | |
| } | |
| void CompositeRowBgra2MaskSimd( | |
| pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src_span, | |
| pdfium::span<const uint8_t> clip_span, | |
| pdfium::span<uint8_t> dest_span) { | |
| CHECK_LE(src_span.size(), dest_span.size()); | |
| CHECK(clip_span.empty() || src_span.size() <= clip_span.size()); | |
| fxge_composite_simd::DispatchBgra2Mask(src_span, clip_span, dest_span); | |
| } | |
| #endif // HWY_ONCE |