blob: dea4e047e85a786a81d1608c67bae706e9c1bcb5 [file]
// Copyright 2026 The PDFium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "core/fxge/dib/cfx_scanlinecompositor_hwy.h"
#include <stddef.h>
#include <stdint.h>
#include "core/fxcrt/check_op.h"
#include "core/fxcrt/compiler_specific.h"
#include "core/fxcrt/span.h"
#include "core/fxcrt/zip.h"
#undef HWY_TARGET_INCLUDE
#define HWY_TARGET_INCLUDE "core/fxge/dib/cfx_scanlinecompositor_hwy.cpp"
#include "third_party/highway/src/hwy/foreach_target.h" // IWYU pragma: keep
#include "third_party/highway/src/hwy/highway.h"
HWY_BEFORE_NAMESPACE();
namespace fxge_composite_simd {
namespace HWY_NAMESPACE {
namespace {
namespace hn = hwy::HWY_NAMESPACE;
// exact floor(x/255), x in [0, 65025]
template <class V>
HWY_INLINE V Div255(V x) {
const hn::DFromV<V> d;
return hn::ShiftRight<8>(
hn::Add(hn::Add(x, hn::Set(d, 1)), hn::ShiftRight<8>(x)));
}
// (back*(255-a) + src*a)/255; operands <= 255 -> sum <= 65025, Div255 exact
template <class V>
HWY_INLINE V AlphaMergeLanes(V back, V src, V alpha, V v255) {
return Div255(
hn::Add(hn::Mul(back, hn::Sub(v255, alpha)), hn::Mul(src, alpha)));
}
// scalar tail; bit-exact with CompositeRowBgra2Bgra() normal/no-clip
inline void CompositePixelBgra2Bgra(const FX_BGRA_STRUCT<uint8_t>& src,
FX_BGRA_STRUCT<uint8_t>& dest) {
if (dest.alpha == 0) {
dest = src;
return;
}
if (src.alpha == 0) {
return;
}
if (src.alpha == 255) {
dest = src;
return;
}
const uint32_t dest_alpha =
dest.alpha + src.alpha - (dest.alpha * src.alpha / 255);
const uint32_t ratio = src.alpha * 255 / dest_alpha;
dest.blue = AlphaMerge(dest.blue, src.blue, ratio);
dest.green = AlphaMerge(dest.green, src.green, ratio);
dest.red = AlphaMerge(dest.red, src.red, ratio);
dest.alpha = dest_alpha;
}
// Porter-Duff over, packed BGRA, normal, no clip. Per pixel:
// dest a == 0 -> copy src src a == 255 -> copy src
// src a == 0 -> keep dest else AlphaMerge/AlphaUnion over
// Byte loads bitcast to u32 lanes -> no 4-byte scanline alignment needed.
void RunBgra2Bgra(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src,
pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest) {
// Make sure the compiler can see equal span sizes after the dispatch.
dest = dest.first(src.size());
#if HWY_TARGET != HWY_SCALAR
const hn::ScalableTag<uint32_t> d;
const hn::ScalableTag<uint8_t> d8;
const size_t n = hn::Lanes(d); // pixels/vector
const auto m255 = hn::Set(d, 0xFF); // byte mask and the value 255
const auto one = hn::Set(d, 1);
const auto zero = hn::Zero(d);
while (src.size() >= n) {
const auto s = hn::BitCast(
d, hn::LoadU(d8, reinterpret_cast<const uint8_t*>(src.data())));
const auto back = hn::BitCast(
d, hn::LoadU(d8, reinterpret_cast<const uint8_t*>(dest.data())));
const auto sa = hn::ShiftRight<24>(s);
const auto da = hn::ShiftRight<24>(back);
const auto copy_src = hn::Or(hn::Eq(da, zero), hn::Eq(sa, m255));
const auto keep_dest = hn::Eq(sa, zero);
// all copy/keep -> skip divide; copy_src ? src : dest
if (hn::AllTrue(d, hn::Or(copy_src, keep_dest))) {
const auto result = hn::IfThenElse(copy_src, s, back);
hn::StoreU(hn::BitCast(d8, result), d8,
reinterpret_cast<uint8_t*>(dest.data()));
src = src.subspan(n);
dest = dest.subspan(n);
continue;
}
// divisor forced >= 1: copy/keep lanes may yield da2 == 0 (unused)
const auto da2 = hn::Sub(hn::Add(da, sa), Div255(hn::Mul(da, sa)));
const auto ratio = hn::Div(hn::Mul(sa, m255), hn::Max(da2, one));
const auto db = hn::And(back, m255);
const auto dg = hn::And(hn::ShiftRight<8>(back), m255);
const auto dr = hn::And(hn::ShiftRight<16>(back), m255);
const auto sb = hn::And(s, m255);
const auto sg = hn::And(hn::ShiftRight<8>(s), m255);
const auto sr = hn::And(hn::ShiftRight<16>(s), m255);
const auto nb = AlphaMergeLanes(db, sb, ratio, m255);
const auto ng = AlphaMergeLanes(dg, sg, ratio, m255);
const auto nr = AlphaMergeLanes(dr, sr, ratio, m255);
const auto gen =
hn::Or(hn::Or(nb, hn::ShiftLeft<8>(ng)),
hn::Or(hn::ShiftLeft<16>(nr), hn::ShiftLeft<24>(da2)));
auto result = hn::IfThenElse(keep_dest, back, gen);
result = hn::IfThenElse(copy_src, s, result);
hn::StoreU(hn::BitCast(d8, result), d8,
reinterpret_cast<uint8_t*>(dest.data()));
src = src.subspan(n);
dest = dest.subspan(n);
}
#endif // HWY_TARGET != HWY_SCALAR
for (auto [src_elem, dest_elem] : fxcrt::Zip(src, dest)) {
CompositePixelBgra2Bgra(src_elem, dest_elem);
}
}
// Opaque BGRA(4) -> BGRA, alpha forced 0xFF (src alpha ignored). Same byte
// order required: src and dest both packed BGRA.
void RunBgra2BgrxForceOpaque(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src,
pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest) {
// Make sure the compiler can see equal span sizes after the dispatch.
dest = dest.first(src.size());
#if HWY_TARGET != HWY_SCALAR
const hn::ScalableTag<uint32_t> d;
const hn::ScalableTag<uint8_t> d8;
const size_t n = hn::Lanes(d);
const auto alpha_ff = hn::Set(d, 0xFF000000u);
while (src.size() >= n) {
const auto v = hn::BitCast(
d, hn::LoadU(d8, reinterpret_cast<const uint8_t*>(src.data())));
hn::StoreU(hn::BitCast(d8, hn::Or(v, alpha_ff)), d8,
reinterpret_cast<uint8_t*>(dest.data()));
src = src.subspan(n);
dest = dest.subspan(n);
}
#endif // HWY_TARGET != HWY_SCALAR
for (auto [src_elem, dest_elem] : fxcrt::Zip(src, dest)) {
dest_elem.blue = src_elem.blue;
dest_elem.green = src_elem.green;
dest_elem.red = src_elem.red;
dest_elem.alpha = 0xFF;
}
}
#if HWY_TARGET != HWY_SCALAR
// Exact floor(x / 255) for uint16_t lanes and x in [0, 65025].
// x + 1 <= 65026, so the addition does not overflow.
template <class V>
HWY_INLINE V Div255U16(V x) {
const hn::DFromV<V> d;
return hn::MulHigh(hn::Add(x, hn::Set(d, 1)), hn::Set(d, 257));
}
#endif
// Keep the clip decision outside the loop. Both products are at most
// 255 * 255, so the exact, truncating division matches AlphaUnion().
template <bool kHasClip>
void Bgra2Mask(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src,
pdfium::span<const uint8_t> clip,
pdfium::span<uint8_t> dest) {
// Make sure the compiler can see equal span sizes after the dispatch.
dest = dest.first(src.size());
if constexpr (kHasClip) {
clip = clip.first(src.size());
}
#if HWY_TARGET != HWY_SCALAR
// Use 16-bit products to process twice as many pixels per iteration as
// 32-bit lanes. This also avoids emulated multiplication on SSE2 and SSSE3.
const hn::ScalableTag<uint32_t> d32;
const hn::ScalableTag<uint8_t> d8;
const hn::ScalableTag<uint16_t> d;
const hn::Half<decltype(d)> dh;
const hn::Rebind<uint8_t, decltype(d)> dn;
const size_t n = hn::Lanes(d);
while (src.size() >= n) {
// Each load covers n/2 BGRA pixels, together supplying n alpha values.
const auto lo = hn::ShiftRight<24>(hn::BitCast(
d32, hn::LoadU(d8, reinterpret_cast<const uint8_t*>(src.data()))));
const auto hi = hn::ShiftRight<24>(
hn::BitCast(d32, hn::LoadU(d8, reinterpret_cast<const uint8_t*>(
src.subspan(n / 2).data()))));
auto alpha = hn::Combine(d, hn::DemoteTo(dh, hi), hn::DemoteTo(dh, lo));
if constexpr (kHasClip) {
alpha = Div255U16(
hn::Mul(alpha, hn::PromoteTo(d, hn::LoadU(dn, clip.data()))));
clip = clip.subspan(n);
}
const auto back = hn::PromoteTo(d, hn::LoadU(dn, dest.data()));
const auto result =
hn::Sub(hn::Add(back, alpha), Div255U16(hn::Mul(back, alpha)));
hn::StoreU(hn::DemoteTo(dn, result), dn, dest.data());
src = src.subspan(n);
dest = dest.subspan(n);
}
#endif // HWY_TARGET != HWY_SCALAR
while (!src.empty()) {
uint32_t alpha = src[0].alpha;
if constexpr (kHasClip) {
alpha = alpha * clip[0] / 255;
clip = clip.subspan<1u>();
}
dest[0] = static_cast<uint8_t>(dest[0] + alpha - dest[0] * alpha / 255);
src = src.subspan<1u>();
dest = dest.subspan<1u>();
}
}
void RunBgra2Mask(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src,
pdfium::span<const uint8_t> clip,
pdfium::span<uint8_t> dest) {
if (clip.empty()) {
Bgra2Mask<false>(src, clip, dest);
return;
}
Bgra2Mask<true>(src, clip, dest);
}
} // namespace
} // namespace HWY_NAMESPACE
} // namespace fxge_composite_simd
HWY_AFTER_NAMESPACE();
#if HWY_ONCE
namespace fxge_composite_simd {
HWY_EXPORT(RunBgra2Bgra);
HWY_EXPORT(RunBgra2BgrxForceOpaque);
HWY_EXPORT(RunBgra2Mask);
// Unqualified name, in-namespace: a qualified name breaks single-target
// HWY_STATIC_DISPATCH.
void DispatchBgra2Bgra(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src,
pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest) {
// SAFETY: dispatch indexes Highway's per-target table; no span macro form.
UNSAFE_BUFFERS(HWY_DYNAMIC_DISPATCH(RunBgra2Bgra)(src, dest));
}
void DispatchBgra2BgrxForceOpaque(
pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src,
pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest) {
// SAFETY: see DispatchBgra2Bgra().
UNSAFE_BUFFERS(HWY_DYNAMIC_DISPATCH(RunBgra2BgrxForceOpaque)(src, dest));
}
void DispatchBgra2Mask(pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src,
pdfium::span<const uint8_t> clip,
pdfium::span<uint8_t> dest) {
// SAFETY: see DispatchBgra2Bgra().
UNSAFE_BUFFERS(HWY_DYNAMIC_DISPATCH(RunBgra2Mask)(src, clip, dest));
}
} // namespace fxge_composite_simd
void CompositeRowBgra2BgraSimd(
pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src_span,
pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest_span) {
// `src_span.size()` pixels, like `fxcrt::Zip(src_span, dest_span)`.
CHECK_LE(src_span.size(), dest_span.size());
fxge_composite_simd::DispatchBgra2Bgra(src_span, dest_span);
}
void CompositeRowBgra2BgrxForceOpaqueSimd(
pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src_span,
pdfium::span<FX_BGRA_STRUCT<uint8_t>> dest_span) {
// `src_span.size()` pixels, like `fxcrt::Zip(src_span, dest_span)`.
CHECK_LE(src_span.size(), dest_span.size());
fxge_composite_simd::DispatchBgra2BgrxForceOpaque(src_span, dest_span);
}
void CompositeRowBgra2MaskSimd(
pdfium::span<const FX_BGRA_STRUCT<uint8_t>> src_span,
pdfium::span<const uint8_t> clip_span,
pdfium::span<uint8_t> dest_span) {
CHECK_LE(src_span.size(), dest_span.size());
CHECK(clip_span.empty() || src_span.size() <= clip_span.size());
fxge_composite_simd::DispatchBgra2Mask(src_span, clip_span, dest_span);
}
#endif // HWY_ONCE