blob: 82075b686be2c87e12f95251960b8a113f98cfeb [file] [edit]
// Copyright 2017 The PDFium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Original code copyright 2014 Foxit Software Inc. http://www.foxitsoftware.com
#include "core/fxge/dib/cstretchengine.h"
#include <math.h>
#include <algorithm>
#include <array>
#include <optional>
#include <type_traits>
#include <utility>
#include "core/fxcrt/check.h"
#include "core/fxcrt/check_op.h"
#include "core/fxcrt/fx_2d_size.h"
#include "core/fxcrt/fx_ceil_div.h"
#include "core/fxcrt/fx_safe_types.h"
#include "core/fxcrt/fx_system.h"
#include "core/fxcrt/numerics/safe_conversions.h"
#include "core/fxcrt/pauseindicator_iface.h"
#include "core/fxcrt/span_util.h"
#include "core/fxcrt/zip.h"
#include "core/fxge/calculate_pitch.h"
#include "core/fxge/dib/cfx_dibbase.h"
#include "core/fxge/dib/cfx_dibitmap.h"
#include "core/fxge/dib/cstretchengine_hwy.h"
#include "core/fxge/dib/fx_dib.h"
#include "core/fxge/dib/scanlinecomposer_iface.h"
static_assert(
std::is_trivially_destructible<CStretchEngine::PixelWeight>::value,
"PixelWeight storage may be re-used without invoking its destructor");
namespace {
size_t TotalBytesForWeightCount(size_t weight_count) {
// Always room for one weight even for empty ranges due to declaration
// of weights_[1] in the header. Don't shrink below this since
// CalculateWeights() relies on this later.
const size_t extra_weights = weight_count > 0 ? weight_count - 1 : 0;
FX_SAFE_SIZE_T total_bytes = extra_weights;
total_bytes *= sizeof(CStretchEngine::PixelWeight::weights_[0]);
total_bytes += sizeof(CStretchEngine::PixelWeight);
return total_bytes.ValueOrDie();
}
} // namespace
// static
bool CStretchEngine::UseInterpolateBilinear(
const FXDIB_ResampleOptions& options,
int dest_width,
int dest_height,
int src_width,
int src_height) {
return !options.bInterpolateBilinear && !options.bNoSmoothing &&
abs(dest_width) != 0 &&
abs(dest_height) / 8 <
static_cast<long long>(src_width) * src_height / abs(dest_width);
}
CStretchEngine::WeightTable::WeightTable() = default;
CStretchEngine::WeightTable::~WeightTable() = default;
bool CStretchEngine::WeightTable::CalculateWeights(
int dest_len,
int dest_min,
int dest_max,
int src_len,
int src_min,
int src_max,
const FXDIB_ResampleOptions& options) {
// 512MB should be large enough for this while preventing OOM.
static constexpr size_t kMaxTableBytesAllowed = 512 * 1024 * 1024;
// Help the compiler realize that these can't change during a loop iteration:
const bool bilinear = options.bInterpolateBilinear;
dest_min_ = 0;
item_size_bytes_ = 0;
weight_tables_size_bytes_ = 0;
weight_tables_.clear();
if (dest_len == 0) {
return true;
}
if (dest_min > dest_max) {
return false;
}
dest_min_ = dest_min;
const double scale = static_cast<double>(src_len) / dest_len;
const double base = dest_len < 0 ? src_len : 0;
const size_t weight_count = static_cast<size_t>(ceil(fabs(scale))) + 1;
item_size_bytes_ = TotalBytesForWeightCount(weight_count);
const size_t dest_range = static_cast<size_t>(dest_max - dest_min);
const size_t kMaxTableItemsAllowed = kMaxTableBytesAllowed / item_size_bytes_;
if (dest_range > kMaxTableItemsAllowed) {
return false;
}
weight_tables_size_bytes_ = dest_range * item_size_bytes_;
weight_tables_.resize(weight_tables_size_bytes_);
if (options.bNoSmoothing || fabs(scale) < 1.0f) {
for (int dest_pixel = dest_min; dest_pixel < dest_max; ++dest_pixel) {
PixelWeight& pixel_weights = *GetPixelWeight(dest_pixel);
double src_pos = dest_pixel * scale + scale / 2 + base;
if (bilinear) {
int src_start = static_cast<int>(floor(src_pos - 0.5));
int src_end = static_cast<int>(floor(src_pos + 0.5));
src_start = std::max(src_start, src_min);
src_end = std::min(src_end, src_max - 1);
pixel_weights.SetStartEnd(src_start, src_end, weight_count);
if (pixel_weights.src_start_ >= pixel_weights.src_end_) {
// Always room for one weight per size calculation.
pixel_weights.weights_[0] = kFixedPointOne;
} else {
const uint32_t second_weight =
FixedFromDouble(src_pos - pixel_weights.src_start_ - 0.5f);
pixel_weights.SetWeightForPosition(pixel_weights.src_start_,
kFixedPointOne - second_weight);
pixel_weights.SetWeightForPosition(pixel_weights.src_start_ + 1,
second_weight);
}
} else {
int pixel_pos = static_cast<int>(floor(src_pos));
int src_start = std::max(pixel_pos, src_min);
int src_end = std::min(pixel_pos, src_max - 1);
pixel_weights.SetStartEnd(src_start, src_end, weight_count);
pixel_weights.weights_[0] = kFixedPointOne;
}
}
return true;
}
for (int dest_pixel = dest_min; dest_pixel < dest_max; ++dest_pixel) {
PixelWeight& pixel_weights = *GetPixelWeight(dest_pixel);
double src_start = dest_pixel * scale + base;
double src_end = src_start + scale;
int start_i = floor(std::min(src_start, src_end));
int end_i = floor(std::max(src_start, src_end));
start_i = std::max(start_i, src_min);
end_i = std::min(end_i, src_max - 1);
if (start_i > end_i) {
// `start_i` is already at least `src_min`, so the lower bound only
// matters when the source range is empty: `src_max` is then 0 and
// clamping from above alone would store a negative position.
start_i = std::clamp(start_i, 0, std::max(src_max - 1, 0));
pixel_weights.SetStartEnd(start_i, start_i, weight_count);
continue;
}
pixel_weights.SetStartEnd(start_i, end_i, weight_count);
uint32_t remaining = kFixedPointOne;
double rounding_error = 0.0;
for (int j = start_i; j < end_i; ++j) {
double dest_start = (j - base) / scale;
double dest_end = (j + 1 - base) / scale;
if (dest_start > dest_end) {
std::swap(dest_start, dest_end);
}
double area_start = std::max(dest_start, static_cast<double>(dest_pixel));
double area_end = std::min(dest_end, static_cast<double>(dest_pixel + 1));
double weight = std::max(0.0, area_end - area_start);
uint32_t fixed_weight = FixedFromDouble(weight + rounding_error);
pixel_weights.SetWeightForPosition(j, fixed_weight);
remaining -= fixed_weight;
rounding_error =
weight - static_cast<double>(fixed_weight) / kFixedPointOne;
}
// Note: underflow is defined behaviour for unsigned types and will
// result in an out-of-range value.
if (remaining && remaining <= kFixedPointOne) {
pixel_weights.SetWeightForPosition(end_i, remaining);
} else {
pixel_weights.RemoveLastWeightAndAdjust(remaining);
}
}
return true;
}
const CStretchEngine::PixelWeight* CStretchEngine::WeightTable::GetPixelWeight(
int pixel) const {
DCHECK_GE(pixel, dest_min_);
return reinterpret_cast<const PixelWeight*>(
&weight_tables_[(pixel - dest_min_) * item_size_bytes_]);
}
CStretchEngine::PixelWeight* CStretchEngine::WeightTable::GetPixelWeight(
int pixel) {
return const_cast<PixelWeight*>(std::as_const(*this).GetPixelWeight(pixel));
}
CStretchEngine::CStretchEngine(ScanlineComposerIface* pDestBitmap,
FXDIB_Format dest_format,
int dest_width,
int dest_height,
const FX_RECT& clip_rect,
const RetainPtr<const CFX_DIBBase>& pSrcBitmap,
const FXDIB_ResampleOptions& options)
: dest_format_(dest_format),
dest_bpp_(GetBppFromFormat(dest_format)),
src_bpp_(pSrcBitmap->GetBPP()),
has_alpha_(pSrcBitmap->IsAlphaFormat()),
source_(pSrcBitmap),
src_palette_(pSrcBitmap->GetPaletteSpan()),
src_width_(pSrcBitmap->GetWidth()),
src_height_(pSrcBitmap->GetHeight()),
dest_bitmap_(pDestBitmap),
dest_width_(dest_width),
dest_height_(dest_height),
dest_clip_(clip_rect) {
if (has_alpha_) {
// TODO(crbug.com/42271020): Consider adding support for
// `FXDIB_Format::kBgraPremul`
DCHECK_EQ(dest_format_, FXDIB_Format::kBgra);
DCHECK_EQ(dest_bpp_, GetBppFromFormat(FXDIB_Format::kBgra));
DCHECK_EQ(source_->GetFormat(), FXDIB_Format::kBgra);
DCHECK_EQ(src_bpp_, GetBppFromFormat(FXDIB_Format::kBgra));
}
std::optional<uint32_t> maybe_size =
fxge::CalculatePitch32(dest_bpp_, clip_rect.Width());
if (!maybe_size.has_value()) {
return;
}
if (!IsValidDestinationDimension(dest_width_) ||
!IsValidDestinationDimension(dest_height_)) {
return;
}
dest_scanline_.resize(maybe_size.value());
if (dest_format == FXDIB_Format::kBgrx) {
std::ranges::fill(dest_scanline_, 255);
}
inter_pitch_ = fxge::CalculatePitch32OrDie(dest_bpp_, dest_clip_.Width());
extra_mask_pitch_ = fxge::CalculatePitch32OrDie(8, dest_clip_.Width());
if (options.bNoSmoothing) {
resample_options_.bNoSmoothing = true;
} else {
if (UseInterpolateBilinear(options, dest_width_, dest_height_, src_width_,
src_height_)) {
resample_options_.bInterpolateBilinear = true;
} else {
resample_options_ = options;
}
}
double scale_x = static_cast<float>(src_width_) / dest_width_;
double scale_y = static_cast<float>(src_height_) / dest_height_;
double base_x = dest_width_ > 0 ? 0.0f : dest_width_;
double base_y = dest_height_ > 0 ? 0.0f : dest_height_;
double src_left = scale_x * (clip_rect.left + base_x);
double src_right = scale_x * (clip_rect.right + base_x);
double src_top = scale_y * (clip_rect.top + base_y);
double src_bottom = scale_y * (clip_rect.bottom + base_y);
if (src_left > src_right) {
std::swap(src_left, src_right);
}
if (src_top > src_bottom) {
std::swap(src_top, src_bottom);
}
src_clip_.left = static_cast<int>(floor(src_left));
src_clip_.right = static_cast<int>(ceil(src_right));
src_clip_.top = static_cast<int>(floor(src_top));
src_clip_.bottom = static_cast<int>(ceil(src_bottom));
FX_RECT src_rect(0, 0, src_width_, src_height_);
src_clip_.Intersect(src_rect);
switch (src_bpp_) {
case 1:
trans_method_ = dest_bpp_ == 8 ? TransformMethod::k1BppTo8Bpp
: TransformMethod::k1BppToManyBpp;
break;
case 8:
trans_method_ = dest_bpp_ == 8 ? TransformMethod::k8BppTo8Bpp
: TransformMethod::k8BppToManyBpp;
break;
default:
trans_method_ = has_alpha_ ? TransformMethod::kManyBpptoManyBppWithAlpha
: TransformMethod::kManyBpptoManyBpp;
break;
}
}
CStretchEngine::~CStretchEngine() = default;
bool CStretchEngine::Continue(PauseIndicatorIface* pPause) {
while (state_ == State::kHorizontal) {
if (ContinueStretchHorz(pPause)) {
return true;
}
state_ = State::kVertical;
StretchVert();
}
return false;
}
bool CStretchEngine::StartStretchHorz() {
if (dest_width_ == 0 || inter_pitch_ == 0 || dest_scanline_.empty()) {
return false;
}
FX_SAFE_SIZE_T safe_size = src_clip_.Height();
safe_size *= inter_pitch_;
const size_t size = safe_size.ValueOrDefault(0);
if (size == 0) {
return false;
}
inter_buf_ = FixedSizeDataVector<uint8_t>::TryZeroed(size);
if (inter_buf_.empty()) {
return false;
}
if (!weight_table_.CalculateWeights(
dest_width_, dest_clip_.left, dest_clip_.right, src_width_,
src_clip_.left, src_clip_.right, resample_options_)) {
return false;
}
if (src_width_ > 0 && (trans_method_ == TransformMethod::k1BppTo8Bpp ||
trans_method_ == TransformMethod::k1BppToManyBpp)) {
// Eight pixels per source byte, so round up to whole source bytes.
FX_SAFE_SIZE_T expanded_size = src_width_;
expanded_size = fxcrt::CeilDiv(expanded_size, 8);
expanded_size *= 8;
expanded_row_ =
FixedSizeDataVector<uint8_t>::TryUninit(expanded_size.ValueOrDie());
if (expanded_row_.empty()) {
return false;
}
}
PlanHorzGroups();
cur_row_ = src_clip_.top;
state_ = State::kHorizontal;
return true;
}
void CStretchEngine::PlanHorzGroups() {
horz_groups_.clear();
horz_runs_.clear();
if (dest_clip_.Width() <= 0) {
return;
}
if (!StretchHorzSimdAvailable()) {
// No vector wide enough for the kernel's gather exists in this build
// (see the HWY_SCALAR branch of cstretchengine_hwy.cpp): leave
// `horz_groups_` and `horz_runs_` empty, so every column takes the
// per-column path below, unchanged, rather than planning groups nothing
// can process.
return;
}
// The two palette arms have no group kernel; every other arm does. (1bpp
// sources take the k8BppTo8Bpp arm, on the row Expand1BppRow() produces.)
if (trans_method_ == TransformMethod::k1BppToManyBpp ||
trans_method_ == TransformMethod::k8BppToManyBpp) {
return;
}
HorzPlanParams params;
// The source pixel stride the gather reads: one byte for k8BppTo8Bpp and
// for an expanded 1bpp row, or the full pixel for kManyBpptoManyBpp /
// WithAlpha (3 or 4, per trans_method_'s own precondition on src_bpp_;
// WithAlpha is always 4).
const bool one_byte_pixels = trans_method_ == TransformMethod::k1BppTo8Bpp ||
trans_method_ == TransformMethod::k8BppTo8Bpp;
params.bpp = one_byte_pixels ? 1 : src_bpp_ / 8;
// How many leading bytes of each source pixel a group loads: fewer than
// `bpp` when the 4th byte of a 4-byte pixel is padding the kernel never
// reads, matching the per-column path. WithAlpha reads all 4 bytes: its
// kernel needs the alpha byte the non-alpha kernel treats as padding.
if (trans_method_ == TransformMethod::kManyBpptoManyBppWithAlpha) {
params.gathered = params.bpp;
} else if (params.bpp == 1) {
params.gathered = 1;
} else {
params.gathered = 3;
}
// The row the kernel will read. For a 1bpp source that is the expanded
// row, which StartStretchHorz() has already sized: whole source bytes of 8
// pixels each, so up to 7 bytes past src_width_ (see Expand1BppRow()).
if (trans_method_ == TransformMethod::k1BppTo8Bpp) {
params.src_row_bytes = expanded_row_.size();
} else {
params.src_row_bytes = Fx2DSizeOrDie(src_width_, params.bpp);
}
// Walk the destination columns in maximal spans of the same tap count and
// plan each span.
int span_start = dest_clip_.left;
uint32_t span_taps = HorzTapCountAt(span_start);
for (int col = span_start + 1; col <= dest_clip_.right; ++col) {
const uint32_t taps = col < dest_clip_.right ? HorzTapCountAt(col) : 0;
if (col == dest_clip_.right || taps != span_taps) {
PlanHorzSpan(span_start, col, span_taps, params);
span_start = col;
span_taps = taps;
}
}
}
// A destination column's tap count if the kernel handles it (1 or 2), else 0.
uint32_t CStretchEngine::HorzTapCountAt(int col) const {
const size_t weight_count =
weight_table_.GetPixelWeight(col)->GetWeights().size();
if (weight_count == 1) {
return 1;
}
if (weight_count == 2) {
return 2;
}
return 0;
}
// A span shorter than this stays on the per-column path entirely: even one
// group's setup -- four weight-table lookups plus the index/weight plan --
// costs more than the per-column dispatch it would replace.
constexpr int kMinSpanLength = 4;
// Plans one maximal span [start, end) of columns sharing `taps`: chops it
// into groups of 4 and, if at least 3 in 4 of them fit the gather, adds all
// the fitting ones, consecutive fitting groups forming one run. Below that
// share a fitting minority is all cost, since the span's leftover columns --
// including every non-fitting group -- fall back to the per-column path
// regardless. Spans with taps 0 (neither 1 nor 2) are never planned, nor
// spans shorter than kMinSpanLength.
void CStretchEngine::PlanHorzSpan(int start,
int end,
uint32_t taps,
const HorzPlanParams& params) {
if (taps == 0 || end - start < kMinSpanLength) {
return;
}
const int candidate_groups = (end - start) / 4;
// Builds directly into `horz_groups_` and `horz_runs_`; reverted below if
// too few groups fit. `original_last_run` guards the one case a plain
// `horz_runs_.resize()` wouldn't undo: this span's first group extending a
// run from an earlier span, mutating its `group_count` in place.
const size_t original_group_count = horz_groups_.size();
const size_t original_run_count = horz_runs_.size();
const std::optional<StretchHorzRun> original_last_run =
horz_runs_.empty() ? std::nullopt
: std::make_optional(horz_runs_.back());
int fitting = 0;
for (int i = 0; i < candidate_groups; ++i) {
std::optional<StretchHorzGroup> group =
BuildHorzGroup(start + 4 * i, taps, params);
if (!group.has_value()) {
continue;
}
++fitting;
// Extend the current run if this group's columns follow its last group's
// directly; otherwise this group starts a new run.
const int dest_col = start + 4 * i;
if (!horz_runs_.empty() &&
horz_runs_.back().dest_col + 4 * horz_runs_.back().group_count ==
static_cast<size_t>(dest_col)) {
++horz_runs_.back().group_count;
} else {
horz_runs_.push_back({dest_col, horz_groups_.size(), 1});
}
horz_groups_.push_back(group.value());
}
if (fitting * 4 < candidate_groups * 3) {
horz_groups_.resize(original_group_count);
horz_runs_.resize(original_run_count);
if (original_last_run.has_value()) {
horz_runs_.back() = original_last_run.value();
}
}
}
// Builds the group covering the 4 destination columns starting at `col`, all
// already known to share `taps`, or returns nullopt if the four columns'
// combined source window does not fit a 16- or 32-byte gather.
// `weight->GetSrcStart()` is a source pixel position; the plan is in bytes.
std::optional<StretchHorzGroup> CStretchEngine::BuildHorzGroup(
int col,
uint32_t taps,
const HorzPlanParams& params) const {
size_t min_start = weight_table_.GetPixelWeight(col)->GetSrcStart();
size_t max_start = min_start;
for (int i = 1; i < 4; ++i) {
const size_t start = weight_table_.GetPixelWeight(col + i)->GetSrcStart();
min_start = std::min(min_start, start);
max_start = std::max(max_start, start);
}
const size_t span_bytes = (max_start - min_start + taps) * params.bpp;
if (span_bytes > 32 || params.src_row_bytes < 16) {
return std::nullopt;
}
const bool wide = span_bytes > 16;
const size_t load_bytes = wide ? 32 : 16;
if (load_bytes > params.src_row_bytes) {
return std::nullopt;
}
// Keep the load inside the source row, sliding its base left if needed
// (mirrors the per-column path's own subspan, which is always in bounds).
size_t src_byte_base = min_start * params.bpp;
if (src_byte_base + load_bytes > params.src_row_bytes) {
src_byte_base = params.src_row_bytes - load_bytes;
}
StretchHorzGroup group;
group.taps = taps;
group.src_byte_base = src_byte_base;
group.wide = wide;
std::ranges::fill(group.idx[0], 0xFF);
std::ranges::fill(group.idx[1], 0xFF);
for (int k = 0; k < 4; ++k) {
const PixelWeight* weight = weight_table_.GetPixelWeight(col + k);
pdfium::span<const uint32_t> weights = weight->GetWeights();
group.weights[0][k] = weights[0];
group.weights[1][k] = taps == 2 ? weights[1] : 0;
for (uint32_t t = 0; t < taps; ++t) {
const size_t src_byte =
(weight->GetSrcStart() + t) * params.bpp - src_byte_base;
for (uint32_t c = 0; c < params.gathered; ++c) {
// Within the 32-byte window, so it fits the byte the gather reads.
group.idx[t][k * 4 + c] = pdfium::checked_cast<uint8_t>(src_byte + c);
}
}
}
return group;
}
// The groups of one planned run, in `horz_groups_`.
pdfium::span<const StretchHorzGroup> CStretchEngine::HorzRunGroups(
const StretchHorzRun& run) const {
return pdfium::span(horz_groups_).subspan(run.first_group, run.group_count);
}
bool CStretchEngine::ContinueStretchHorz(PauseIndicatorIface* pPause) {
if (!dest_width_) {
return false;
}
if (source_->SkipToScanline(cur_row_, pPause)) {
return true;
}
const int Bpp = dest_bpp_ / 8;
static const int kStrechPauseRows = 10;
int rows_to_go = kStrechPauseRows;
for (; cur_row_ < src_clip_.bottom; ++cur_row_) {
if (rows_to_go == 0) {
if (pPause && pPause->NeedToPauseNow()) {
return true;
}
rows_to_go = kStrechPauseRows;
}
pdfium::span<const uint8_t> src_row_span = source_->GetScanline(cur_row_);
pdfium::span<uint8_t> dest_span = inter_buf_.subspan(
(cur_row_ - src_clip_.top) * inter_pitch_, inter_pitch_);
size_t dest_span_index = 0;
// TODO(crbug.com/567627774): reduce duplicated code here
switch (trans_method_) {
case TransformMethod::k1BppTo8Bpp:
src_row_span = Expand1BppRow(src_row_span);
[[fallthrough]];
case TransformMethod::k8BppTo8Bpp: {
pdfium::span<const StretchHorzRun> runs(horz_runs_);
for (int col = dest_clip_.left; col < dest_clip_.right;) {
if (!runs.empty() && runs.front().dest_col == col) {
// A run of the 4-column groups PlanHorzGroups() found: one
// gather, one multiply-accumulate per tap, covering 4 columns at
// once. The index plan and weights come from the weight table,
// copied verbatim, and the terms are summed in tap order, so the
// output matches the per-column path below bit for bit.
const StretchHorzRun& run = runs.front();
StretchHorzRunSimd(HorzRunGroups(run), src_row_span,
dest_span.subspan(dest_span_index));
dest_span_index += 4 * run.group_count;
col += 4 * run.group_count;
runs = runs.subspan<1u>();
continue;
}
const PixelWeight* pWeights = weight_table_.GetPixelWeight(col);
pdfium::span<const uint32_t> weights = pWeights->GetWeights();
// One validated view of exactly the source bytes this column taps.
auto src_window =
src_row_span.subspan(pWeights->GetSrcStart(), weights.size());
uint32_t dest_a = 0;
// See the matching unrolls in the kManyBpptoManyBpp case below:
// one and two taps are the common trip counts and the loop is all
// overhead there. Tap order is unchanged.
switch (weights.size()) {
case 1:
dest_a = weights[0] * src_window[0];
break;
case 2:
dest_a = weights[0] * src_window[0] + weights[1] * src_window[1];
break;
default:
for (auto [pixel_weight, src_byte] :
fxcrt::Zip(weights, src_window)) {
dest_a += pixel_weight * src_byte;
}
break;
}
dest_span[dest_span_index++] = PixelFromFixed(dest_a);
++col;
}
break;
}
case TransformMethod::k1BppToManyBpp:
src_row_span = Expand1BppRow(src_row_span);
[[fallthrough]];
case TransformMethod::k8BppToManyBpp: {
// 3 bytes per destination pixel, written contiguously.
auto dest_pixels =
fxcrt::reinterpret_span<FX_BGR_STRUCT<uint8_t>>(dest_span);
size_t dest_index = 0;
for (int col = dest_clip_.left; col < dest_clip_.right; ++col) {
const PixelWeight* pWeights = weight_table_.GetPixelWeight(col);
pdfium::span<const uint32_t> weights = pWeights->GetWeights();
auto src_window =
src_row_span.subspan(pWeights->GetSrcStart(), weights.size());
uint32_t dest_r = 0;
uint32_t dest_g = 0;
uint32_t dest_b = 0;
for (auto [pixel_weight, src_byte] :
fxcrt::Zip(weights, src_window)) {
FX_ARGB argb = src_palette_[src_byte];
if (dest_format_ == FXDIB_Format::kBgr) {
dest_r += pixel_weight * static_cast<uint8_t>(argb >> 16);
dest_g += pixel_weight * static_cast<uint8_t>(argb >> 8);
dest_b += pixel_weight * static_cast<uint8_t>(argb);
} else {
dest_b += pixel_weight * static_cast<uint8_t>(argb >> 24);
dest_g += pixel_weight * static_cast<uint8_t>(argb >> 16);
dest_r += pixel_weight * static_cast<uint8_t>(argb >> 8);
}
}
FX_BGR_STRUCT<uint8_t>& dest = dest_pixels[dest_index++];
dest.blue = PixelFromFixed(dest_b);
dest.green = PixelFromFixed(dest_g);
dest.red = PixelFromFixed(dest_r);
}
break;
}
case TransformMethod::kManyBpptoManyBpp: {
// `Bpp` is the source pixel stride as well as the destination's:
// this method is only selected for >8bpp sources, and
// CFX_ImageStretcher::GetStretchedFormat() maps such sources to the
// same format on the destination side, so both are kBgr (3 bytes) or
// kBgrx (4 bytes, whose 4th byte this ignores). The generic lambda
// below is instantiated once per branch, so each instantiation gets a
// compile-time source pixel size.
auto stretch_row_from = [this, &dest_span](auto src_pixels) {
// The destination pixels have the same layout as the source
// pixels: 3 bytes fully written, or 4 bytes of which the 4th is
// left untouched (both here and on the generic path below).
using DestPixel =
std::remove_const_t<typename decltype(src_pixels)::element_type>;
auto dest_pixels = fxcrt::reinterpret_span<DestPixel>(dest_span);
pdfium::span<const StretchHorzRun> runs(horz_runs_);
size_t dest_index = 0;
for (int col = dest_clip_.left; col < dest_clip_.right;) {
if (!runs.empty() && runs.front().dest_col == col) {
// See the matching kernel in the k8BppTo8Bpp case above: one
// gather and multiply-accumulate per tap for all 4 columns of
// each group in the run, reading the same weights and source
// pixels, in the same tap order, as the per-column path below.
const StretchHorzRun& run = runs.front();
StretchHorzRunSimd(HorzRunGroups(run), src_pixels,
dest_pixels.subspan(dest_index));
dest_index += 4 * run.group_count;
col += 4 * run.group_count;
runs = runs.subspan<1u>();
continue;
}
const PixelWeight* pWeights = weight_table_.GetPixelWeight(col);
pdfium::span<const uint32_t> weights = pWeights->GetWeights();
auto window =
src_pixels.subspan(pWeights->GetSrcStart(), weights.size());
uint32_t dest_r = 0;
uint32_t dest_g = 0;
uint32_t dest_b = 0;
// One and two taps are the overwhelmingly common cases -- one is
// no scaling in this axis, two is bilinear -- and at that trip
// count the loop is all overhead. Unrolled explicitly; the terms
// are summed in tap order, so the result is identical.
switch (weights.size()) {
case 1: {
const auto& src0 = window[0];
dest_b = weights[0] * src0.blue;
dest_g = weights[0] * src0.green;
dest_r = weights[0] * src0.red;
break;
}
case 2: {
const auto& src0 = window[0];
const auto& src1 = window[1];
dest_b = weights[0] * src0.blue + weights[1] * src1.blue;
dest_g = weights[0] * src0.green + weights[1] * src1.green;
dest_r = weights[0] * src0.red + weights[1] * src1.red;
break;
}
default:
for (auto [pixel_weight, src] : fxcrt::Zip(weights, window)) {
dest_b += pixel_weight * src.blue;
dest_g += pixel_weight * src.green;
dest_r += pixel_weight * src.red;
}
break;
}
DestPixel& dest = dest_pixels[dest_index++];
dest.blue = PixelFromFixed(dest_b);
dest.green = PixelFromFixed(dest_g);
dest.red = PixelFromFixed(dest_r);
++col;
}
};
if (Bpp == 3) {
stretch_row_from(
fxcrt::reinterpret_span<const FX_BGR_STRUCT<uint8_t>>(
src_row_span));
} else {
CHECK_EQ(Bpp, 4);
stretch_row_from(
fxcrt::reinterpret_span<const FX_BGRA_STRUCT<uint8_t>>(
src_row_span));
}
break;
}
case TransformMethod::kManyBpptoManyBppWithAlpha: {
CHECK(has_alpha_);
// `has_alpha_` implies both source and destination are kBgra
// (asserted in the constructor), so the source pixels are 4-byte
// BGRA structs.
CHECK_EQ(Bpp, 4);
auto src_pixels =
fxcrt::reinterpret_span<const FX_BGRA_STRUCT<uint8_t>>(
src_row_span);
auto dest_pixels =
fxcrt::reinterpret_span<FX_BGRA_STRUCT<uint8_t>>(dest_span);
pdfium::span<const StretchHorzRun> runs(horz_runs_);
size_t dest_index = 0;
for (int col = dest_clip_.left; col < dest_clip_.right;) {
if (!runs.empty() && runs.front().dest_col == col) {
// See the matching kernels above: one gather and
// multiply-accumulate per tap for all 4 columns of each group in
// the run, including the per-lane weight*alpha/255 divide the
// per-column path below performs, in the same tap order.
const StretchHorzRun& run = runs.front();
StretchHorzRunAlphaSimd(HorzRunGroups(run), src_pixels,
dest_pixels.subspan(dest_index));
dest_index += 4 * run.group_count;
col += 4 * run.group_count;
runs = runs.subspan<1u>();
continue;
}
const PixelWeight* pWeights = weight_table_.GetPixelWeight(col);
pdfium::span<const uint32_t> weights = pWeights->GetWeights();
auto window =
src_pixels.subspan(pWeights->GetSrcStart(), weights.size());
uint32_t dest_a = 0;
uint32_t dest_r = 0;
uint32_t dest_g = 0;
uint32_t dest_b = 0;
// At one and two taps, the loop's own setup costs more than the
// multiply-adds it guards, so those counts get straight-line code.
// The terms are summed in tap order, so the result is identical to
// the loop below.
switch (weights.size()) {
case 1: {
const auto& src0 = window[0];
const uint32_t pw0 = weights[0] * src0.alpha / 255;
dest_b = pw0 * src0.blue;
dest_g = pw0 * src0.green;
dest_r = pw0 * src0.red;
dest_a = pw0;
break;
}
case 2: {
const auto& src0 = window[0];
const auto& src1 = window[1];
const uint32_t pw0 = weights[0] * src0.alpha / 255;
const uint32_t pw1 = weights[1] * src1.alpha / 255;
dest_b = pw0 * src0.blue + pw1 * src1.blue;
dest_g = pw0 * src0.green + pw1 * src1.green;
dest_r = pw0 * src0.red + pw1 * src1.red;
dest_a = pw0 + pw1;
break;
}
default:
for (auto [weight, src] : fxcrt::Zip(weights, window)) {
uint32_t pixel_weight = weight * src.alpha / 255;
dest_b += pixel_weight * src.blue;
dest_g += pixel_weight * src.green;
dest_r += pixel_weight * src.red;
dest_a += pixel_weight;
}
break;
}
FX_BGRA_STRUCT<uint8_t>& dest = dest_pixels[dest_index++];
dest.blue = PixelFromFixed(dest_b);
dest.green = PixelFromFixed(dest_g);
dest.red = PixelFromFixed(dest_r);
dest.alpha = PixelFromFixed(255 * dest_a);
++col;
}
break;
}
}
rows_to_go--;
}
return false;
}
void CStretchEngine::StretchVert() {
if (dest_height_ == 0) {
return;
}
WeightTable table;
if (!table.CalculateWeights(dest_height_, dest_clip_.top, dest_clip_.bottom,
src_height_, src_clip_.top, src_clip_.bottom,
resample_options_)) {
return;
}
const int dest_cols = dest_clip_.right - dest_clip_.left;
if (dest_cols <= 0) {
return;
}
const int DestBpp = dest_bpp_ / 8;
const size_t row_bytes = Fx2DSizeOrDie(dest_cols, DestBpp);
// Taps are the outer loop, so each one reads a contiguous row of the
// intermediate buffer into a row-wide accumulator. Per byte the taps are
// still summed in tap order, so the output is unchanged.
//
// Allocated once outside the row loop rather than constructed inside it:
// `row_bytes` uint32s is tens of kilobytes for a wide destination, and one
// allocate/free pair per destination row is a large fraction of a scanned
// page's render time.
//
// Uninitialised, because the first tap of each row assigns rather than
// accumulates and so writes every element before anything reads it. The
// one case with no first tap -- an empty weight range -- zeroes the
// accumulator explicitly below.
auto accum = FixedSizeDataVector<uint32_t>::Uninit(row_bytes);
pdfium::span<uint32_t> accum_span = accum.span();
for (int row = dest_clip_.top; row < dest_clip_.bottom; ++row) {
const PixelWeight* pWeights = table.GetPixelWeight(row);
pdfium::span<const uint32_t> weights = pWeights->GetWeights();
if (!weights.empty()) {
CHECK_GE(pWeights->src_start_, src_clip_.top);
CHECK_LT(pWeights->src_end_, src_clip_.bottom);
}
// No taps for this row, so the tap loop below writes nothing: zero the
// accumulator so the conversion switch sees defined values.
if (weights.empty()) {
std::ranges::fill(accum_span, 0u);
}
for (size_t i = 0; i < weights.size(); ++i) {
// Hoisted deliberately: `weights` and `accum_span` are both spans of
// uint32_t, so the compiler cannot prove the accumulation below does
// not alias the weight, and would otherwise reload it every iteration
// and decline to vectorize.
const uint32_t pixel_weight = weights[i];
auto src_row = inter_buf_.subspan(
(static_cast<size_t>(pWeights->src_start_ - src_clip_.top) + i) *
inter_pitch_,
row_bytes);
// The first tap assigns rather than accumulates. That initialises the
// whole accumulator, so no separate zeroing pass over the row is
// needed -- one full-width write instead of a write plus a
// read-modify-write.
if (i == 0) {
for (auto [acc, src_byte] : fxcrt::Zip(accum_span, src_row)) {
acc = pixel_weight * src_byte;
}
} else {
for (auto [acc, src_byte] : fxcrt::Zip(accum_span, src_row)) {
acc += pixel_weight * src_byte;
}
}
}
auto dest_row = pdfium::span(dest_scanline_).first(row_bytes);
switch (trans_method_) {
case TransformMethod::k1BppTo8Bpp:
case TransformMethod::k1BppToManyBpp:
case TransformMethod::k8BppTo8Bpp: {
// Only the first channel of each destination pixel carries data on
// these paths; any remaining bytes are left untouched. Typed pixel
// views give every width the same loop shape. The generic lambda
// below is instantiated once per branch, so each instantiation gets a
// compile-time destination pixel size.
auto write_first_channel = [](auto dest_pixels, auto acc_pixels) {
for (auto [dest, acc] : fxcrt::Zip(dest_pixels, acc_pixels)) {
dest.blue = PixelFromFixed(acc.blue);
}
};
if (DestBpp == 1) {
for (auto [dest, acc] : fxcrt::Zip(dest_row, accum_span)) {
dest = PixelFromFixed(acc);
}
} else if (DestBpp == 3) {
write_first_channel(
fxcrt::reinterpret_span<FX_BGR_STRUCT<uint8_t>>(dest_row),
fxcrt::reinterpret_span<const FX_BGR_STRUCT<uint32_t>>(
accum_span));
} else {
CHECK_EQ(DestBpp, 4);
write_first_channel(
fxcrt::reinterpret_span<FX_BGRA_STRUCT<uint8_t>>(dest_row),
fxcrt::reinterpret_span<const FX_BGRA_STRUCT<uint32_t>>(
accum_span));
}
break;
}
case TransformMethod::k8BppToManyBpp:
case TransformMethod::kManyBpptoManyBpp: {
// The destination is kBgr (3 bytes per pixel, fully written) or
// kBgrx (4 bytes per pixel, 4th byte left untouched); see the
// matching dispatch in ContinueStretchHorz().
if (DestBpp == 3) {
for (auto [dest, acc] : fxcrt::Zip(dest_row, accum_span)) {
dest = PixelFromFixed(acc);
}
} else {
CHECK_EQ(DestBpp, 4);
auto dest_pixels =
fxcrt::reinterpret_span<FX_BGRA_STRUCT<uint8_t>>(dest_row);
auto acc_pixels =
fxcrt::reinterpret_span<const FX_BGRA_STRUCT<uint32_t>>(
accum_span);
for (auto [dest, acc] : fxcrt::Zip(dest_pixels, acc_pixels)) {
dest.blue = PixelFromFixed(acc.blue);
dest.green = PixelFromFixed(acc.green);
dest.red = PixelFromFixed(acc.red);
}
}
break;
}
case TransformMethod::kManyBpptoManyBppWithAlpha: {
CHECK(has_alpha_);
CHECK_EQ(DestBpp, 4);
auto dest_pixels =
fxcrt::reinterpret_span<FX_BGRA_STRUCT<uint8_t>>(dest_row);
auto acc_pixels =
fxcrt::reinterpret_span<const FX_BGRA_STRUCT<uint32_t>>(accum_span);
for (auto [dest, acc] : fxcrt::Zip(dest_pixels, acc_pixels)) {
if (acc.alpha) {
int r = acc.red * 255 / acc.alpha;
int g = acc.green * 255 / acc.alpha;
int b = acc.blue * 255 / acc.alpha;
dest.blue = std::clamp(b, 0, 255);
dest.green = std::clamp(g, 0, 255);
dest.red = std::clamp(r, 0, 255);
}
dest.alpha = PixelFromFixed(acc.alpha);
}
break;
}
}
dest_bitmap_->ComposeScanline(row - dest_clip_.top, dest_scanline_);
}
}
pdfium::span<const uint8_t> CStretchEngine::Expand1BppRow(
pdfium::span<const uint8_t> src_row) {
// Maps a byte of eight 1bpp pixels, most significant bit first, to eight
// bytes that are 0 or 255.
static constinit auto kBitsToBytes = [] {
std::array<std::array<uint8_t, 8>, 256> table{};
for (size_t value = 0; value < table.size(); ++value) {
for (size_t bit = 0; bit < 8; ++bit) {
table[value][bit] = (value >> (7 - bit)) & 1 ? 255 : 0;
}
}
return table;
}();
pdfium::span<uint8_t> expanded = expanded_row_.span();
const size_t src_bytes = expanded.size() / 8;
for (size_t i = 0; i < src_bytes; ++i) {
fxcrt::spancpy(expanded.subspan(i * 8, 8u),
pdfium::span(kBitsToBytes[src_row[i]]));
}
return expanded;
}