Update PartitionAlloc from Chromium at r599712.
BUG=pdfium:1170
Change-Id: I0f8dfb3d517beaa682a9ca7ad4831c5a7a10dc3b
Reviewed-on: https://pdfium-review.googlesource.com/c/44073
Reviewed-by: Tom Sepez <tsepez@chromium.org>
Commit-Queue: Lei Zhang <thestig@chromium.org>
diff --git a/core/fxcrt/string_data_template.h b/core/fxcrt/string_data_template.h
index f65d37c..0fe679d 100644
--- a/core/fxcrt/string_data_template.h
+++ b/core/fxcrt/string_data_template.h
@@ -36,8 +36,8 @@
size_t usableLen = (totalSize - overhead) / sizeof(CharType);
ASSERT(usableLen >= nLen);
- void* pData = pdfium::base::PartitionAllocGeneric(
- gStringPartitionAllocator.root(), totalSize, "StringDataTemplate");
+ void* pData = gStringPartitionAllocator.root()->Alloc(totalSize,
+ "StringDataTemplate");
return new (pData) StringDataTemplate(nLen, usableLen);
}
@@ -50,8 +50,7 @@
void Retain() { ++m_nRefs; }
void Release() {
if (--m_nRefs <= 0)
- pdfium::base::PartitionFreeGeneric(gStringPartitionAllocator.root(),
- this);
+ gStringPartitionAllocator.root()->Free(this);
}
bool CanOperateInPlace(size_t nTotalLen) const {
diff --git a/fxjs/cfx_v8.cpp b/fxjs/cfx_v8.cpp
index 1d2dd94..26c9a43 100644
--- a/fxjs/cfx_v8.cpp
+++ b/fxjs/cfx_v8.cpp
@@ -215,11 +215,10 @@
void* CFX_V8ArrayBufferAllocator::AllocateUninitialized(size_t length) {
if (length > kMaxAllowedBytes)
return nullptr;
- return pdfium::base::PartitionAllocGeneric(
- gArrayBufferPartitionAllocator.root(), length, "CFX_V8ArrayBuffer");
+ return gArrayBufferPartitionAllocator.root()->Alloc(length,
+ "CFX_V8ArrayBuffer");
}
void CFX_V8ArrayBufferAllocator::Free(void* data, size_t length) {
- pdfium::base::PartitionFreeGeneric(gArrayBufferPartitionAllocator.root(),
- data);
+ gArrayBufferPartitionAllocator.root()->Free(data);
}
diff --git a/third_party/BUILD.gn b/third_party/BUILD.gn
index 7a5a6c3..2dbf920 100644
--- a/third_party/BUILD.gn
+++ b/third_party/BUILD.gn
@@ -552,10 +552,28 @@
"base/allocator/partition_allocator/address_space_randomization.cc",
"base/allocator/partition_allocator/address_space_randomization.h",
"base/allocator/partition_allocator/oom.h",
+ "base/allocator/partition_allocator/oom_callback.cc",
+ "base/allocator/partition_allocator/oom_callback.h",
"base/allocator/partition_allocator/page_allocator.cc",
"base/allocator/partition_allocator/page_allocator.h",
+ "base/allocator/partition_allocator/page_allocator_constants.h",
+ "base/allocator/partition_allocator/page_allocator_internal.h",
+ "base/allocator/partition_allocator/page_allocator_internals_posix.h",
+ "base/allocator/partition_allocator/page_allocator_internals_win.h",
"base/allocator/partition_allocator/partition_alloc.cc",
"base/allocator/partition_allocator/partition_alloc.h",
+ "base/allocator/partition_allocator/partition_alloc_constants.h",
+ "base/allocator/partition_allocator/partition_bucket.cc",
+ "base/allocator/partition_allocator/partition_bucket.h",
+ "base/allocator/partition_allocator/partition_cookie.h",
+ "base/allocator/partition_allocator/partition_direct_map_extent.h",
+ "base/allocator/partition_allocator/partition_freelist_entry.h",
+ "base/allocator/partition_allocator/partition_oom.cc",
+ "base/allocator/partition_allocator/partition_oom.h",
+ "base/allocator/partition_allocator/partition_page.cc",
+ "base/allocator/partition_allocator/partition_page.h",
+ "base/allocator/partition_allocator/partition_root_base.cc",
+ "base/allocator/partition_allocator/partition_root_base.h",
"base/allocator/partition_allocator/spin_lock.cc",
"base/allocator/partition_allocator/spin_lock.h",
"base/base_export.h",
diff --git a/third_party/base/allocator/partition_allocator/OWNERS b/third_party/base/allocator/partition_allocator/OWNERS
index 95d9982..b0a2a85 100644
--- a/third_party/base/allocator/partition_allocator/OWNERS
+++ b/third_party/base/allocator/partition_allocator/OWNERS
@@ -1,2 +1,8 @@
+ajwong@chromium.org
+haraken@chromium.org
palmer@chromium.org
tsepez@chromium.org
+
+# TEAM: platform-architecture-dev@chromium.org
+# Also: security-dev@chromium.org
+# COMPONENT: Blink>MemoryAllocator>Partition
diff --git a/third_party/base/allocator/partition_allocator/address_space_randomization.cc b/third_party/base/allocator/partition_allocator/address_space_randomization.cc
index d16970a..135c67d 100644
--- a/third_party/base/allocator/partition_allocator/address_space_randomization.cc
+++ b/third_party/base/allocator/partition_allocator/address_space_randomization.cc
@@ -7,19 +7,17 @@
#include "build/build_config.h"
#include "third_party/base/allocator/partition_allocator/page_allocator.h"
#include "third_party/base/allocator/partition_allocator/spin_lock.h"
+#include "third_party/base/logging.h"
#if defined(OS_WIN)
-#include <windows.h>
+#include <windows.h> // Must be in front of other Windows header files.
+
+#include <VersionHelpers.h>
#else
#include <sys/time.h>
#include <unistd.h>
#endif
-// VersionHelpers.h must be included after windows.h.
-#if defined(OS_WIN)
-#include <VersionHelpers.h>
-#endif
-
namespace pdfium {
namespace base {
@@ -27,7 +25,7 @@
// This is the same PRNG as used by tcmalloc for mapping address randomness;
// see http://burtleburtle.net/bob/rand/smallprng.html
-struct ranctx {
+struct RandomContext {
subtle::SpinLock lock;
bool initialized;
uint32_t a;
@@ -36,9 +34,16 @@
uint32_t d;
};
+RandomContext* GetRandomContext() {
+ static RandomContext* s_RandomContext = nullptr;
+ if (!s_RandomContext)
+ s_RandomContext = new RandomContext();
+ return s_RandomContext;
+}
+
#define rot(x, k) (((x) << (k)) | ((x) >> (32 - (k))))
-uint32_t ranvalInternal(ranctx* x) {
+uint32_t RandomValueInternal(RandomContext* x) {
uint32_t e = x->a - rot(x->b, 27);
x->a = x->b ^ rot(x->c, 17);
x->b = x->c + x->d;
@@ -49,7 +54,7 @@
#undef rot
-uint32_t ranval(ranctx* x) {
+uint32_t RandomValue(RandomContext* x) {
subtle::SpinLock::Guard guard(x->lock);
if (UNLIKELY(!x->initialized)) {
x->initialized = true;
@@ -73,29 +78,34 @@
x->a = 0xf1ea5eed;
x->b = x->c = x->d = seed;
for (int i = 0; i < 20; ++i) {
- (void)ranvalInternal(x);
+ RandomValueInternal(x);
}
}
- uint32_t ret = ranvalInternal(x);
- return ret;
-}
-static struct ranctx s_ranctx;
+ return RandomValueInternal(x);
+}
} // namespace
-// Calculates a random preferred mapping address. In calculating an address, we
-// balance good ASLR against not fragmenting the address space too badly.
+void SetRandomPageBaseSeed(int64_t seed) {
+ RandomContext* x = GetRandomContext();
+ subtle::SpinLock::Guard guard(x->lock);
+ // Set RNG to initial state.
+ x->initialized = true;
+ x->a = x->b = static_cast<uint32_t>(seed);
+ x->c = x->d = static_cast<uint32_t>(seed >> 32);
+}
+
void* GetRandomPageBase() {
- uintptr_t random;
- random = static_cast<uintptr_t>(ranval(&s_ranctx));
-#if defined(ARCH_CPU_X86_64)
- random <<= 32UL;
- random |= static_cast<uintptr_t>(ranval(&s_ranctx));
-// This address mask gives a low likelihood of address space collisions. We
-// handle the situation gracefully if there is a collision.
-#if defined(OS_WIN)
- random &= 0x3ffffffffffUL;
+ uintptr_t random = static_cast<uintptr_t>(RandomValue(GetRandomContext()));
+
+#if defined(ARCH_CPU_64_BITS)
+ random <<= 32ULL;
+ random |= static_cast<uintptr_t>(RandomValue(GetRandomContext()));
+
+// The kASLRMask and kASLROffset constants will be suitable for the
+// OS and build configuration.
+#if defined(OS_WIN) && !defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
// Windows >= 8.1 has the full 47 bits. Use them where available.
static bool windows_81 = false;
static bool windows_81_initialized = false;
@@ -104,38 +114,32 @@
windows_81_initialized = true;
}
if (!windows_81) {
- random += 0x10000000000UL;
+ random &= internal::kASLRMaskBefore8_10;
+ } else {
+ random &= internal::kASLRMask;
}
-#elif defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
- // This range is copied from the TSan source, but works for all tools.
- random &= 0x007fffffffffUL;
- random += 0x7e8000000000UL;
+ random += internal::kASLROffset;
#else
- // Linux and OS X support the full 47-bit user space of x64 processors.
- random &= 0x3fffffffffffUL;
-#endif
-#elif defined(ARCH_CPU_ARM64)
- // ARM64 on Linux has 39-bit user space.
- random &= 0x3fffffffffUL;
- random += 0x1000000000UL;
-#else // !defined(ARCH_CPU_X86_64) && !defined(ARCH_CPU_ARM64)
+ random &= internal::kASLRMask;
+ random += internal::kASLROffset;
+#endif // defined(OS_WIN) && !defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
+#else // defined(ARCH_CPU_32_BITS)
#if defined(OS_WIN)
// On win32 host systems the randomization plus huge alignment causes
// excessive fragmentation. Plus most of these systems lack ASLR, so the
// randomization isn't buying anything. In that case we just skip it.
// TODO(jschuh): Just dump the randomization when HE-ASLR is present.
- static BOOL isWow64 = -1;
- if (isWow64 == -1 && !IsWow64Process(GetCurrentProcess(), &isWow64))
- isWow64 = FALSE;
- if (!isWow64)
+ static BOOL is_wow64 = -1;
+ if (is_wow64 == -1 && !IsWow64Process(GetCurrentProcess(), &is_wow64))
+ is_wow64 = FALSE;
+ if (!is_wow64)
return nullptr;
#endif // defined(OS_WIN)
- // This is a good range on Windows, Linux and Mac.
- // Allocates in the 0.5-1.5GB region.
- random &= 0x3fffffff;
- random += 0x20000000;
-#endif // defined(ARCH_CPU_X86_64)
- random &= kPageAllocationGranularityBaseMask;
+ random &= internal::kASLRMask;
+ random += internal::kASLROffset;
+#endif // defined(ARCH_CPU_32_BITS)
+
+ DCHECK_EQ(0ULL, (random & kPageAllocationGranularityOffsetMask));
return reinterpret_cast<void*>(random);
}
diff --git a/third_party/base/allocator/partition_allocator/address_space_randomization.h b/third_party/base/allocator/partition_allocator/address_space_randomization.h
index 97c5f60..efad668 100644
--- a/third_party/base/allocator/partition_allocator/address_space_randomization.h
+++ b/third_party/base/allocator/partition_allocator/address_space_randomization.h
@@ -2,17 +2,207 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-#ifndef BASE_ALLOCATOR_PARTITION_ALLOCATOR_ADDRESS_SPACE_RANDOMIZATION
-#define BASE_ALLOCATOR_PARTITION_ALLOCATOR_ADDRESS_SPACE_RANDOMIZATION
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_ADDRESS_SPACE_RANDOMIZATION_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_ADDRESS_SPACE_RANDOMIZATION_H_
+
+#include "build/build_config.h"
+#include "third_party/base/allocator/partition_allocator/page_allocator.h"
+#include "third_party/base/base_export.h"
namespace pdfium {
namespace base {
+// Sets the seed for the random number generator used by GetRandomPageBase in
+// order to generate a predictable sequence of addresses. May be called multiple
+// times.
+BASE_EXPORT void SetRandomPageBaseSeed(int64_t seed);
+
// Calculates a random preferred mapping address. In calculating an address, we
// balance good ASLR against not fragmenting the address space too badly.
-void* GetRandomPageBase();
+BASE_EXPORT void* GetRandomPageBase();
+
+namespace internal {
+
+constexpr uintptr_t AslrAddress(uintptr_t mask) {
+ return mask & kPageAllocationGranularityBaseMask;
+}
+constexpr uintptr_t AslrMask(uintptr_t bits) {
+ return AslrAddress((1ULL << bits) - 1ULL);
+}
+
+// Turn off formatting, because the thicket of nested ifdefs below is
+// incomprehensible without indentation. It is also incomprehensible with
+// indentation, but the only other option is a combinatorial explosion of
+// *_{win,linux,mac,foo}_{32,64}.h files.
+//
+// clang-format off
+
+#if defined(ARCH_CPU_64_BITS)
+
+ #if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
+
+ // We shouldn't allocate system pages at all for sanitizer builds. However,
+ // we do, and if random hint addresses interfere with address ranges
+ // hard-coded in those tools, bad things happen. This address range is
+ // copied from TSAN source but works with all tools. See
+ // https://crbug.com/539863.
+ constexpr uintptr_t kASLRMask = AslrAddress(0x007fffffffffULL);
+ constexpr uintptr_t kASLROffset = AslrAddress(0x7e8000000000ULL);
+
+ #elif defined(OS_WIN)
+
+ // Windows 8.10 and newer support the full 48 bit address range. Older
+ // versions of Windows only support 44 bits. Since kASLROffset is non-zero
+ // and may cause a carry, use 47 and 43 bit masks. See
+ // http://www.alex-ionescu.com/?p=246
+ constexpr uintptr_t kASLRMask = AslrMask(47);
+ constexpr uintptr_t kASLRMaskBefore8_10 = AslrMask(43);
+ // Try not to map pages into the range where Windows loads DLLs by default.
+ constexpr uintptr_t kASLROffset = 0x80000000ULL;
+
+ #elif defined(OS_MACOSX)
+
+ // macOS as of 10.12.5 does not clean up entries in page map levels 3/4
+ // [PDP/PML4] created from mmap or mach_vm_allocate, even after the region
+ // is destroyed. Using a virtual address space that is too large causes a
+ // leak of about 1 wired [can never be paged out] page per call to mmap. The
+ // page is only reclaimed when the process is killed. Confine the hint to a
+ // 39-bit section of the virtual address space.
+ //
+ // This implementation adapted from
+ // https://chromium-review.googlesource.com/c/v8/v8/+/557958. The difference
+ // is that here we clamp to 39 bits, not 32.
+ //
+ // TODO(crbug.com/738925): Remove this limitation if/when the macOS behavior
+ // changes.
+ constexpr uintptr_t kASLRMask = AslrMask(38);
+ constexpr uintptr_t kASLROffset = AslrAddress(0x1000000000ULL);
+
+ #elif defined(OS_POSIX) || defined(OS_FUCHSIA)
+
+ #if defined(ARCH_CPU_X86_64)
+
+ // Linux (and macOS) support the full 47-bit user space of x64 processors.
+ // Use only 46 to allow the kernel a chance to fulfill the request.
+ constexpr uintptr_t kASLRMask = AslrMask(46);
+ constexpr uintptr_t kASLROffset = AslrAddress(0);
+
+ #elif defined(ARCH_CPU_ARM64)
+
+ #if defined(OS_ANDROID)
+
+ // Restrict the address range on Android to avoid a large performance
+ // regression in single-process WebViews. See https://crbug.com/837640.
+ constexpr uintptr_t kASLRMask = AslrMask(30);
+ constexpr uintptr_t kASLROffset = AslrAddress(0x20000000ULL);
+
+ #else
+
+ // ARM64 on Linux has 39-bit user space. Use 38 bits since kASLROffset
+ // could cause a carry.
+ constexpr uintptr_t kASLRMask = AslrMask(38);
+ constexpr uintptr_t kASLROffset = AslrAddress(0x1000000000ULL);
+
+ #endif
+
+ #elif defined(ARCH_CPU_PPC64)
+
+ #if defined(OS_AIX)
+
+ // AIX has 64 bits of virtual addressing, but we limit the address range
+ // to (a) minimize segment lookaside buffer (SLB) misses; and (b) use
+ // extra address space to isolate the mmap regions.
+ constexpr uintptr_t kASLRMask = AslrMask(30);
+ constexpr uintptr_t kASLROffset = AslrAddress(0x400000000000ULL);
+
+ #elif defined(ARCH_CPU_BIG_ENDIAN)
+
+ // Big-endian Linux PPC has 44 bits of virtual addressing. Use 42.
+ constexpr uintptr_t kASLRMask = AslrMask(42);
+ constexpr uintptr_t kASLROffset = AslrAddress(0);
+
+ #else // !defined(OS_AIX) && !defined(ARCH_CPU_BIG_ENDIAN)
+
+ // Little-endian Linux PPC has 48 bits of virtual addressing. Use 46.
+ constexpr uintptr_t kASLRMask = AslrMask(46);
+ constexpr uintptr_t kASLROffset = AslrAddress(0);
+
+ #endif // !defined(OS_AIX) && !defined(ARCH_CPU_BIG_ENDIAN)
+
+ #elif defined(ARCH_CPU_S390X)
+
+ // Linux on Z uses bits 22 - 32 for Region Indexing, which translates to
+ // 42 bits of virtual addressing. Truncate to 40 bits to allow kernel a
+ // chance to fulfill the request.
+ constexpr uintptr_t kASLRMask = AslrMask(40);
+ constexpr uintptr_t kASLROffset = AslrAddress(0);
+
+ #elif defined(ARCH_CPU_S390)
+
+ // 31 bits of virtual addressing. Truncate to 29 bits to allow the kernel
+ // a chance to fulfill the request.
+ constexpr uintptr_t kASLRMask = AslrMask(29);
+ constexpr uintptr_t kASLROffset = AslrAddress(0);
+
+ #else // !defined(ARCH_CPU_X86_64) && !defined(ARCH_CPU_PPC64) &&
+ // !defined(ARCH_CPU_S390X) && !defined(ARCH_CPU_S390)
+
+ // For all other POSIX variants, use 30 bits.
+ constexpr uintptr_t kASLRMask = AslrMask(30);
+
+ #if defined(OS_SOLARIS)
+
+ // For our Solaris/illumos mmap hint, we pick a random address in the
+ // bottom half of the top half of the address space (that is, the third
+ // quarter). Because we do not MAP_FIXED, this will be treated only as a
+ // hint -- the system will not fail to mmap because something else
+ // happens to already be mapped at our random address. We deliberately
+ // set the hint high enough to get well above the system's break (that
+ // is, the heap); Solaris and illumos will try the hint and if that
+ // fails allocate as if there were no hint at all. The high hint
+ // prevents the break from getting hemmed in at low values, ceding half
+ // of the address space to the system heap.
+ constexpr uintptr_t kASLROffset = AslrAddress(0x80000000ULL);
+
+ #elif defined(OS_AIX)
+
+ // The range 0x30000000 - 0xD0000000 is available on AIX; choose the
+ // upper range.
+ constexpr uintptr_t kASLROffset = AslrAddress(0x90000000ULL);
+
+ #else // !defined(OS_SOLARIS) && !defined(OS_AIX)
+
+ // The range 0x20000000 - 0x60000000 is relatively unpopulated across a
+ // variety of ASLR modes (PAE kernel, NX compat mode, etc) and on macOS
+ // 10.6 and 10.7.
+ constexpr uintptr_t kASLROffset = AslrAddress(0x20000000ULL);
+
+ #endif // !defined(OS_SOLARIS) && !defined(OS_AIX)
+
+ #endif // !defined(ARCH_CPU_X86_64) && !defined(ARCH_CPU_PPC64) &&
+ // !defined(ARCH_CPU_S390X) && !defined(ARCH_CPU_S390)
+
+ #endif // defined(OS_POSIX)
+
+#elif defined(ARCH_CPU_32_BITS)
+
+ // This is a good range on 32-bit Windows and Android (the only platforms on
+ // which we support 32-bitness). Allocates in the 0.5 - 1.5 GiB region. There
+ // is no issue with carries here.
+ constexpr uintptr_t kASLRMask = AslrMask(30);
+ constexpr uintptr_t kASLROffset = AslrAddress(0x20000000ULL);
+
+#else
+
+ #error Please tell us about your exotic hardware! Sounds interesting.
+
+#endif // defined(ARCH_CPU_32_BITS)
+
+// clang-format on
+
+} // namespace internal
} // namespace base
} // namespace pdfium
-#endif // BASE_ALLOCATOR_PARTITION_ALLOCATOR_ADDRESS_SPACE_RANDOMIZATION
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_ADDRESS_SPACE_RANDOMIZATION_H_
diff --git a/third_party/base/allocator/partition_allocator/oom.h b/third_party/base/allocator/partition_allocator/oom.h
index 41f29b5..bbd1ead 100644
--- a/third_party/base/allocator/partition_allocator/oom.h
+++ b/third_party/base/allocator/partition_allocator/oom.h
@@ -2,9 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-#ifndef BASE_ALLOCATOR_OOM_H
-#define BASE_ALLOCATOR_OOM_H
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_OOM_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_OOM_H_
+#include "third_party/base/allocator/partition_allocator/oom_callback.h"
#include "third_party/base/logging.h"
#if defined(OS_WIN)
@@ -23,15 +24,17 @@
#define OOM_CRASH() \
do { \
OOM_CRASH_PREVENT_ICF(); \
+ base::internal::RunPartitionAllocOomCallback(); \
::RaiseException(0xE0000008, EXCEPTION_NONCONTINUABLE, 0, nullptr); \
IMMEDIATE_CRASH(); \
} while (0)
#else
-#define OOM_CRASH() \
- do { \
- OOM_CRASH_PREVENT_ICF(); \
- IMMEDIATE_CRASH(); \
+#define OOM_CRASH() \
+ do { \
+ base::internal::RunPartitionAllocOomCallback(); \
+ OOM_CRASH_PREVENT_ICF(); \
+ IMMEDIATE_CRASH(); \
} while (0)
#endif
-#endif // BASE_ALLOCATOR_OOM_H
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_OOM_H_
diff --git a/third_party/base/allocator/partition_allocator/oom_callback.cc b/third_party/base/allocator/partition_allocator/oom_callback.cc
new file mode 100644
index 0000000..36c6e97
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/oom_callback.cc
@@ -0,0 +1,28 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/base/allocator/partition_allocator/oom_callback.h"
+#include "third_party/base/logging.h"
+
+namespace pdfium {
+namespace base {
+
+namespace {
+PartitionAllocOomCallback g_oom_callback;
+} // namespace
+
+void SetPartitionAllocOomCallback(PartitionAllocOomCallback callback) {
+ DCHECK(!g_oom_callback);
+ g_oom_callback = callback;
+}
+
+namespace internal {
+void RunPartitionAllocOomCallback() {
+ if (g_oom_callback)
+ g_oom_callback();
+}
+} // namespace internal
+
+} // namespace base
+} // namespace pdfium
diff --git a/third_party/base/allocator/partition_allocator/oom_callback.h b/third_party/base/allocator/partition_allocator/oom_callback.h
new file mode 100644
index 0000000..044b167
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/oom_callback.h
@@ -0,0 +1,26 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_OOM_CALLBACK_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_OOM_CALLBACK_H_
+
+#include "third_party/base/base_export.h"
+
+namespace pdfium {
+namespace base {
+typedef void (*PartitionAllocOomCallback)();
+// Registers a callback to be invoked during an OOM_CRASH(). OOM_CRASH is
+// invoked by users of PageAllocator (including PartitionAlloc) to signify an
+// allocation failure from the platform.
+BASE_EXPORT void SetPartitionAllocOomCallback(
+ PartitionAllocOomCallback callback);
+
+namespace internal {
+BASE_EXPORT void RunPartitionAllocOomCallback();
+} // namespace internal
+
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_OOM_CALLBACK_H_
diff --git a/third_party/base/allocator/partition_allocator/page_allocator.cc b/third_party/base/allocator/partition_allocator/page_allocator.cc
index 0869bdb..a65fbaa 100644
--- a/third_party/base/allocator/partition_allocator/page_allocator.cc
+++ b/third_party/base/allocator/partition_allocator/page_allocator.cc
@@ -10,151 +10,163 @@
#include "build/build_config.h"
#include "third_party/base/allocator/partition_allocator/address_space_randomization.h"
-#include "third_party/base/base_export.h"
+#include "third_party/base/allocator/partition_allocator/page_allocator_internal.h"
+#include "third_party/base/allocator/partition_allocator/spin_lock.h"
#include "third_party/base/logging.h"
+#include "third_party/base/numerics/safe_math.h"
-#if defined(OS_POSIX)
-
-#include <errno.h>
-#include <sys/mman.h>
-
-#ifndef MADV_FREE
-#define MADV_FREE MADV_DONTNEED
-#endif
-
-#ifndef MAP_ANONYMOUS
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
-// On POSIX |mmap| uses a nearby address if the hint address is blocked.
-static const bool kHintIsAdvisory = true;
-static std::atomic<int32_t> s_allocPageErrorCode{0};
-
-#elif defined(OS_WIN)
-
+#if defined(OS_WIN)
#include <windows.h>
+#endif
-// |VirtualAlloc| will fail if allocation at the hint address is blocked.
-static const bool kHintIsAdvisory = false;
-static std::atomic<int32_t> s_allocPageErrorCode{ERROR_SUCCESS};
-
+#if defined(OS_WIN)
+#include "third_party/base/allocator/partition_allocator/page_allocator_internals_win.h"
+#elif defined(OS_POSIX) || defined(OS_FUCHSIA)
+#include "third_party/base/allocator/partition_allocator/page_allocator_internals_posix.h"
#else
-#error Unknown OS
-#endif // defined(OS_POSIX)
+#error Platform not supported.
+#endif
namespace pdfium {
namespace base {
-// This internal function wraps the OS-specific page allocation call:
-// |VirtualAlloc| on Windows, and |mmap| on POSIX.
-static void* SystemAllocPages(
- void* hint,
- size_t length,
- PageAccessibilityConfiguration page_accessibility) {
- DCHECK(!(length & kPageAllocationGranularityOffsetMask));
- DCHECK(!(reinterpret_cast<uintptr_t>(hint) &
- kPageAllocationGranularityOffsetMask));
- void* ret;
-#if defined(OS_WIN)
- DWORD access_flag =
- page_accessibility == PageAccessible ? PAGE_READWRITE : PAGE_NOACCESS;
- ret = VirtualAlloc(hint, length, MEM_RESERVE | MEM_COMMIT, access_flag);
- if (!ret)
- s_allocPageErrorCode = GetLastError();
-#else
- int access_flag = page_accessibility == PageAccessible
- ? (PROT_READ | PROT_WRITE)
- : PROT_NONE;
- ret = mmap(hint, length, access_flag, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
- if (ret == MAP_FAILED) {
- s_allocPageErrorCode = errno;
- ret = 0;
+namespace {
+
+// We may reserve/release address space on different threads.
+subtle::SpinLock* GetReserveLock() {
+ static subtle::SpinLock* s_reserveLock = nullptr;
+ if (!s_reserveLock)
+ s_reserveLock = new subtle::SpinLock();
+ return s_reserveLock;
+}
+
+// We only support a single block of reserved address space.
+void* s_reservation_address = nullptr;
+size_t s_reservation_size = 0;
+
+void* AllocPagesIncludingReserved(void* address,
+ size_t length,
+ PageAccessibilityConfiguration accessibility,
+ PageTag page_tag,
+ bool commit) {
+ void* ret =
+ SystemAllocPages(address, length, accessibility, page_tag, commit);
+ if (ret == nullptr) {
+ const bool cant_alloc_length = kHintIsAdvisory || address == nullptr;
+ if (cant_alloc_length) {
+ // The system cannot allocate |length| bytes. Release any reserved address
+ // space and try once more.
+ ReleaseReservation();
+ ret = SystemAllocPages(address, length, accessibility, page_tag, commit);
+ }
}
-#endif
return ret;
}
-// Trims base to given length and alignment. Windows returns null on failure and
-// frees base.
-static void* TrimMapping(void* base,
- size_t base_length,
- size_t trim_length,
- uintptr_t align,
- PageAccessibilityConfiguration page_accessibility) {
- size_t pre_slack = reinterpret_cast<uintptr_t>(base) & (align - 1);
- if (pre_slack)
- pre_slack = align - pre_slack;
+// Trims |base| to given |trim_length| and |alignment|.
+//
+// On failure, on Windows, this function returns nullptr and frees |base|.
+void* TrimMapping(void* base,
+ size_t base_length,
+ size_t trim_length,
+ uintptr_t alignment,
+ PageAccessibilityConfiguration accessibility,
+ bool commit) {
+ size_t pre_slack = reinterpret_cast<uintptr_t>(base) & (alignment - 1);
+ if (pre_slack) {
+ pre_slack = alignment - pre_slack;
+ }
size_t post_slack = base_length - pre_slack - trim_length;
DCHECK(base_length >= trim_length || pre_slack || post_slack);
DCHECK(pre_slack < base_length);
DCHECK(post_slack < base_length);
- void* ret = base;
+ return TrimMappingInternal(base, base_length, trim_length, accessibility,
+ commit, pre_slack, post_slack);
+}
-#if defined(OS_POSIX) // On POSIX we can resize the allocation run.
- (void)page_accessibility;
- if (pre_slack) {
- int res = munmap(base, pre_slack);
- CHECK(!res);
- ret = reinterpret_cast<char*>(base) + pre_slack;
- }
- if (post_slack) {
- int res = munmap(reinterpret_cast<char*>(ret) + trim_length, post_slack);
- CHECK(!res);
- }
-#else // On Windows we can't resize the allocation run.
- if (pre_slack || post_slack) {
- ret = reinterpret_cast<char*>(base) + pre_slack;
- FreePages(base, base_length);
- ret = SystemAllocPages(ret, trim_length, page_accessibility);
- }
-#endif
+} // namespace
- return ret;
+void* SystemAllocPages(void* hint,
+ size_t length,
+ PageAccessibilityConfiguration accessibility,
+ PageTag page_tag,
+ bool commit) {
+ DCHECK(!(length & kPageAllocationGranularityOffsetMask));
+ DCHECK(!(reinterpret_cast<uintptr_t>(hint) &
+ kPageAllocationGranularityOffsetMask));
+ DCHECK(commit || accessibility == PageInaccessible);
+ return SystemAllocPagesInternal(hint, length, accessibility, page_tag,
+ commit);
}
void* AllocPages(void* address,
size_t length,
size_t align,
- PageAccessibilityConfiguration page_accessibility) {
+ PageAccessibilityConfiguration accessibility,
+ PageTag page_tag,
+ bool commit) {
DCHECK(length >= kPageAllocationGranularity);
DCHECK(!(length & kPageAllocationGranularityOffsetMask));
DCHECK(align >= kPageAllocationGranularity);
- DCHECK(!(align & kPageAllocationGranularityOffsetMask));
+ // Alignment must be power of 2 for masking math to work.
+ DCHECK_EQ(align & (align - 1), 0UL);
DCHECK(!(reinterpret_cast<uintptr_t>(address) &
kPageAllocationGranularityOffsetMask));
uintptr_t align_offset_mask = align - 1;
uintptr_t align_base_mask = ~align_offset_mask;
DCHECK(!(reinterpret_cast<uintptr_t>(address) & align_offset_mask));
+#if defined(OS_LINUX) && defined(ARCH_CPU_64_BITS)
+ // On 64 bit Linux, we may need to adjust the address space limit for
+ // guarded allocations.
+ if (length >= kMinimumGuardedMemorySize) {
+ CHECK(PageInaccessible == accessibility);
+ CHECK(!commit);
+ if (!AdjustAddressSpaceLimit(base::checked_cast<int64_t>(length))) {
+ // Fall through. Try the allocation, since we may have a reserve.
+ }
+ }
+#endif
+
// If the client passed null as the address, choose a good one.
- if (!address) {
+ if (address == nullptr) {
address = GetRandomPageBase();
address = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(address) &
align_base_mask);
}
// First try to force an exact-size, aligned allocation from our random base.
- for (int count = 0; count < 3; ++count) {
- void* ret = SystemAllocPages(address, length, page_accessibility);
- if (kHintIsAdvisory || ret) {
+#if defined(ARCH_CPU_32_BITS)
+ // On 32 bit systems, first try one random aligned address, and then try an
+ // aligned address derived from the value of |ret|.
+ constexpr int kExactSizeTries = 2;
+#else
+ // On 64 bit systems, try 3 random aligned addresses.
+ constexpr int kExactSizeTries = 3;
+#endif
+
+ for (int i = 0; i < kExactSizeTries; ++i) {
+ void* ret = AllocPagesIncludingReserved(address, length, accessibility,
+ page_tag, commit);
+ if (ret != nullptr) {
// If the alignment is to our liking, we're done.
if (!(reinterpret_cast<uintptr_t>(ret) & align_offset_mask))
return ret;
+ // Free the memory and try again.
FreePages(ret, length);
-#if defined(ARCH_CPU_32_BITS)
- address = reinterpret_cast<void*>(
- (reinterpret_cast<uintptr_t>(ret) + align) & align_base_mask);
-#endif
- } else if (!address) { // We know we're OOM when an unhinted allocation
- // fails.
- return nullptr;
} else {
-#if defined(ARCH_CPU_32_BITS)
- address = reinterpret_cast<char*>(address) + align;
-#endif
+ // |ret| is null; if this try was unhinted, we're OOM.
+ if (kHintIsAdvisory || address == nullptr)
+ return nullptr;
}
-#if !defined(ARCH_CPU_32_BITS)
+#if defined(ARCH_CPU_32_BITS)
+ // For small address spaces, try the first aligned address >= |ret|. Note
+ // |ret| may be null, in which case |address| becomes null.
+ address = reinterpret_cast<void*>(
+ (reinterpret_cast<uintptr_t>(ret) + align_offset_mask) &
+ align_base_mask);
+#else // defined(ARCH_CPU_64_BITS)
// Keep trying random addresses on systems that have a large address space.
address = GetRandomPageBase();
address = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(address) &
@@ -162,21 +174,21 @@
#endif
}
- // Map a larger allocation so we can force alignment, but continue randomizing
- // only on 64-bit POSIX.
+ // Make a larger allocation so we can force alignment.
size_t try_length = length + (align - kPageAllocationGranularity);
CHECK(try_length >= length);
void* ret;
do {
- // Don't continue to burn cycles on mandatory hints (Windows).
+ // Continue randomizing only on POSIX.
address = kHintIsAdvisory ? GetRandomPageBase() : nullptr;
- ret = SystemAllocPages(address, try_length, page_accessibility);
+ ret = AllocPagesIncludingReserved(address, try_length, accessibility,
+ page_tag, commit);
// The retries are for Windows, where a race can steal our mapping on
// resize.
- } while (ret &&
- (ret = TrimMapping(ret, try_length, length, align,
- page_accessibility)) == nullptr);
+ } while (ret != nullptr &&
+ (ret = TrimMapping(ret, try_length, length, align, accessibility,
+ commit)) == nullptr);
return ret;
}
@@ -185,92 +197,60 @@
DCHECK(!(reinterpret_cast<uintptr_t>(address) &
kPageAllocationGranularityOffsetMask));
DCHECK(!(length & kPageAllocationGranularityOffsetMask));
-#if defined(OS_POSIX)
- int ret = munmap(address, length);
- CHECK(!ret);
-#else
- BOOL ret = VirtualFree(address, 0, MEM_RELEASE);
- CHECK(ret);
-#endif
+ FreePagesInternal(address, length);
}
-void SetSystemPagesInaccessible(void* address, size_t length) {
+bool SetSystemPagesAccess(void* address,
+ size_t length,
+ PageAccessibilityConfiguration accessibility) {
DCHECK(!(length & kSystemPageOffsetMask));
-#if defined(OS_POSIX)
- int ret = mprotect(address, length, PROT_NONE);
- CHECK(!ret);
-#else
- BOOL ret = VirtualFree(address, length, MEM_DECOMMIT);
- CHECK(ret);
-#endif
-}
-
-bool SetSystemPagesAccessible(void* address, size_t length) {
- DCHECK(!(length & kSystemPageOffsetMask));
-#if defined(OS_POSIX)
- return !mprotect(address, length, PROT_READ | PROT_WRITE);
-#else
- return !!VirtualAlloc(address, length, MEM_COMMIT, PAGE_READWRITE);
-#endif
+ return SetSystemPagesAccessInternal(address, length, accessibility);
}
void DecommitSystemPages(void* address, size_t length) {
- DCHECK(!(length & kSystemPageOffsetMask));
-#if defined(OS_POSIX)
- int ret = madvise(address, length, MADV_FREE);
- if (ret != 0 && errno == EINVAL) {
- // MADV_FREE only works on Linux 4.5+ . If request failed,
- // retry with older MADV_DONTNEED . Note that MADV_FREE
- // being defined at compile time doesn't imply runtime support.
- ret = madvise(address, length, MADV_DONTNEED);
- }
- CHECK(!ret);
-#else
- SetSystemPagesInaccessible(address, length);
-#endif
+ DCHECK_EQ(0UL, length & kSystemPageOffsetMask);
+ DecommitSystemPagesInternal(address, length);
}
-void RecommitSystemPages(void* address, size_t length) {
- DCHECK(!(length & kSystemPageOffsetMask));
-#if defined(OS_POSIX)
- (void)address;
-#else
- CHECK(SetSystemPagesAccessible(address, length));
-#endif
+bool RecommitSystemPages(void* address,
+ size_t length,
+ PageAccessibilityConfiguration accessibility) {
+ DCHECK_EQ(0UL, length & kSystemPageOffsetMask);
+ DCHECK(PageInaccessible != accessibility);
+ return RecommitSystemPagesInternal(address, length, accessibility);
}
void DiscardSystemPages(void* address, size_t length) {
- DCHECK(!(length & kSystemPageOffsetMask));
-#if defined(OS_POSIX)
- // On POSIX, the implementation detail is that discard and decommit are the
- // same, and lead to pages that are returned to the system immediately and
- // get replaced with zeroed pages when touched. So we just call
- // DecommitSystemPages() here to avoid code duplication.
- DecommitSystemPages(address, length);
-#else
- // On Windows discarded pages are not returned to the system immediately and
- // not guaranteed to be zeroed when returned to the application.
- using DiscardVirtualMemoryFunction =
- DWORD(WINAPI*)(PVOID virtualAddress, SIZE_T size);
- static DiscardVirtualMemoryFunction discard_virtual_memory =
- reinterpret_cast<DiscardVirtualMemoryFunction>(-1);
- if (discard_virtual_memory ==
- reinterpret_cast<DiscardVirtualMemoryFunction>(-1))
- discard_virtual_memory =
- reinterpret_cast<DiscardVirtualMemoryFunction>(GetProcAddress(
- GetModuleHandle(L"Kernel32.dll"), "DiscardVirtualMemory"));
- // Use DiscardVirtualMemory when available because it releases faster than
- // MEM_RESET.
- DWORD ret = 1;
- if (discard_virtual_memory)
- ret = discard_virtual_memory(address, length);
- // DiscardVirtualMemory is buggy in Win10 SP0, so fall back to MEM_RESET on
- // failure.
- if (ret) {
- void* ret = VirtualAlloc(address, length, MEM_RESET, PAGE_READWRITE);
- CHECK(ret);
+ DCHECK_EQ(0UL, length & kSystemPageOffsetMask);
+ DiscardSystemPagesInternal(address, length);
+}
+
+bool ReserveAddressSpace(size_t size) {
+ // To avoid deadlock, call only SystemAllocPages.
+ subtle::SpinLock::Guard guard(*GetReserveLock());
+ if (s_reservation_address == nullptr) {
+ void* mem = SystemAllocPages(nullptr, size, PageInaccessible,
+ PageTag::kChromium, false);
+ if (mem != nullptr) {
+ // We guarantee this alignment when reserving address space.
+ DCHECK(!(reinterpret_cast<uintptr_t>(mem) &
+ kPageAllocationGranularityOffsetMask));
+ s_reservation_address = mem;
+ s_reservation_size = size;
+ return true;
+ }
}
-#endif
+ return false;
+}
+
+void ReleaseReservation() {
+ // To avoid deadlock, call only FreePages.
+ subtle::SpinLock::Guard guard(*GetReserveLock());
+ if (s_reservation_address != nullptr) {
+ FreePages(s_reservation_address, s_reservation_size);
+ s_reservation_address = nullptr;
+ s_reservation_size = 0;
+ }
}
uint32_t GetAllocPageErrorCode() {
diff --git a/third_party/base/allocator/partition_allocator/page_allocator.h b/third_party/base/allocator/partition_allocator/page_allocator.h
index bf9c098..64be33c 100644
--- a/third_party/base/allocator/partition_allocator/page_allocator.h
+++ b/third_party/base/allocator/partition_allocator/page_allocator.h
@@ -2,132 +2,180 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-#ifndef BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_H
-#define BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_H
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_H_
#include <stdint.h>
#include <cstddef>
#include "build/build_config.h"
+#include "third_party/base/allocator/partition_allocator/page_allocator_constants.h"
#include "third_party/base/base_export.h"
#include "third_party/base/compiler_specific.h"
namespace pdfium {
namespace base {
-#if defined(OS_WIN)
-static const size_t kPageAllocationGranularityShift = 16; // 64KB
-#elif defined(_MIPS_ARCH_LOONGSON)
-static const size_t kPageAllocationGranularityShift = 14; // 16KB
-#else
-static const size_t kPageAllocationGranularityShift = 12; // 4KB
-#endif
-static const size_t kPageAllocationGranularity =
- 1 << kPageAllocationGranularityShift;
-static const size_t kPageAllocationGranularityOffsetMask =
- kPageAllocationGranularity - 1;
-static const size_t kPageAllocationGranularityBaseMask =
- ~kPageAllocationGranularityOffsetMask;
-
-// All Blink-supported systems have 4096 sized system pages and can handle
-// permissions and commit / decommit at this granularity.
-// Loongson have 16384 sized system pages.
-#if defined(_MIPS_ARCH_LOONGSON)
-static const size_t kSystemPageSize = 16384;
-#else
-static const size_t kSystemPageSize = 4096;
-#endif
-static const size_t kSystemPageOffsetMask = kSystemPageSize - 1;
-static const size_t kSystemPageBaseMask = ~kSystemPageOffsetMask;
-
enum PageAccessibilityConfiguration {
- PageAccessible,
PageInaccessible,
+ PageRead,
+ PageReadWrite,
+ PageReadExecute,
+ // This flag is deprecated and will go away soon.
+ // TODO(bbudge) Remove this as soon as V8 doesn't need RWX pages.
+ PageReadWriteExecute,
+};
+
+// Mac OSX supports tagged memory regions, to help in debugging.
+enum class PageTag {
+ kFirst = 240, // Minimum tag value.
+ kChromium = 254, // Chromium page, including off-heap V8 ArrayBuffers.
+ kV8 = 255, // V8 heap pages.
+ kLast = kV8 // Maximum tag value.
};
// Allocate one or more pages.
-// The requested address is just a hint; the actual address returned may
-// differ. The returned address will be aligned at least to align bytes.
-// len is in bytes, and must be a multiple of kPageAllocationGranularity.
-// align is in bytes, and must be a power-of-two multiple of
-// kPageAllocationGranularity.
-// If addr is null, then a suitable and randomized address will be chosen
+//
+// The requested |address| is just a hint; the actual address returned may
+// differ. The returned address will be aligned at least to |align| bytes.
+// |length| is in bytes, and must be a multiple of |kPageAllocationGranularity|.
+// |align| is in bytes, and must be a power-of-two multiple of
+// |kPageAllocationGranularity|.
+//
+// If |address| is null, then a suitable and randomized address will be chosen
// automatically.
-// PageAccessibilityConfiguration controls the permission of the
-// allocated pages.
+//
+// |page_accessibility| controls the permission of the allocated pages.
+//
// This call will return null if the allocation cannot be satisfied.
BASE_EXPORT void* AllocPages(void* address,
- size_t len,
+ size_t length,
size_t align,
- PageAccessibilityConfiguration);
+ PageAccessibilityConfiguration page_accessibility,
+ PageTag tag = PageTag::kChromium,
+ bool commit = true);
-// Free one or more pages.
-// addr and len must match a previous call to allocPages().
+// Free one or more pages starting at |address| and continuing for |length|
+// bytes.
+//
+// |address| and |length| must match a previous call to |AllocPages|. Therefore,
+// |address| must be aligned to |kPageAllocationGranularity| bytes, and |length|
+// must be a multiple of |kPageAllocationGranularity|.
BASE_EXPORT void FreePages(void* address, size_t length);
-// Mark one or more system pages as being inaccessible.
-// Subsequently accessing any address in the range will fault, and the
-// addresses will not be re-used by future allocations.
-// len must be a multiple of kSystemPageSize bytes.
-BASE_EXPORT void SetSystemPagesInaccessible(void* address, size_t length);
+// Mark one or more system pages, starting at |address| with the given
+// |page_accessibility|. |length| must be a multiple of |kSystemPageSize| bytes.
+//
+// Returns true if the permission change succeeded. In most cases you must
+// |CHECK| the result.
+BASE_EXPORT WARN_UNUSED_RESULT bool SetSystemPagesAccess(
+ void* address,
+ size_t length,
+ PageAccessibilityConfiguration page_accessibility);
-// Mark one or more system pages as being accessible.
-// The pages will be readable and writeable.
-// len must be a multiple of kSystemPageSize bytes.
-// The result bool value indicates whether the permission
-// change succeeded or not. You must check the result
-// (in most cases you need to CHECK that it is true).
-BASE_EXPORT WARN_UNUSED_RESULT bool SetSystemPagesAccessible(void* address,
- size_t length);
-
-// Decommit one or more system pages. Decommitted means that the physical memory
-// is released to the system, but the virtual address space remains reserved.
-// System pages are re-committed by calling recommitSystemPages(). Touching
-// a decommitted page _may_ fault.
-// Clients should not make any assumptions about the contents of decommitted
-// system pages, before or after they write to the page. The only guarantee
-// provided is that the contents of the system page will be deterministic again
-// after recommitting and writing to it. In particlar note that system pages are
-// not guaranteed to be zero-filled upon re-commit. len must be a multiple of
-// kSystemPageSize bytes.
+// Decommit one or more system pages starting at |address| and continuing for
+// |length| bytes. |length| must be a multiple of |kSystemPageSize|.
+//
+// Decommitted means that physical resources (RAM or swap) backing the allocated
+// virtual address range are released back to the system, but the address space
+// is still allocated to the process (possibly using up page table entries or
+// other accounting resources). Any access to a decommitted region of memory
+// is an error and will generate a fault.
+//
+// This operation is not atomic on all platforms.
+//
+// Note: "Committed memory" is a Windows Memory Subsystem concept that ensures
+// processes will not fault when touching a committed memory region. There is
+// no analogue in the POSIX memory API where virtual memory pages are
+// best-effort allocated resources on the first touch. To create a
+// platform-agnostic abstraction, this API simulates the Windows "decommit"
+// state by both discarding the region (allowing the OS to avoid swap
+// operations) and changing the page protections so accesses fault.
+//
+// TODO(ajwong): This currently does not change page protections on POSIX
+// systems due to a perf regression. Tracked at http://crbug.com/766882.
BASE_EXPORT void DecommitSystemPages(void* address, size_t length);
-// Recommit one or more system pages. Decommitted system pages must be
-// recommitted before they are read are written again.
-// Note that this operation may be a no-op on some platforms.
-// len must be a multiple of kSystemPageSize bytes.
-BASE_EXPORT void RecommitSystemPages(void* address, size_t length);
+// Recommit one or more system pages, starting at |address| and continuing for
+// |length| bytes with the given |page_accessibility|. |length| must be a
+// multiple of |kSystemPageSize|.
+//
+// Decommitted system pages must be recommitted with their original permissions
+// before they are used again.
+//
+// Returns true if the recommit change succeeded. In most cases you must |CHECK|
+// the result.
+BASE_EXPORT WARN_UNUSED_RESULT bool RecommitSystemPages(
+ void* address,
+ size_t length,
+ PageAccessibilityConfiguration page_accessibility);
-// Discard one or more system pages. Discarding is a hint to the system that
-// the page is no longer required. The hint may:
-// - Do nothing.
-// - Discard the page immediately, freeing up physical pages.
-// - Discard the page at some time in the future in response to memory pressure.
-// Only committed pages should be discarded. Discarding a page does not
-// decommit it, and it is valid to discard an already-discarded page.
-// A read or write to a discarded page will not fault.
-// Reading from a discarded page may return the original page content, or a
-// page full of zeroes.
+// Discard one or more system pages starting at |address| and continuing for
+// |length| bytes. |length| must be a multiple of |kSystemPageSize|.
+//
+// Discarding is a hint to the system that the page is no longer required. The
+// hint may:
+// - Do nothing.
+// - Discard the page immediately, freeing up physical pages.
+// - Discard the page at some time in the future in response to memory
+// pressure.
+//
+// Only committed pages should be discarded. Discarding a page does not decommit
+// it, and it is valid to discard an already-discarded page. A read or write to
+// a discarded page will not fault.
+//
+// Reading from a discarded page may return the original page content, or a page
+// full of zeroes.
+//
// Writing to a discarded page is the only guaranteed way to tell the system
// that the page is required again. Once written to, the content of the page is
// guaranteed stable once more. After being written to, the page content may be
// based on the original page content, or a page of zeroes.
-// len must be a multiple of kSystemPageSize bytes.
BASE_EXPORT void DiscardSystemPages(void* address, size_t length);
-ALWAYS_INLINE uintptr_t RoundUpToSystemPage(uintptr_t address) {
+// Rounds up |address| to the next multiple of |kSystemPageSize|. Returns
+// 0 for an |address| of 0.
+constexpr ALWAYS_INLINE uintptr_t RoundUpToSystemPage(uintptr_t address) {
return (address + kSystemPageOffsetMask) & kSystemPageBaseMask;
}
-ALWAYS_INLINE uintptr_t RoundDownToSystemPage(uintptr_t address) {
+// Rounds down |address| to the previous multiple of |kSystemPageSize|. Returns
+// 0 for an |address| of 0.
+constexpr ALWAYS_INLINE uintptr_t RoundDownToSystemPage(uintptr_t address) {
return address & kSystemPageBaseMask;
}
-// Returns errno (or GetLastError code) when mmap (or VirtualAlloc) fails.
+// Rounds up |address| to the next multiple of |kPageAllocationGranularity|.
+// Returns 0 for an |address| of 0.
+constexpr ALWAYS_INLINE uintptr_t
+RoundUpToPageAllocationGranularity(uintptr_t address) {
+ return (address + kPageAllocationGranularityOffsetMask) &
+ kPageAllocationGranularityBaseMask;
+}
+
+// Rounds down |address| to the previous multiple of
+// |kPageAllocationGranularity|. Returns 0 for an |address| of 0.
+constexpr ALWAYS_INLINE uintptr_t
+RoundDownToPageAllocationGranularity(uintptr_t address) {
+ return address & kPageAllocationGranularityBaseMask;
+}
+
+// Reserves (at least) |size| bytes of address space, aligned to
+// |kPageAllocationGranularity|. This can be called early on to make it more
+// likely that large allocations will succeed. Returns true if the reservation
+// succeeded, false if the reservation failed or a reservation was already made.
+BASE_EXPORT bool ReserveAddressSpace(size_t size);
+
+// Releases any reserved address space. |AllocPages| calls this automatically on
+// an allocation failure. External allocators may also call this on failure.
+BASE_EXPORT void ReleaseReservation();
+
+// Returns |errno| (POSIX) or the result of |GetLastError| (Windows) when |mmap|
+// (POSIX) or |VirtualAlloc| (Windows) fails.
BASE_EXPORT uint32_t GetAllocPageErrorCode();
} // namespace base
} // namespace pdfium
-#endif // BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_H
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_H_
diff --git a/third_party/base/allocator/partition_allocator/page_allocator_constants.h b/third_party/base/allocator/partition_allocator/page_allocator_constants.h
new file mode 100644
index 0000000..945273b
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/page_allocator_constants.h
@@ -0,0 +1,44 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_CONSTANTS_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_CONSTANTS_H_
+
+#include <stddef.h>
+
+#include "build/build_config.h"
+
+namespace pdfium {
+namespace base {
+#if defined(OS_WIN)
+static constexpr size_t kPageAllocationGranularityShift = 16; // 64KB
+#elif defined(_MIPS_ARCH_LOONGSON)
+static constexpr size_t kPageAllocationGranularityShift = 14; // 16KB
+#else
+static constexpr size_t kPageAllocationGranularityShift = 12; // 4KB
+#endif
+static constexpr size_t kPageAllocationGranularity =
+ 1 << kPageAllocationGranularityShift;
+static constexpr size_t kPageAllocationGranularityOffsetMask =
+ kPageAllocationGranularity - 1;
+static constexpr size_t kPageAllocationGranularityBaseMask =
+ ~kPageAllocationGranularityOffsetMask;
+
+#if defined(_MIPS_ARCH_LOONGSON)
+static constexpr size_t kSystemPageSize = 16384;
+#else
+static constexpr size_t kSystemPageSize = 4096;
+#endif
+static constexpr size_t kSystemPageOffsetMask = kSystemPageSize - 1;
+static_assert((kSystemPageSize & (kSystemPageSize - 1)) == 0,
+ "kSystemPageSize must be power of 2");
+static constexpr size_t kSystemPageBaseMask = ~kSystemPageOffsetMask;
+
+static constexpr size_t kPageMetadataShift = 5; // 32 bytes per partition page.
+static constexpr size_t kPageMetadataSize = 1 << kPageMetadataShift;
+
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_CONSTANTS_H_
diff --git a/third_party/base/allocator/partition_allocator/page_allocator_internal.h b/third_party/base/allocator/partition_allocator/page_allocator_internal.h
new file mode 100644
index 0000000..2284314
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/page_allocator_internal.h
@@ -0,0 +1,20 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_INTERNAL_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_INTERNAL_H_
+
+namespace pdfium {
+namespace base {
+
+void* SystemAllocPages(void* hint,
+ size_t length,
+ PageAccessibilityConfiguration accessibility,
+ PageTag page_tag,
+ bool commit);
+
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_INTERNAL_H_
diff --git a/third_party/base/allocator/partition_allocator/page_allocator_internals_posix.h b/third_party/base/allocator/partition_allocator/page_allocator_internals_posix.h
new file mode 100644
index 0000000..0622222
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/page_allocator_internals_posix.h
@@ -0,0 +1,187 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_INTERNALS_POSIX_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_INTERNALS_POSIX_H_
+
+#include <errno.h>
+#include <sys/mman.h>
+
+#include "build/build_config.h"
+
+#if defined(OS_MACOSX)
+#include <mach/mach.h>
+#endif
+#if defined(OS_LINUX)
+#include <sys/resource.h>
+
+#include <algorithm>
+#endif
+
+#ifndef MAP_ANONYMOUS
+#define MAP_ANONYMOUS MAP_ANON
+#endif
+
+namespace pdfium {
+namespace base {
+
+// |mmap| uses a nearby address if the hint address is blocked.
+constexpr bool kHintIsAdvisory = true;
+std::atomic<int32_t> s_allocPageErrorCode{0};
+
+int GetAccessFlags(PageAccessibilityConfiguration accessibility) {
+ switch (accessibility) {
+ case PageRead:
+ return PROT_READ;
+ case PageReadWrite:
+ return PROT_READ | PROT_WRITE;
+ case PageReadExecute:
+ return PROT_READ | PROT_EXEC;
+ case PageReadWriteExecute:
+ return PROT_READ | PROT_WRITE | PROT_EXEC;
+ default:
+ NOTREACHED();
+ FALLTHROUGH;
+ case PageInaccessible:
+ return PROT_NONE;
+ }
+}
+
+#if defined(OS_LINUX) && defined(ARCH_CPU_64_BITS)
+
+// Multiple guarded memory regions may exceed the process address space limit.
+// This function will raise or lower the limit by |amount|.
+bool AdjustAddressSpaceLimit(int64_t amount) {
+ struct rlimit old_rlimit;
+ if (getrlimit(RLIMIT_AS, &old_rlimit))
+ return false;
+ const rlim_t new_limit =
+ CheckAdd(old_rlimit.rlim_cur, amount).ValueOrDefault(old_rlimit.rlim_max);
+ const struct rlimit new_rlimit = {std::min(new_limit, old_rlimit.rlim_max),
+ old_rlimit.rlim_max};
+ // setrlimit will fail if limit > old_rlimit.rlim_max.
+ return setrlimit(RLIMIT_AS, &new_rlimit) == 0;
+}
+
+// Current WASM guarded memory regions have 8 GiB of address space. There are
+// schemes that reduce that to 4 GiB.
+constexpr size_t kMinimumGuardedMemorySize = 1ULL << 32; // 4 GiB
+
+#endif // defined(OS_LINUX) && defined(ARCH_CPU_64_BITS)
+
+void* SystemAllocPagesInternal(void* hint,
+ size_t length,
+ PageAccessibilityConfiguration accessibility,
+ PageTag page_tag,
+ bool commit) {
+#if defined(OS_MACOSX)
+ // Use a custom tag to make it easier to distinguish Partition Alloc regions
+ // in vmmap(1). Tags between 240-255 are supported.
+ DCHECK(PageTag::kFirst <= page_tag);
+ DCHECK(PageTag::kLast >= page_tag);
+ int fd = VM_MAKE_TAG(static_cast<int>(page_tag));
+#else
+ int fd = -1;
+#endif
+
+ int access_flag = GetAccessFlags(accessibility);
+ void* ret =
+ mmap(hint, length, access_flag, MAP_ANONYMOUS | MAP_PRIVATE, fd, 0);
+ if (ret == MAP_FAILED) {
+ s_allocPageErrorCode = errno;
+ ret = nullptr;
+ }
+ return ret;
+}
+
+void* TrimMappingInternal(void* base,
+ size_t base_length,
+ size_t trim_length,
+ PageAccessibilityConfiguration accessibility,
+ bool commit,
+ size_t pre_slack,
+ size_t post_slack) {
+ void* ret = base;
+ // We can resize the allocation run. Release unneeded memory before and after
+ // the aligned range.
+ if (pre_slack) {
+ int res = munmap(base, pre_slack);
+ CHECK(!res);
+ ret = reinterpret_cast<char*>(base) + pre_slack;
+ }
+ if (post_slack) {
+ int res = munmap(reinterpret_cast<char*>(ret) + trim_length, post_slack);
+ CHECK(!res);
+ }
+ return ret;
+}
+
+bool SetSystemPagesAccessInternal(
+ void* address,
+ size_t length,
+ PageAccessibilityConfiguration accessibility) {
+ return 0 == mprotect(address, length, GetAccessFlags(accessibility));
+}
+
+void FreePagesInternal(void* address, size_t length) {
+ CHECK(!munmap(address, length));
+
+#if defined(OS_LINUX) && defined(ARCH_CPU_64_BITS)
+ // Restore the address space limit.
+ if (length >= kMinimumGuardedMemorySize) {
+ CHECK(AdjustAddressSpaceLimit(-base::checked_cast<int64_t>(length)));
+ }
+#endif
+}
+
+void DecommitSystemPagesInternal(void* address, size_t length) {
+ // In POSIX, there is no decommit concept. Discarding is an effective way of
+ // implementing the Windows semantics where the OS is allowed to not swap the
+ // pages in the region.
+ //
+ // TODO(ajwong): Also explore setting PageInaccessible to make the protection
+ // semantics consistent between Windows and POSIX. This might have a perf cost
+ // though as both decommit and recommit would incur an extra syscall.
+ // http://crbug.com/766882
+ DiscardSystemPages(address, length);
+}
+
+bool RecommitSystemPagesInternal(void* address,
+ size_t length,
+ PageAccessibilityConfiguration accessibility) {
+#if defined(OS_MACOSX)
+ // On macOS, to update accounting, we need to make another syscall. For more
+ // details, see https://crbug.com/823915.
+ madvise(address, length, MADV_FREE_REUSE);
+#endif
+
+ // On POSIX systems, the caller need simply read the memory to recommit it.
+ // This has the correct behavior because the API requires the permissions to
+ // be the same as before decommitting and all configurations can read.
+ return true;
+}
+
+void DiscardSystemPagesInternal(void* address, size_t length) {
+#if defined(OS_MACOSX)
+ int ret = madvise(address, length, MADV_FREE_REUSABLE);
+ if (ret) {
+ // MADV_FREE_REUSABLE sometimes fails, so fall back to MADV_DONTNEED.
+ ret = madvise(address, length, MADV_DONTNEED);
+ }
+ CHECK(0 == ret);
+#else
+ // We have experimented with other flags, but with suboptimal results.
+ //
+ // MADV_FREE (Linux): Makes our memory measurements less predictable;
+ // performance benefits unclear.
+ //
+ // Therefore, we just do the simple thing: MADV_DONTNEED.
+ CHECK(!madvise(address, length, MADV_DONTNEED));
+#endif
+}
+
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_INTERNALS_POSIX_H_
diff --git a/third_party/base/allocator/partition_allocator/page_allocator_internals_win.h b/third_party/base/allocator/partition_allocator/page_allocator_internals_win.h
new file mode 100644
index 0000000..57a11c5
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/page_allocator_internals_win.h
@@ -0,0 +1,123 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_INTERNALS_WIN_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_INTERNALS_WIN_H_
+
+#include "third_party/base/allocator/partition_allocator/page_allocator_internal.h"
+
+namespace pdfium {
+namespace base {
+
+// |VirtualAlloc| will fail if allocation at the hint address is blocked.
+constexpr bool kHintIsAdvisory = false;
+std::atomic<int32_t> s_allocPageErrorCode{ERROR_SUCCESS};
+
+int GetAccessFlags(PageAccessibilityConfiguration accessibility) {
+ switch (accessibility) {
+ case PageRead:
+ return PAGE_READONLY;
+ case PageReadWrite:
+ return PAGE_READWRITE;
+ case PageReadExecute:
+ return PAGE_EXECUTE_READ;
+ case PageReadWriteExecute:
+ return PAGE_EXECUTE_READWRITE;
+ default:
+ NOTREACHED();
+ FALLTHROUGH;
+ case PageInaccessible:
+ return PAGE_NOACCESS;
+ }
+}
+
+void* SystemAllocPagesInternal(void* hint,
+ size_t length,
+ PageAccessibilityConfiguration accessibility,
+ PageTag page_tag,
+ bool commit) {
+ DWORD access_flag = GetAccessFlags(accessibility);
+ const DWORD type_flags = commit ? (MEM_RESERVE | MEM_COMMIT) : MEM_RESERVE;
+ void* ret = VirtualAlloc(hint, length, type_flags, access_flag);
+ if (ret == nullptr) {
+ s_allocPageErrorCode = GetLastError();
+ }
+ return ret;
+}
+
+void* TrimMappingInternal(void* base,
+ size_t base_length,
+ size_t trim_length,
+ PageAccessibilityConfiguration accessibility,
+ bool commit,
+ size_t pre_slack,
+ size_t post_slack) {
+ void* ret = base;
+ if (pre_slack || post_slack) {
+ // We cannot resize the allocation run. Free it and retry at the aligned
+ // address within the freed range.
+ ret = reinterpret_cast<char*>(base) + pre_slack;
+ FreePages(base, base_length);
+ ret = SystemAllocPages(ret, trim_length, accessibility, PageTag::kChromium,
+ commit);
+ }
+ return ret;
+}
+
+bool SetSystemPagesAccessInternal(
+ void* address,
+ size_t length,
+ PageAccessibilityConfiguration accessibility) {
+ if (accessibility == PageInaccessible) {
+ return VirtualFree(address, length, MEM_DECOMMIT) != 0;
+ } else {
+ return nullptr != VirtualAlloc(address, length, MEM_COMMIT,
+ GetAccessFlags(accessibility));
+ }
+}
+
+void FreePagesInternal(void* address, size_t length) {
+ CHECK(VirtualFree(address, 0, MEM_RELEASE));
+}
+
+void DecommitSystemPagesInternal(void* address, size_t length) {
+ CHECK(SetSystemPagesAccess(address, length, PageInaccessible));
+}
+
+bool RecommitSystemPagesInternal(void* address,
+ size_t length,
+ PageAccessibilityConfiguration accessibility) {
+ return SetSystemPagesAccess(address, length, accessibility);
+}
+
+void DiscardSystemPagesInternal(void* address, size_t length) {
+ // On Windows, discarded pages are not returned to the system immediately and
+ // not guaranteed to be zeroed when returned to the application.
+ using DiscardVirtualMemoryFunction =
+ DWORD(WINAPI*)(PVOID virtualAddress, SIZE_T size);
+ static DiscardVirtualMemoryFunction discard_virtual_memory =
+ reinterpret_cast<DiscardVirtualMemoryFunction>(-1);
+ if (discard_virtual_memory ==
+ reinterpret_cast<DiscardVirtualMemoryFunction>(-1))
+ discard_virtual_memory =
+ reinterpret_cast<DiscardVirtualMemoryFunction>(GetProcAddress(
+ GetModuleHandle(L"Kernel32.dll"), "DiscardVirtualMemory"));
+ // Use DiscardVirtualMemory when available because it releases faster than
+ // MEM_RESET.
+ DWORD ret = 1;
+ if (discard_virtual_memory) {
+ ret = discard_virtual_memory(address, length);
+ }
+ // DiscardVirtualMemory is buggy in Win10 SP0, so fall back to MEM_RESET on
+ // failure.
+ if (ret) {
+ void* ptr = VirtualAlloc(address, length, MEM_RESET, PAGE_READWRITE);
+ CHECK(ptr);
+ }
+}
+
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PAGE_ALLOCATOR_INTERNALS_WIN_H_
diff --git a/third_party/base/allocator/partition_allocator/partition_alloc.cc b/third_party/base/allocator/partition_allocator/partition_alloc.cc
index e8aad94..d0d58df 100644
--- a/third_party/base/allocator/partition_allocator/partition_alloc.cc
+++ b/third_party/base/allocator/partition_allocator/partition_alloc.cc
@@ -6,172 +6,113 @@
#include <string.h>
-#include "third_party/base/allocator/partition_allocator/oom.h"
-#include "third_party/base/allocator/partition_allocator/spin_lock.h"
-#include "third_party/base/compiler_specific.h"
+#include <memory>
+#include <type_traits>
-// Two partition pages are used as guard / metadata page so make sure the super
-// page size is bigger.
-static_assert(pdfium::base::kPartitionPageSize * 4 <=
- pdfium::base::kSuperPageSize,
- "ok super page size");
-static_assert(!(pdfium::base::kSuperPageSize %
- pdfium::base::kPartitionPageSize),
- "ok super page multiple");
-// Four system pages gives us room to hack out a still-guard-paged piece
-// of metadata in the middle of a guard partition page.
-static_assert(pdfium::base::kSystemPageSize * 4 <=
- pdfium::base::kPartitionPageSize,
- "ok partition page size");
-static_assert(!(pdfium::base::kPartitionPageSize %
- pdfium::base::kSystemPageSize),
- "ok partition page multiple");
-static_assert(sizeof(pdfium::base::PartitionPage) <=
- pdfium::base::kPageMetadataSize,
- "PartitionPage should not be too big");
-static_assert(sizeof(pdfium::base::PartitionBucket) <=
- pdfium::base::kPageMetadataSize,
- "PartitionBucket should not be too big");
-static_assert(sizeof(pdfium::base::PartitionSuperPageExtentEntry) <=
- pdfium::base::kPageMetadataSize,
- "PartitionSuperPageExtentEntry should not be too big");
-static_assert(pdfium::base::kPageMetadataSize *
- pdfium::base::kNumPartitionPagesPerSuperPage <=
- pdfium::base::kSystemPageSize,
- "page metadata fits in hole");
-// Check that some of our zanier calculations worked out as expected.
-static_assert(pdfium::base::kGenericSmallestBucket == 8,
- "generic smallest bucket");
-static_assert(pdfium::base::kGenericMaxBucketed == 983040,
- "generic max bucketed");
-static_assert(pdfium::base::kMaxSystemPagesPerSlotSpan < (1 << 8),
- "System pages per slot span must be less than 128.");
+#include "third_party/base/allocator/partition_allocator/partition_direct_map_extent.h"
+#include "third_party/base/allocator/partition_allocator/partition_oom.h"
+#include "third_party/base/allocator/partition_allocator/partition_page.h"
+#include "third_party/base/allocator/partition_allocator/spin_lock.h"
namespace pdfium {
namespace base {
-subtle::SpinLock PartitionRootBase::gInitializedLock;
-bool PartitionRootBase::gInitialized = false;
-PartitionPage PartitionRootBase::gSeedPage;
-PartitionBucket PartitionRootBase::gPagedBucket;
-void (*PartitionRootBase::gOomHandlingFunction)() = nullptr;
+// Two partition pages are used as guard / metadata page so make sure the super
+// page size is bigger.
+static_assert(kPartitionPageSize * 4 <= kSuperPageSize, "ok super page size");
+static_assert(!(kSuperPageSize % kPartitionPageSize), "ok super page multiple");
+// Four system pages gives us room to hack out a still-guard-paged piece
+// of metadata in the middle of a guard partition page.
+static_assert(kSystemPageSize * 4 <= kPartitionPageSize,
+ "ok partition page size");
+static_assert(!(kPartitionPageSize % kSystemPageSize),
+ "ok partition page multiple");
+static_assert(sizeof(internal::PartitionPage) <= kPageMetadataSize,
+ "PartitionPage should not be too big");
+static_assert(sizeof(internal::PartitionBucket) <= kPageMetadataSize,
+ "PartitionBucket should not be too big");
+static_assert(sizeof(internal::PartitionSuperPageExtentEntry) <=
+ kPageMetadataSize,
+ "PartitionSuperPageExtentEntry should not be too big");
+static_assert(kPageMetadataSize * kNumPartitionPagesPerSuperPage <=
+ kSystemPageSize,
+ "page metadata fits in hole");
+// Limit to prevent callers accidentally overflowing an int size.
+static_assert(kGenericMaxDirectMapped <=
+ (1UL << 31) + kPageAllocationGranularity,
+ "maximum direct mapped allocation");
+// Check that some of our zanier calculations worked out as expected.
+static_assert(kGenericSmallestBucket == 8, "generic smallest bucket");
+static_assert(kGenericMaxBucketed == 983040, "generic max bucketed");
+static_assert(kMaxSystemPagesPerSlotSpan < (1 << 8),
+ "System pages per slot span must be less than 128.");
+
+internal::PartitionRootBase::PartitionRootBase() = default;
+internal::PartitionRootBase::~PartitionRootBase() = default;
+PartitionRoot::PartitionRoot() = default;
+PartitionRoot::~PartitionRoot() = default;
+PartitionRootGeneric::PartitionRootGeneric() = default;
+PartitionRootGeneric::~PartitionRootGeneric() = default;
+PartitionAllocatorGeneric::PartitionAllocatorGeneric() = default;
+PartitionAllocatorGeneric::~PartitionAllocatorGeneric() = default;
+
+subtle::SpinLock* GetLock() {
+ static subtle::SpinLock* s_initialized_lock = nullptr;
+ if (!s_initialized_lock)
+ s_initialized_lock = new subtle::SpinLock();
+ return s_initialized_lock;
+}
+
+static bool g_initialized = false;
+
+void (*internal::PartitionRootBase::gOomHandlingFunction)() = nullptr;
PartitionAllocHooks::AllocationHook* PartitionAllocHooks::allocation_hook_ =
nullptr;
PartitionAllocHooks::FreeHook* PartitionAllocHooks::free_hook_ = nullptr;
-static uint8_t PartitionBucketNumSystemPages(size_t size) {
- // This works out reasonably for the current bucket sizes of the generic
- // allocator, and the current values of partition page size and constants.
- // Specifically, we have enough room to always pack the slots perfectly into
- // some number of system pages. The only waste is the waste associated with
- // unfaulted pages (i.e. wasted address space).
- // TODO: we end up using a lot of system pages for very small sizes. For
- // example, we'll use 12 system pages for slot size 24. The slot size is
- // so small that the waste would be tiny with just 4, or 1, system pages.
- // Later, we can investigate whether there are anti-fragmentation benefits
- // to using fewer system pages.
- double best_waste_ratio = 1.0f;
- uint16_t best_pages = 0;
- if (size > kMaxSystemPagesPerSlotSpan * kSystemPageSize) {
- DCHECK(!(size % kSystemPageSize));
- best_pages = static_cast<uint16_t>(size / kSystemPageSize);
- CHECK(best_pages < (1 << 8));
- return static_cast<uint8_t>(best_pages);
- }
- DCHECK(size <= kMaxSystemPagesPerSlotSpan * kSystemPageSize);
- for (uint16_t i = kNumSystemPagesPerPartitionPage - 1;
- i <= kMaxSystemPagesPerSlotSpan; ++i) {
- size_t page_size = kSystemPageSize * i;
- size_t num_slots = page_size / size;
- size_t waste = page_size - (num_slots * size);
- // Leaving a page unfaulted is not free; the page will occupy an empty page
- // table entry. Make a simple attempt to account for that.
- size_t num_remainder_pages = i & (kNumSystemPagesPerPartitionPage - 1);
- size_t num_unfaulted_pages =
- num_remainder_pages
- ? (kNumSystemPagesPerPartitionPage - num_remainder_pages)
- : 0;
- waste += sizeof(void*) * num_unfaulted_pages;
- double waste_ratio = (double)waste / (double)page_size;
- if (waste_ratio < best_waste_ratio) {
- best_waste_ratio = waste_ratio;
- best_pages = i;
- }
- }
- DCHECK(best_pages > 0);
- CHECK(best_pages <= kMaxSystemPagesPerSlotSpan);
- return static_cast<uint8_t>(best_pages);
-}
-
-static void PartitionAllocBaseInit(PartitionRootBase* root) {
+static void PartitionAllocBaseInit(internal::PartitionRootBase* root) {
DCHECK(!root->initialized);
{
- subtle::SpinLock::Guard guard(PartitionRootBase::gInitializedLock);
- if (!PartitionRootBase::gInitialized) {
- PartitionRootBase::gInitialized = true;
- // We mark the seed page as free to make sure it is skipped by our
- // logic to find a new active page.
- PartitionRootBase::gPagedBucket.active_pages_head =
- &PartitionRootGeneric::gSeedPage;
+ subtle::SpinLock::Guard guard(*GetLock());
+ if (!g_initialized) {
+ g_initialized = true;
+ // We mark the sentinel bucket/page as free to make sure it is skipped by
+ // our logic to find a new active page.
+ internal::PartitionBucket::get_sentinel_bucket()->active_pages_head =
+ internal::PartitionPage::get_sentinel_page();
}
}
root->initialized = true;
- root->total_size_of_committed_pages = 0;
- root->total_size_of_super_pages = 0;
- root->total_size_of_direct_mapped_pages = 0;
- root->next_super_page = 0;
- root->next_partition_page = 0;
- root->next_partition_page_end = 0;
- root->first_extent = 0;
- root->current_extent = 0;
- root->direct_map_list = 0;
-
- memset(&root->global_empty_page_ring, '\0',
- sizeof(root->global_empty_page_ring));
- root->global_empty_page_ring_index = 0;
// This is a "magic" value so we can test if a root pointer is valid.
root->inverted_self = ~reinterpret_cast<uintptr_t>(root);
}
-static void PartitionBucketInitBase(PartitionBucket* bucket,
- PartitionRootBase* root) {
- bucket->active_pages_head = &PartitionRootGeneric::gSeedPage;
- bucket->empty_pages_head = 0;
- bucket->decommitted_pages_head = 0;
- bucket->num_full_pages = 0;
- bucket->num_system_pages_per_slot_span =
- PartitionBucketNumSystemPages(bucket->slot_size);
-}
-
void PartitionAllocGlobalInit(void (*oom_handling_function)()) {
DCHECK(oom_handling_function);
- PartitionRootBase::gOomHandlingFunction = oom_handling_function;
+ internal::PartitionRootBase::gOomHandlingFunction = oom_handling_function;
}
-void PartitionAllocInit(PartitionRoot* root,
- size_t num_buckets,
- size_t max_allocation) {
- PartitionAllocBaseInit(root);
+void PartitionRoot::Init(size_t num_buckets, size_t max_allocation) {
+ PartitionAllocBaseInit(this);
- root->num_buckets = num_buckets;
- root->max_allocation = max_allocation;
+ this->num_buckets = num_buckets;
+ this->max_allocation = max_allocation;
size_t i;
- for (i = 0; i < root->num_buckets; ++i) {
- PartitionBucket* bucket = &root->buckets()[i];
+ for (i = 0; i < this->num_buckets; ++i) {
+ internal::PartitionBucket* bucket = &this->buckets()[i];
if (!i)
- bucket->slot_size = kAllocationGranularity;
+ bucket->Init(kAllocationGranularity);
else
- bucket->slot_size = i << kBucketShift;
- PartitionBucketInitBase(bucket, root);
+ bucket->Init(i << kBucketShift);
}
}
-void PartitionAllocGenericInit(PartitionRootGeneric* root) {
- subtle::SpinLock::Guard guard(root->lock);
+void PartitionRootGeneric::Init() {
+ subtle::SpinLock::Guard guard(this->lock);
- PartitionAllocBaseInit(root);
+ PartitionAllocBaseInit(this);
// Precalculate some shift and mask constants used in the hot path.
// Example: malloc(41) == 101001 binary.
@@ -187,7 +128,7 @@
order_index_shift = 0;
else
order_index_shift = order - (kGenericNumBucketsPerOrderBits + 1);
- root->order_index_shifts[order] = order_index_shift;
+ this->order_index_shifts[order] = order_index_shift;
size_t sub_order_index_mask;
if (order == kBitsPerSizeT) {
// This avoids invoking undefined behavior for an excessive shift.
@@ -197,7 +138,7 @@
sub_order_index_mask = ((static_cast<size_t>(1) << order) - 1) >>
(kGenericNumBucketsPerOrderBits + 1);
}
- root->order_sub_index_masks[order] = sub_order_index_mask;
+ this->order_sub_index_masks[order] = sub_order_index_mask;
}
// Set up the actual usable buckets first.
@@ -208,768 +149,61 @@
// code simpler and the structures more generic.
size_t i, j;
size_t current_size = kGenericSmallestBucket;
- size_t currentIncrement =
+ size_t current_increment =
kGenericSmallestBucket >> kGenericNumBucketsPerOrderBits;
- PartitionBucket* bucket = &root->buckets[0];
+ internal::PartitionBucket* bucket = &this->buckets[0];
for (i = 0; i < kGenericNumBucketedOrders; ++i) {
for (j = 0; j < kGenericNumBucketsPerOrder; ++j) {
- bucket->slot_size = current_size;
- PartitionBucketInitBase(bucket, root);
+ bucket->Init(current_size);
// Disable psuedo buckets so that touching them faults.
if (current_size % kGenericSmallestBucket)
- bucket->active_pages_head = 0;
- current_size += currentIncrement;
+ bucket->active_pages_head = nullptr;
+ current_size += current_increment;
++bucket;
}
- currentIncrement <<= 1;
+ current_increment <<= 1;
}
DCHECK(current_size == 1 << kGenericMaxBucketedOrder);
- DCHECK(bucket == &root->buckets[0] + kGenericNumBuckets);
+ DCHECK(bucket == &this->buckets[0] + kGenericNumBuckets);
// Then set up the fast size -> bucket lookup table.
- bucket = &root->buckets[0];
- PartitionBucket** bucketPtr = &root->bucket_lookups[0];
+ bucket = &this->buckets[0];
+ internal::PartitionBucket** bucket_ptr = &this->bucket_lookups[0];
for (order = 0; order <= kBitsPerSizeT; ++order) {
for (j = 0; j < kGenericNumBucketsPerOrder; ++j) {
if (order < kGenericMinBucketedOrder) {
// Use the bucket of the finest granularity for malloc(0) etc.
- *bucketPtr++ = &root->buckets[0];
+ *bucket_ptr++ = &this->buckets[0];
} else if (order > kGenericMaxBucketedOrder) {
- *bucketPtr++ = &PartitionRootGeneric::gPagedBucket;
+ *bucket_ptr++ = internal::PartitionBucket::get_sentinel_bucket();
} else {
- PartitionBucket* validBucket = bucket;
+ internal::PartitionBucket* valid_bucket = bucket;
// Skip over invalid buckets.
- while (validBucket->slot_size % kGenericSmallestBucket)
- validBucket++;
- *bucketPtr++ = validBucket;
+ while (valid_bucket->slot_size % kGenericSmallestBucket)
+ valid_bucket++;
+ *bucket_ptr++ = valid_bucket;
bucket++;
}
}
}
- DCHECK(bucket == &root->buckets[0] + kGenericNumBuckets);
- DCHECK(bucketPtr ==
- &root->bucket_lookups[0] +
- ((kBitsPerSizeT + 1) * kGenericNumBucketsPerOrder));
+ DCHECK(bucket == &this->buckets[0] + kGenericNumBuckets);
+ DCHECK(bucket_ptr == &this->bucket_lookups[0] +
+ ((kBitsPerSizeT + 1) * kGenericNumBucketsPerOrder));
// And there's one last bucket lookup that will be hit for e.g. malloc(-1),
// which tries to overflow to a non-existant order.
- *bucketPtr = &PartitionRootGeneric::gPagedBucket;
+ *bucket_ptr = internal::PartitionBucket::get_sentinel_bucket();
}
-#if !defined(ARCH_CPU_64_BITS)
-static NOINLINE void PartitionOutOfMemoryWithLotsOfUncommitedPages() {
- OOM_CRASH();
-}
-#endif
-
-static NOINLINE void PartitionOutOfMemory(const PartitionRootBase* root) {
-#if !defined(ARCH_CPU_64_BITS)
- // Check whether this OOM is due to a lot of super pages that are allocated
- // but not committed, probably due to http://crbug.com/421387.
- if (root->total_size_of_super_pages +
- root->total_size_of_direct_mapped_pages -
- root->total_size_of_committed_pages >
- kReasonableSizeOfUnusedPages) {
- PartitionOutOfMemoryWithLotsOfUncommitedPages();
- }
-#endif
- if (PartitionRootBase::gOomHandlingFunction)
- (*PartitionRootBase::gOomHandlingFunction)();
- OOM_CRASH();
-}
-
-static NOINLINE void PartitionExcessiveAllocationSize() {
- OOM_CRASH();
-}
-
-static NOINLINE void PartitionBucketFull() {
- OOM_CRASH();
-}
-
-// partitionPageStateIs*
-// Note that it's only valid to call these functions on pages found on one of
-// the page lists. Specifically, you can't call these functions on full pages
-// that were detached from the active list.
-static bool ALWAYS_INLINE
-PartitionPageStateIsActive(const PartitionPage* page) {
- DCHECK(page != &PartitionRootGeneric::gSeedPage);
- DCHECK(!page->page_offset);
- return (page->num_allocated_slots > 0 &&
- (page->freelist_head || page->num_unprovisioned_slots));
-}
-
-static bool ALWAYS_INLINE PartitionPageStateIsFull(const PartitionPage* page) {
- DCHECK(page != &PartitionRootGeneric::gSeedPage);
- DCHECK(!page->page_offset);
- bool ret = (page->num_allocated_slots == PartitionBucketSlots(page->bucket));
- if (ret) {
- DCHECK(!page->freelist_head);
- DCHECK(!page->num_unprovisioned_slots);
- }
- return ret;
-}
-
-static bool ALWAYS_INLINE PartitionPageStateIsEmpty(const PartitionPage* page) {
- DCHECK(page != &PartitionRootGeneric::gSeedPage);
- DCHECK(!page->page_offset);
- return (!page->num_allocated_slots && page->freelist_head);
-}
-
-static bool ALWAYS_INLINE
-PartitionPageStateIsDecommitted(const PartitionPage* page) {
- DCHECK(page != &PartitionRootGeneric::gSeedPage);
- DCHECK(!page->page_offset);
- bool ret = (!page->num_allocated_slots && !page->freelist_head);
- if (ret) {
- DCHECK(!page->num_unprovisioned_slots);
- DCHECK(page->empty_cache_index == -1);
- }
- return ret;
-}
-
-static void PartitionIncreaseCommittedPages(PartitionRootBase* root,
- size_t len) {
- root->total_size_of_committed_pages += len;
- DCHECK(root->total_size_of_committed_pages <=
- root->total_size_of_super_pages +
- root->total_size_of_direct_mapped_pages);
-}
-
-static void PartitionDecreaseCommittedPages(PartitionRootBase* root,
- size_t len) {
- root->total_size_of_committed_pages -= len;
- DCHECK(root->total_size_of_committed_pages <=
- root->total_size_of_super_pages +
- root->total_size_of_direct_mapped_pages);
-}
-
-static ALWAYS_INLINE void PartitionDecommitSystemPages(PartitionRootBase* root,
- void* address,
- size_t length) {
- DecommitSystemPages(address, length);
- PartitionDecreaseCommittedPages(root, length);
-}
-
-static ALWAYS_INLINE void PartitionRecommitSystemPages(PartitionRootBase* root,
- void* address,
- size_t length) {
- RecommitSystemPages(address, length);
- PartitionIncreaseCommittedPages(root, length);
-}
-
-static ALWAYS_INLINE void* PartitionAllocPartitionPages(
- PartitionRootBase* root,
- int flags,
- uint16_t num_partition_pages) {
- DCHECK(!(reinterpret_cast<uintptr_t>(root->next_partition_page) %
- kPartitionPageSize));
- DCHECK(!(reinterpret_cast<uintptr_t>(root->next_partition_page_end) %
- kPartitionPageSize));
- DCHECK(num_partition_pages <= kNumPartitionPagesPerSuperPage);
- size_t total_size = kPartitionPageSize * num_partition_pages;
- size_t num_partition_pages_left =
- (root->next_partition_page_end - root->next_partition_page) >>
- kPartitionPageShift;
- if (LIKELY(num_partition_pages_left >= num_partition_pages)) {
- // In this case, we can still hand out pages from the current super page
- // allocation.
- char* ret = root->next_partition_page;
- root->next_partition_page += total_size;
- PartitionIncreaseCommittedPages(root, total_size);
- return ret;
- }
-
- // Need a new super page. We want to allocate super pages in a continguous
- // address region as much as possible. This is important for not causing
- // page table bloat and not fragmenting address spaces in 32 bit
- // architectures.
- char* requestedAddress = root->next_super_page;
- char* super_page = reinterpret_cast<char*>(AllocPages(
- requestedAddress, kSuperPageSize, kSuperPageSize, PageAccessible));
- if (UNLIKELY(!super_page))
- return 0;
-
- root->total_size_of_super_pages += kSuperPageSize;
- PartitionIncreaseCommittedPages(root, total_size);
-
- root->next_super_page = super_page + kSuperPageSize;
- char* ret = super_page + kPartitionPageSize;
- root->next_partition_page = ret + total_size;
- root->next_partition_page_end = root->next_super_page - kPartitionPageSize;
- // Make the first partition page in the super page a guard page, but leave a
- // hole in the middle.
- // This is where we put page metadata and also a tiny amount of extent
- // metadata.
- SetSystemPagesInaccessible(super_page, kSystemPageSize);
- SetSystemPagesInaccessible(super_page + (kSystemPageSize * 2),
- kPartitionPageSize - (kSystemPageSize * 2));
- // Also make the last partition page a guard page.
- SetSystemPagesInaccessible(super_page + (kSuperPageSize - kPartitionPageSize),
- kPartitionPageSize);
-
- // If we were after a specific address, but didn't get it, assume that
- // the system chose a lousy address. Here most OS'es have a default
- // algorithm that isn't randomized. For example, most Linux
- // distributions will allocate the mapping directly before the last
- // successful mapping, which is far from random. So we just get fresh
- // randomness for the next mapping attempt.
- if (requestedAddress && requestedAddress != super_page)
- root->next_super_page = 0;
-
- // We allocated a new super page so update super page metadata.
- // First check if this is a new extent or not.
- PartitionSuperPageExtentEntry* latest_extent =
- reinterpret_cast<PartitionSuperPageExtentEntry*>(
- PartitionSuperPageToMetadataArea(super_page));
- // By storing the root in every extent metadata object, we have a fast way
- // to go from a pointer within the partition to the root object.
- latest_extent->root = root;
- // Most new extents will be part of a larger extent, and these three fields
- // are unused, but we initialize them to 0 so that we get a clear signal
- // in case they are accidentally used.
- latest_extent->super_page_base = 0;
- latest_extent->super_pages_end = 0;
- latest_extent->next = 0;
-
- PartitionSuperPageExtentEntry* current_extent = root->current_extent;
- bool isNewExtent = (super_page != requestedAddress);
- if (UNLIKELY(isNewExtent)) {
- if (UNLIKELY(!current_extent)) {
- DCHECK(!root->first_extent);
- root->first_extent = latest_extent;
- } else {
- DCHECK(current_extent->super_page_base);
- current_extent->next = latest_extent;
- }
- root->current_extent = latest_extent;
- latest_extent->super_page_base = super_page;
- latest_extent->super_pages_end = super_page + kSuperPageSize;
- } else {
- // We allocated next to an existing extent so just nudge the size up a
- // little.
- DCHECK(current_extent->super_pages_end);
- current_extent->super_pages_end += kSuperPageSize;
- DCHECK(ret >= current_extent->super_page_base &&
- ret < current_extent->super_pages_end);
- }
- return ret;
-}
-
-static ALWAYS_INLINE uint16_t
-PartitionBucketPartitionPages(const PartitionBucket* bucket) {
- return (bucket->num_system_pages_per_slot_span +
- (kNumSystemPagesPerPartitionPage - 1)) /
- kNumSystemPagesPerPartitionPage;
-}
-
-static ALWAYS_INLINE void PartitionPageReset(PartitionPage* page) {
- DCHECK(PartitionPageStateIsDecommitted(page));
-
- page->num_unprovisioned_slots = PartitionBucketSlots(page->bucket);
- DCHECK(page->num_unprovisioned_slots);
-
- page->next_page = nullptr;
-}
-
-static ALWAYS_INLINE void PartitionPageSetup(PartitionPage* page,
- PartitionBucket* bucket) {
- // The bucket never changes. We set it up once.
- page->bucket = bucket;
- page->empty_cache_index = -1;
-
- PartitionPageReset(page);
-
- // If this page has just a single slot, do not set up page offsets for any
- // page metadata other than the first one. This ensures that attempts to
- // touch invalid page metadata fail.
- if (page->num_unprovisioned_slots == 1)
- return;
-
- uint16_t num_partition_pages = PartitionBucketPartitionPages(bucket);
- char* page_char_ptr = reinterpret_cast<char*>(page);
- for (uint16_t i = 1; i < num_partition_pages; ++i) {
- page_char_ptr += kPageMetadataSize;
- PartitionPage* secondary_page =
- reinterpret_cast<PartitionPage*>(page_char_ptr);
- secondary_page->page_offset = i;
- }
-}
-
-static ALWAYS_INLINE char* PartitionPageAllocAndFillFreelist(
- PartitionPage* page) {
- DCHECK(page != &PartitionRootGeneric::gSeedPage);
- uint16_t num_slots = page->num_unprovisioned_slots;
- DCHECK(num_slots);
- PartitionBucket* bucket = page->bucket;
- // We should only get here when _every_ slot is either used or unprovisioned.
- // (The third state is "on the freelist". If we have a non-empty freelist, we
- // should not get here.)
- DCHECK(num_slots + page->num_allocated_slots == PartitionBucketSlots(bucket));
- // Similarly, make explicitly sure that the freelist is empty.
- DCHECK(!page->freelist_head);
- DCHECK(page->num_allocated_slots >= 0);
-
- size_t size = bucket->slot_size;
- char* base = reinterpret_cast<char*>(PartitionPageToPointer(page));
- char* return_object = base + (size * page->num_allocated_slots);
- char* firstFreelistPointer = return_object + size;
- char* firstFreelistPointerExtent =
- firstFreelistPointer + sizeof(PartitionFreelistEntry*);
- // Our goal is to fault as few system pages as possible. We calculate the
- // page containing the "end" of the returned slot, and then allow freelist
- // pointers to be written up to the end of that page.
- char* sub_page_limit = reinterpret_cast<char*>(
- RoundUpToSystemPage(reinterpret_cast<size_t>(firstFreelistPointer)));
- char* slots_limit = return_object + (size * num_slots);
- char* freelist_limit = sub_page_limit;
- if (UNLIKELY(slots_limit < freelist_limit))
- freelist_limit = slots_limit;
-
- uint16_t num_new_freelist_entries = 0;
- if (LIKELY(firstFreelistPointerExtent <= freelist_limit)) {
- // Only consider used space in the slot span. If we consider wasted
- // space, we may get an off-by-one when a freelist pointer fits in the
- // wasted space, but a slot does not.
- // We know we can fit at least one freelist pointer.
- num_new_freelist_entries = 1;
- // Any further entries require space for the whole slot span.
- num_new_freelist_entries += static_cast<uint16_t>(
- (freelist_limit - firstFreelistPointerExtent) / size);
- }
-
- // We always return an object slot -- that's the +1 below.
- // We do not neccessarily create any new freelist entries, because we cross
- // sub page boundaries frequently for large bucket sizes.
- DCHECK(num_new_freelist_entries + 1 <= num_slots);
- num_slots -= (num_new_freelist_entries + 1);
- page->num_unprovisioned_slots = num_slots;
- page->num_allocated_slots++;
-
- if (LIKELY(num_new_freelist_entries)) {
- char* freelist_pointer = firstFreelistPointer;
- PartitionFreelistEntry* entry =
- reinterpret_cast<PartitionFreelistEntry*>(freelist_pointer);
- page->freelist_head = entry;
- while (--num_new_freelist_entries) {
- freelist_pointer += size;
- PartitionFreelistEntry* next_entry =
- reinterpret_cast<PartitionFreelistEntry*>(freelist_pointer);
- entry->next = PartitionFreelistMask(next_entry);
- entry = next_entry;
- }
- entry->next = PartitionFreelistMask(0);
- } else {
- page->freelist_head = 0;
- }
- return return_object;
-}
-
-// This helper function scans a bucket's active page list for a suitable new
-// active page.
-// When it finds a suitable new active page (one that has free slots and is not
-// empty), it is set as the new active page. If there is no suitable new
-// active page, the current active page is set to the seed page.
-// As potential pages are scanned, they are tidied up according to their state.
-// Empty pages are swept on to the empty page list, decommitted pages on to the
-// decommitted page list and full pages are unlinked from any list.
-static bool PartitionSetNewActivePage(PartitionBucket* bucket) {
- PartitionPage* page = bucket->active_pages_head;
- if (page == &PartitionRootBase::gSeedPage)
- return false;
-
- PartitionPage* next_page;
-
- for (; page; page = next_page) {
- next_page = page->next_page;
- DCHECK(page->bucket == bucket);
- DCHECK(page != bucket->empty_pages_head);
- DCHECK(page != bucket->decommitted_pages_head);
-
- // Deal with empty and decommitted pages.
- if (LIKELY(PartitionPageStateIsActive(page))) {
- // This page is usable because it has freelist entries, or has
- // unprovisioned slots we can create freelist entries from.
- bucket->active_pages_head = page;
- return true;
- }
- if (LIKELY(PartitionPageStateIsEmpty(page))) {
- page->next_page = bucket->empty_pages_head;
- bucket->empty_pages_head = page;
- } else if (LIKELY(PartitionPageStateIsDecommitted(page))) {
- page->next_page = bucket->decommitted_pages_head;
- bucket->decommitted_pages_head = page;
- } else {
- DCHECK(PartitionPageStateIsFull(page));
- // If we get here, we found a full page. Skip over it too, and also
- // tag it as full (via a negative value). We need it tagged so that
- // free'ing can tell, and move it back into the active page list.
- page->num_allocated_slots = -page->num_allocated_slots;
- ++bucket->num_full_pages;
- // num_full_pages is a uint16_t for efficient packing so guard against
- // overflow to be safe.
- if (UNLIKELY(!bucket->num_full_pages))
- PartitionBucketFull();
- // Not necessary but might help stop accidents.
- page->next_page = 0;
- }
- }
-
- bucket->active_pages_head = &PartitionRootGeneric::gSeedPage;
- return false;
-}
-
-static ALWAYS_INLINE PartitionDirectMapExtent* partitionPageToDirectMapExtent(
- PartitionPage* page) {
- DCHECK(PartitionBucketIsDirectMapped(page->bucket));
- return reinterpret_cast<PartitionDirectMapExtent*>(
- reinterpret_cast<char*>(page) + 3 * kPageMetadataSize);
-}
-
-static ALWAYS_INLINE void PartitionPageSetRawSize(PartitionPage* page,
- size_t size) {
- size_t* raw_size_ptr = PartitionPageGetRawSizePtr(page);
- if (UNLIKELY(raw_size_ptr != nullptr))
- *raw_size_ptr = size;
-}
-
-static ALWAYS_INLINE PartitionPage* PartitionDirectMap(PartitionRootBase* root,
- int flags,
- size_t raw_size) {
- size_t size = PartitionDirectMapSize(raw_size);
-
- // Because we need to fake looking like a super page, we need to allocate
- // a bunch of system pages more than "size":
- // - The first few system pages are the partition page in which the super
- // page metadata is stored. We fault just one system page out of a partition
- // page sized clump.
- // - We add a trailing guard page on 32-bit (on 64-bit we rely on the
- // massive address space plus randomization instead).
- size_t map_size = size + kPartitionPageSize;
-#if !defined(ARCH_CPU_64_BITS)
- map_size += kSystemPageSize;
-#endif
- // Round up to the allocation granularity.
- map_size += kPageAllocationGranularityOffsetMask;
- map_size &= kPageAllocationGranularityBaseMask;
-
- // TODO: these pages will be zero-filled. Consider internalizing an
- // allocZeroed() API so we can avoid a memset() entirely in this case.
- char* ptr = reinterpret_cast<char*>(
- AllocPages(0, map_size, kSuperPageSize, PageAccessible));
- if (UNLIKELY(!ptr))
- return nullptr;
-
- size_t committed_page_size = size + kSystemPageSize;
- root->total_size_of_direct_mapped_pages += committed_page_size;
- PartitionIncreaseCommittedPages(root, committed_page_size);
-
- char* slot = ptr + kPartitionPageSize;
- SetSystemPagesInaccessible(ptr + (kSystemPageSize * 2),
- kPartitionPageSize - (kSystemPageSize * 2));
-#if !defined(ARCH_CPU_64_BITS)
- SetSystemPagesInaccessible(ptr, kSystemPageSize);
- SetSystemPagesInaccessible(slot + size, kSystemPageSize);
-#endif
-
- PartitionSuperPageExtentEntry* extent =
- reinterpret_cast<PartitionSuperPageExtentEntry*>(
- PartitionSuperPageToMetadataArea(ptr));
- extent->root = root;
- // The new structures are all located inside a fresh system page so they
- // will all be zeroed out. These DCHECKs are for documentation.
- DCHECK(!extent->super_page_base);
- DCHECK(!extent->super_pages_end);
- DCHECK(!extent->next);
- PartitionPage* page = PartitionPointerToPageNoAlignmentCheck(slot);
- PartitionBucket* bucket = reinterpret_cast<PartitionBucket*>(
- reinterpret_cast<char*>(page) + (kPageMetadataSize * 2));
- DCHECK(!page->next_page);
- DCHECK(!page->num_allocated_slots);
- DCHECK(!page->num_unprovisioned_slots);
- DCHECK(!page->page_offset);
- DCHECK(!page->empty_cache_index);
- page->bucket = bucket;
- page->freelist_head = reinterpret_cast<PartitionFreelistEntry*>(slot);
- PartitionFreelistEntry* next_entry =
- reinterpret_cast<PartitionFreelistEntry*>(slot);
- next_entry->next = PartitionFreelistMask(0);
-
- DCHECK(!bucket->active_pages_head);
- DCHECK(!bucket->empty_pages_head);
- DCHECK(!bucket->decommitted_pages_head);
- DCHECK(!bucket->num_system_pages_per_slot_span);
- DCHECK(!bucket->num_full_pages);
- bucket->slot_size = size;
-
- PartitionDirectMapExtent* map_extent = partitionPageToDirectMapExtent(page);
- map_extent->map_size = map_size - kPartitionPageSize - kSystemPageSize;
- map_extent->bucket = bucket;
-
- // Maintain the doubly-linked list of all direct mappings.
- map_extent->next_extent = root->direct_map_list;
- if (map_extent->next_extent)
- map_extent->next_extent->prev_extent = map_extent;
- map_extent->prev_extent = nullptr;
- root->direct_map_list = map_extent;
-
- return page;
-}
-
-static ALWAYS_INLINE void PartitionDirectUnmap(PartitionPage* page) {
- PartitionRootBase* root = PartitionPageToRoot(page);
- const PartitionDirectMapExtent* extent = partitionPageToDirectMapExtent(page);
- size_t unmap_size = extent->map_size;
-
- // Maintain the doubly-linked list of all direct mappings.
- if (extent->prev_extent) {
- DCHECK(extent->prev_extent->next_extent == extent);
- extent->prev_extent->next_extent = extent->next_extent;
- } else {
- root->direct_map_list = extent->next_extent;
- }
- if (extent->next_extent) {
- DCHECK(extent->next_extent->prev_extent == extent);
- extent->next_extent->prev_extent = extent->prev_extent;
- }
-
- // Add on the size of the trailing guard page and preceeding partition
- // page.
- unmap_size += kPartitionPageSize + kSystemPageSize;
-
- size_t uncommitted_page_size = page->bucket->slot_size + kSystemPageSize;
- PartitionDecreaseCommittedPages(root, uncommitted_page_size);
- DCHECK(root->total_size_of_direct_mapped_pages >= uncommitted_page_size);
- root->total_size_of_direct_mapped_pages -= uncommitted_page_size;
-
- DCHECK(!(unmap_size & kPageAllocationGranularityOffsetMask));
-
- char* ptr = reinterpret_cast<char*>(PartitionPageToPointer(page));
- // Account for the mapping starting a partition page before the actual
- // allocation address.
- ptr -= kPartitionPageSize;
-
- FreePages(ptr, unmap_size);
-}
-
-void* PartitionAllocSlowPath(PartitionRootBase* root,
- int flags,
- size_t size,
- PartitionBucket* bucket) {
- // The slow path is called when the freelist is empty.
- DCHECK(!bucket->active_pages_head->freelist_head);
-
- PartitionPage* new_page = nullptr;
-
- // For the PartitionAllocGeneric API, we have a bunch of buckets marked
- // as special cases. We bounce them through to the slow path so that we
- // can still have a blazing fast hot path due to lack of corner-case
- // branches.
- bool returnNull = flags & PartitionAllocReturnNull;
- if (UNLIKELY(PartitionBucketIsDirectMapped(bucket))) {
- DCHECK(size > kGenericMaxBucketed);
- DCHECK(bucket == &PartitionRootBase::gPagedBucket);
- DCHECK(bucket->active_pages_head == &PartitionRootGeneric::gSeedPage);
- if (size > kGenericMaxDirectMapped) {
- if (returnNull)
- return nullptr;
- PartitionExcessiveAllocationSize();
- }
- new_page = PartitionDirectMap(root, flags, size);
- } else if (LIKELY(PartitionSetNewActivePage(bucket))) {
- // First, did we find an active page in the active pages list?
- new_page = bucket->active_pages_head;
- DCHECK(PartitionPageStateIsActive(new_page));
- } else if (LIKELY(bucket->empty_pages_head != nullptr) ||
- LIKELY(bucket->decommitted_pages_head != nullptr)) {
- // Second, look in our lists of empty and decommitted pages.
- // Check empty pages first, which are preferred, but beware that an
- // empty page might have been decommitted.
- while (LIKELY((new_page = bucket->empty_pages_head) != nullptr)) {
- DCHECK(new_page->bucket == bucket);
- DCHECK(PartitionPageStateIsEmpty(new_page) ||
- PartitionPageStateIsDecommitted(new_page));
- bucket->empty_pages_head = new_page->next_page;
- // Accept the empty page unless it got decommitted.
- if (new_page->freelist_head) {
- new_page->next_page = nullptr;
- break;
- }
- DCHECK(PartitionPageStateIsDecommitted(new_page));
- new_page->next_page = bucket->decommitted_pages_head;
- bucket->decommitted_pages_head = new_page;
- }
- if (UNLIKELY(!new_page) &&
- LIKELY(bucket->decommitted_pages_head != nullptr)) {
- new_page = bucket->decommitted_pages_head;
- DCHECK(new_page->bucket == bucket);
- DCHECK(PartitionPageStateIsDecommitted(new_page));
- bucket->decommitted_pages_head = new_page->next_page;
- void* addr = PartitionPageToPointer(new_page);
- PartitionRecommitSystemPages(root, addr,
- PartitionBucketBytes(new_page->bucket));
- PartitionPageReset(new_page);
- }
- DCHECK(new_page);
- } else {
- // Third. If we get here, we need a brand new page.
- uint16_t num_partition_pages = PartitionBucketPartitionPages(bucket);
- void* rawPages =
- PartitionAllocPartitionPages(root, flags, num_partition_pages);
- if (LIKELY(rawPages != nullptr)) {
- new_page = PartitionPointerToPageNoAlignmentCheck(rawPages);
- PartitionPageSetup(new_page, bucket);
- }
- }
-
- // Bail if we had a memory allocation failure.
- if (UNLIKELY(!new_page)) {
- DCHECK(bucket->active_pages_head == &PartitionRootGeneric::gSeedPage);
- if (returnNull)
- return nullptr;
- PartitionOutOfMemory(root);
- }
-
- bucket = new_page->bucket;
- DCHECK(bucket != &PartitionRootBase::gPagedBucket);
- bucket->active_pages_head = new_page;
- PartitionPageSetRawSize(new_page, size);
-
- // If we found an active page with free slots, or an empty page, we have a
- // usable freelist head.
- if (LIKELY(new_page->freelist_head != nullptr)) {
- PartitionFreelistEntry* entry = new_page->freelist_head;
- PartitionFreelistEntry* new_head = PartitionFreelistMask(entry->next);
- new_page->freelist_head = new_head;
- new_page->num_allocated_slots++;
- return entry;
- }
- // Otherwise, we need to build the freelist.
- DCHECK(new_page->num_unprovisioned_slots);
- return PartitionPageAllocAndFillFreelist(new_page);
-}
-
-static ALWAYS_INLINE void PartitionDecommitPage(PartitionRootBase* root,
- PartitionPage* page) {
- DCHECK(PartitionPageStateIsEmpty(page));
- DCHECK(!PartitionBucketIsDirectMapped(page->bucket));
- void* addr = PartitionPageToPointer(page);
- PartitionDecommitSystemPages(root, addr, PartitionBucketBytes(page->bucket));
-
- // We actually leave the decommitted page in the active list. We'll sweep
- // it on to the decommitted page list when we next walk the active page
- // list.
- // Pulling this trick enables us to use a singly-linked page list for all
- // cases, which is critical in keeping the page metadata structure down to
- // 32 bytes in size.
- page->freelist_head = 0;
- page->num_unprovisioned_slots = 0;
- DCHECK(PartitionPageStateIsDecommitted(page));
-}
-
-static void PartitionDecommitPageIfPossible(PartitionRootBase* root,
- PartitionPage* page) {
- DCHECK(page->empty_cache_index >= 0);
- DCHECK(static_cast<unsigned>(page->empty_cache_index) < kMaxFreeableSpans);
- DCHECK(page == root->global_empty_page_ring[page->empty_cache_index]);
- page->empty_cache_index = -1;
- if (PartitionPageStateIsEmpty(page))
- PartitionDecommitPage(root, page);
-}
-
-static ALWAYS_INLINE void PartitionRegisterEmptyPage(PartitionPage* page) {
- DCHECK(PartitionPageStateIsEmpty(page));
- PartitionRootBase* root = PartitionPageToRoot(page);
-
- // If the page is already registered as empty, give it another life.
- if (page->empty_cache_index != -1) {
- DCHECK(page->empty_cache_index >= 0);
- DCHECK(static_cast<unsigned>(page->empty_cache_index) < kMaxFreeableSpans);
- DCHECK(root->global_empty_page_ring[page->empty_cache_index] == page);
- root->global_empty_page_ring[page->empty_cache_index] = 0;
- }
-
- int16_t current_index = root->global_empty_page_ring_index;
- PartitionPage* pageToDecommit = root->global_empty_page_ring[current_index];
- // The page might well have been re-activated, filled up, etc. before we get
- // around to looking at it here.
- if (pageToDecommit)
- PartitionDecommitPageIfPossible(root, pageToDecommit);
-
- // We put the empty slot span on our global list of "pages that were once
- // empty". thus providing it a bit of breathing room to get re-used before
- // we really free it. This improves performance, particularly on Mac OS X
- // which has subpar memory management performance.
- root->global_empty_page_ring[current_index] = page;
- page->empty_cache_index = current_index;
- ++current_index;
- if (current_index == kMaxFreeableSpans)
- current_index = 0;
- root->global_empty_page_ring_index = current_index;
-}
-
-static void PartitionDecommitEmptyPages(PartitionRootBase* root) {
- for (size_t i = 0; i < kMaxFreeableSpans; ++i) {
- PartitionPage* page = root->global_empty_page_ring[i];
- if (page)
- PartitionDecommitPageIfPossible(root, page);
- root->global_empty_page_ring[i] = nullptr;
- }
-}
-
-void PartitionFreeSlowPath(PartitionPage* page) {
- PartitionBucket* bucket = page->bucket;
- DCHECK(page != &PartitionRootGeneric::gSeedPage);
- if (LIKELY(page->num_allocated_slots == 0)) {
- // Page became fully unused.
- if (UNLIKELY(PartitionBucketIsDirectMapped(bucket))) {
- PartitionDirectUnmap(page);
- return;
- }
- // If it's the current active page, change it. We bounce the page to
- // the empty list as a force towards defragmentation.
- if (LIKELY(page == bucket->active_pages_head))
- (void)PartitionSetNewActivePage(bucket);
- DCHECK(bucket->active_pages_head != page);
-
- PartitionPageSetRawSize(page, 0);
- DCHECK(!PartitionPageGetRawSize(page));
-
- PartitionRegisterEmptyPage(page);
- } else {
- DCHECK(!PartitionBucketIsDirectMapped(bucket));
- // Ensure that the page is full. That's the only valid case if we
- // arrive here.
- DCHECK(page->num_allocated_slots < 0);
- // A transition of num_allocated_slots from 0 to -1 is not legal, and
- // likely indicates a double-free.
- CHECK(page->num_allocated_slots != -1);
- page->num_allocated_slots = -page->num_allocated_slots - 2;
- DCHECK(page->num_allocated_slots == PartitionBucketSlots(bucket) - 1);
- // Fully used page became partially used. It must be put back on the
- // non-full page list. Also make it the current page to increase the
- // chances of it being filled up again. The old current page will be
- // the next page.
- DCHECK(!page->next_page);
- if (LIKELY(bucket->active_pages_head != &PartitionRootGeneric::gSeedPage))
- page->next_page = bucket->active_pages_head;
- bucket->active_pages_head = page;
- --bucket->num_full_pages;
- // Special case: for a partition page with just a single slot, it may
- // now be empty and we want to run it through the empty logic.
- if (UNLIKELY(page->num_allocated_slots == 0))
- PartitionFreeSlowPath(page);
- }
-}
-
-bool partitionReallocDirectMappedInPlace(PartitionRootGeneric* root,
- PartitionPage* page,
+bool PartitionReallocDirectMappedInPlace(PartitionRootGeneric* root,
+ internal::PartitionPage* page,
size_t raw_size) {
- DCHECK(PartitionBucketIsDirectMapped(page->bucket));
+ DCHECK(page->bucket->is_direct_mapped());
- raw_size = PartitionCookieSizeAdjustAdd(raw_size);
+ raw_size = internal::PartitionCookieSizeAdjustAdd(raw_size);
// Note that the new size might be a bucketed size; this function is called
// whenever we're reallocating a direct mapped allocation.
- size_t new_size = PartitionDirectMapSize(raw_size);
+ size_t new_size = internal::PartitionBucket::get_direct_map_size(raw_size);
if (new_size < kGenericMinDirectMappedDownsize)
return false;
@@ -978,10 +212,11 @@
if (new_size == current_size)
return true;
- char* char_ptr = static_cast<char*>(PartitionPageToPointer(page));
+ char* char_ptr = static_cast<char*>(internal::PartitionPage::ToPointer(page));
if (new_size < current_size) {
- size_t map_size = partitionPageToDirectMapExtent(page)->map_size;
+ size_t map_size =
+ internal::PartitionDirectMapExtent::FromPage(page)->map_size;
// Don't reallocate in-place if new size is less than 80 % of the full
// map size, to avoid holding on to too much unused address space.
@@ -989,16 +224,18 @@
return false;
// Shrink by decommitting unneeded pages and making them inaccessible.
- size_t decommitSize = current_size - new_size;
- PartitionDecommitSystemPages(root, char_ptr + new_size, decommitSize);
- SetSystemPagesInaccessible(char_ptr + new_size, decommitSize);
- } else if (new_size <= partitionPageToDirectMapExtent(page)->map_size) {
+ size_t decommit_size = current_size - new_size;
+ root->DecommitSystemPages(char_ptr + new_size, decommit_size);
+ CHECK(SetSystemPagesAccess(char_ptr + new_size, decommit_size,
+ PageInaccessible));
+ } else if (new_size <=
+ internal::PartitionDirectMapExtent::FromPage(page)->map_size) {
// Grow within the actually allocated memory. Just need to make the
// pages accessible again.
size_t recommit_size = new_size - current_size;
- bool ret = SetSystemPagesAccessible(char_ptr + current_size, recommit_size);
- CHECK(ret);
- PartitionRecommitSystemPages(root, char_ptr + current_size, recommit_size);
+ CHECK(SetSystemPagesAccess(char_ptr + current_size, recommit_size,
+ PageReadWrite));
+ root->RecommitSystemPages(char_ptr + current_size, recommit_size);
#if DCHECK_IS_ON()
memset(char_ptr + current_size, kUninitializedByte, recommit_size);
@@ -1011,11 +248,12 @@
#if DCHECK_IS_ON()
// Write a new trailing cookie.
- PartitionCookieWriteValue(char_ptr + raw_size - kCookieSize);
+ internal::PartitionCookieWriteValue(char_ptr + raw_size -
+ internal::kCookieSize);
#endif
- PartitionPageSetRawSize(page, raw_size);
- DCHECK(PartitionPageGetRawSize(page) == raw_size);
+ page->set_raw_size(raw_size);
+ DCHECK(page->get_raw_size() == raw_size);
page->bucket->slot_size = new_size;
return true;
@@ -1034,49 +272,48 @@
if (UNLIKELY(!ptr))
return PartitionAllocGenericFlags(root, flags, new_size, type_name);
if (UNLIKELY(!new_size)) {
- PartitionFreeGeneric(root, ptr);
+ root->Free(ptr);
return nullptr;
}
if (new_size > kGenericMaxDirectMapped) {
if (flags & PartitionAllocReturnNull)
return nullptr;
- else
- PartitionExcessiveAllocationSize();
+ internal::PartitionExcessiveAllocationSize();
}
- DCHECK(PartitionPointerIsValid(PartitionCookieFreePointerAdjust(ptr)));
+ internal::PartitionPage* page = internal::PartitionPage::FromPointer(
+ internal::PartitionCookieFreePointerAdjust(ptr));
+ // TODO(palmer): See if we can afford to make this a CHECK.
+ DCHECK(root->IsValidPage(page));
- PartitionPage* page =
- PartitionPointerToPage(PartitionCookieFreePointerAdjust(ptr));
-
- if (UNLIKELY(PartitionBucketIsDirectMapped(page->bucket))) {
+ if (UNLIKELY(page->bucket->is_direct_mapped())) {
// We may be able to perform the realloc in place by changing the
// accessibility of memory pages and, if reducing the size, decommitting
// them.
- if (partitionReallocDirectMappedInPlace(root, page, new_size)) {
+ if (PartitionReallocDirectMappedInPlace(root, page, new_size)) {
PartitionAllocHooks::ReallocHookIfEnabled(ptr, ptr, new_size, type_name);
return ptr;
}
}
- size_t actual_new_size = PartitionAllocActualSize(root, new_size);
+ size_t actual_new_size = root->ActualSize(new_size);
size_t actual_old_size = PartitionAllocGetSize(ptr);
// TODO: note that tcmalloc will "ignore" a downsizing realloc() unless the
// new size is a significant percentage smaller. We could do the same if we
// determine it is a win.
if (actual_new_size == actual_old_size) {
- // Trying to allocate a block of size new_size would give us a block of
+ // Trying to allocate a block of size |new_size| would give us a block of
// the same size as the one we've already got, so re-use the allocation
// after updating statistics (and cookies, if present).
- PartitionPageSetRawSize(page, PartitionCookieSizeAdjustAdd(new_size));
+ page->set_raw_size(internal::PartitionCookieSizeAdjustAdd(new_size));
#if DCHECK_IS_ON()
// Write a new trailing cookie when it is possible to keep track of
// |new_size| via the raw size pointer.
- if (PartitionPageGetRawSizePtr(page))
- PartitionCookieWriteValue(static_cast<char*>(ptr) + new_size);
-#endif // DCHECK_IS_ON()
+ if (page->get_raw_size_ptr())
+ internal::PartitionCookieWriteValue(static_cast<char*>(ptr) + new_size);
+#endif
return ptr;
}
@@ -1085,8 +322,7 @@
if (!ret) {
if (flags & PartitionAllocReturnNull)
return nullptr;
- else
- PartitionExcessiveAllocationSize();
+ internal::PartitionExcessiveAllocationSize();
}
size_t copy_size = actual_old_size;
@@ -1094,67 +330,80 @@
copy_size = new_size;
memcpy(ret, ptr, copy_size);
- PartitionFreeGeneric(root, ptr);
+ root->Free(ptr);
return ret;
-#endif // defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
+#endif
}
-void* PartitionReallocGeneric(PartitionRootGeneric* root,
- void* ptr,
- size_t new_size,
- const char* type_name) {
- return PartitionReallocGenericFlags(root, 0, ptr, new_size, type_name);
+void* PartitionRootGeneric::Realloc(void* ptr,
+ size_t new_size,
+ const char* type_name) {
+ return PartitionReallocGenericFlags(this, 0, ptr, new_size, type_name);
}
-static size_t PartitionPurgePage(PartitionPage* page, bool discard) {
- const PartitionBucket* bucket = page->bucket;
+void* PartitionRootGeneric::TryRealloc(void* ptr,
+ size_t new_size,
+ const char* type_name) {
+ return PartitionReallocGenericFlags(this, PartitionAllocReturnNull, ptr,
+ new_size, type_name);
+}
+
+static size_t PartitionPurgePage(internal::PartitionPage* page, bool discard) {
+ const internal::PartitionBucket* bucket = page->bucket;
size_t slot_size = bucket->slot_size;
if (slot_size < kSystemPageSize || !page->num_allocated_slots)
return 0;
- size_t bucket_num_slots = PartitionBucketSlots(bucket);
+ size_t bucket_num_slots = bucket->get_slots_per_span();
size_t discardable_bytes = 0;
- size_t raw_size = PartitionPageGetRawSize(const_cast<PartitionPage*>(page));
+ size_t raw_size = page->get_raw_size();
if (raw_size) {
- uint32_t usedBytes = static_cast<uint32_t>(RoundUpToSystemPage(raw_size));
- discardable_bytes = bucket->slot_size - usedBytes;
+ uint32_t used_bytes = static_cast<uint32_t>(RoundUpToSystemPage(raw_size));
+ discardable_bytes = bucket->slot_size - used_bytes;
if (discardable_bytes && discard) {
- char* ptr = reinterpret_cast<char*>(PartitionPageToPointer(page));
- ptr += usedBytes;
+ char* ptr =
+ reinterpret_cast<char*>(internal::PartitionPage::ToPointer(page));
+ ptr += used_bytes;
DiscardSystemPages(ptr, discardable_bytes);
}
return discardable_bytes;
}
- const size_t max_slot_count =
+ constexpr size_t kMaxSlotCount =
(kPartitionPageSize * kMaxPartitionPagesPerSlotSpan) / kSystemPageSize;
- DCHECK(bucket_num_slots <= max_slot_count);
+ DCHECK(bucket_num_slots <= kMaxSlotCount);
DCHECK(page->num_unprovisioned_slots < bucket_num_slots);
size_t num_slots = bucket_num_slots - page->num_unprovisioned_slots;
- char slot_usage[max_slot_count];
+ char slot_usage[kMaxSlotCount];
+#if !defined(OS_WIN)
+ // The last freelist entry should not be discarded when using OS_WIN.
+ // DiscardVirtualMemory makes the contents of discarded memory undefined.
size_t last_slot = static_cast<size_t>(-1);
+#endif
memset(slot_usage, 1, num_slots);
- char* ptr = reinterpret_cast<char*>(PartitionPageToPointer(page));
- PartitionFreelistEntry* entry = page->freelist_head;
+ char* ptr = reinterpret_cast<char*>(internal::PartitionPage::ToPointer(page));
// First, walk the freelist for this page and make a bitmap of which slots
// are not in use.
- while (entry) {
- size_t slotIndex = (reinterpret_cast<char*>(entry) - ptr) / slot_size;
- DCHECK(slotIndex < num_slots);
- slot_usage[slotIndex] = 0;
- entry = PartitionFreelistMask(entry->next);
- // If we have a slot where the masked freelist entry is 0, we can
- // actually discard that freelist entry because touching a discarded
- // page is guaranteed to return original content or 0.
- // (Note that this optimization won't fire on big endian machines
- // because the masking function is negation.)
- if (!PartitionFreelistMask(entry))
- last_slot = slotIndex;
+ for (internal::PartitionFreelistEntry* entry = page->freelist_head; entry;
+ /**/) {
+ size_t slot_index = (reinterpret_cast<char*>(entry) - ptr) / slot_size;
+ DCHECK(slot_index < num_slots);
+ slot_usage[slot_index] = 0;
+ entry = internal::PartitionFreelistEntry::Transform(entry->next);
+#if !defined(OS_WIN)
+ // If we have a slot where the masked freelist entry is 0, we can actually
+ // discard that freelist entry because touching a discarded page is
+ // guaranteed to return original content or 0. (Note that this optimization
+ // won't fire on big-endian machines because the masking function is
+ // negation.)
+ if (!internal::PartitionFreelistEntry::Transform(entry))
+ last_slot = slot_index;
+#endif
}
- // If the slot(s) at the end of the slot span are not in used, we can
- // truncate them entirely and rewrite the freelist.
+ // If the slot(s) at the end of the slot span are not in used, we can truncate
+ // them entirely and rewrite the freelist.
size_t truncated_slots = 0;
while (!slot_usage[num_slots - 1]) {
truncated_slots++;
@@ -1163,62 +412,67 @@
}
// First, do the work of calculating the discardable bytes. Don't actually
// discard anything unless the discard flag was passed in.
- char* begin_ptr = nullptr;
- char* end_ptr = nullptr;
- size_t unprovisioned_bytes = 0;
if (truncated_slots) {
- begin_ptr = ptr + (num_slots * slot_size);
- end_ptr = begin_ptr + (slot_size * truncated_slots);
+ size_t unprovisioned_bytes = 0;
+ char* begin_ptr = ptr + (num_slots * slot_size);
+ char* end_ptr = begin_ptr + (slot_size * truncated_slots);
begin_ptr = reinterpret_cast<char*>(
RoundUpToSystemPage(reinterpret_cast<size_t>(begin_ptr)));
- // We round the end pointer here up and not down because we're at the
- // end of a slot span, so we "own" all the way up the page boundary.
+ // We round the end pointer here up and not down because we're at the end of
+ // a slot span, so we "own" all the way up the page boundary.
end_ptr = reinterpret_cast<char*>(
RoundUpToSystemPage(reinterpret_cast<size_t>(end_ptr)));
- DCHECK(end_ptr <= ptr + PartitionBucketBytes(bucket));
+ DCHECK(end_ptr <= ptr + bucket->get_bytes_per_span());
if (begin_ptr < end_ptr) {
unprovisioned_bytes = end_ptr - begin_ptr;
discardable_bytes += unprovisioned_bytes;
}
- }
- if (unprovisioned_bytes && discard) {
- DCHECK(truncated_slots > 0);
- size_t num_new_entries = 0;
- page->num_unprovisioned_slots += static_cast<uint16_t>(truncated_slots);
- // Rewrite the freelist.
- PartitionFreelistEntry** entry_ptr = &page->freelist_head;
- for (size_t slotIndex = 0; slotIndex < num_slots; ++slotIndex) {
- if (slot_usage[slotIndex])
- continue;
- PartitionFreelistEntry* entry = reinterpret_cast<PartitionFreelistEntry*>(
- ptr + (slot_size * slotIndex));
- *entry_ptr = PartitionFreelistMask(entry);
- entry_ptr = reinterpret_cast<PartitionFreelistEntry**>(entry);
- num_new_entries++;
+ if (unprovisioned_bytes && discard) {
+ DCHECK(truncated_slots > 0);
+ size_t num_new_entries = 0;
+ page->num_unprovisioned_slots += static_cast<uint16_t>(truncated_slots);
+ // Rewrite the freelist.
+ internal::PartitionFreelistEntry** entry_ptr = &page->freelist_head;
+ for (size_t slot_index = 0; slot_index < num_slots; ++slot_index) {
+ if (slot_usage[slot_index])
+ continue;
+ auto* entry = reinterpret_cast<internal::PartitionFreelistEntry*>(
+ ptr + (slot_size * slot_index));
+ *entry_ptr = internal::PartitionFreelistEntry::Transform(entry);
+ entry_ptr = reinterpret_cast<internal::PartitionFreelistEntry**>(entry);
+ num_new_entries++;
+#if !defined(OS_WIN)
+ last_slot = slot_index;
+#endif
+ }
+ // Terminate the freelist chain.
+ *entry_ptr = nullptr;
+ // The freelist head is stored unmasked.
+ page->freelist_head =
+ internal::PartitionFreelistEntry::Transform(page->freelist_head);
+ DCHECK(num_new_entries == num_slots - page->num_allocated_slots);
+ // Discard the memory.
+ DiscardSystemPages(begin_ptr, unprovisioned_bytes);
}
- // Terminate the freelist chain.
- *entry_ptr = nullptr;
- // The freelist head is stored unmasked.
- page->freelist_head = PartitionFreelistMask(page->freelist_head);
- DCHECK(num_new_entries == num_slots - page->num_allocated_slots);
- // Discard the memory.
- DiscardSystemPages(begin_ptr, unprovisioned_bytes);
}
- // Next, walk the slots and for any not in use, consider where the system
- // page boundaries occur. We can release any system pages back to the
- // system as long as we don't interfere with a freelist pointer or an
- // adjacent slot.
+ // Next, walk the slots and for any not in use, consider where the system page
+ // boundaries occur. We can release any system pages back to the system as
+ // long as we don't interfere with a freelist pointer or an adjacent slot.
for (size_t i = 0; i < num_slots; ++i) {
if (slot_usage[i])
continue;
// The first address we can safely discard is just after the freelist
- // pointer. There's one quirk: if the freelist pointer is actually a
- // null, we can discard that pointer value too.
+ // pointer. There's one quirk: if the freelist pointer is actually NULL, we
+ // can discard that pointer value too.
char* begin_ptr = ptr + (i * slot_size);
char* end_ptr = begin_ptr + slot_size;
+#if !defined(OS_WIN)
if (i != last_slot)
- begin_ptr += sizeof(PartitionFreelistEntry);
+ begin_ptr += sizeof(internal::PartitionFreelistEntry);
+#else
+ begin_ptr += sizeof(internal::PartitionFreelistEntry);
+#endif
begin_ptr = reinterpret_cast<char*>(
RoundUpToSystemPage(reinterpret_cast<size_t>(begin_ptr)));
end_ptr = reinterpret_cast<char*>(
@@ -1233,32 +487,33 @@
return discardable_bytes;
}
-static void PartitionPurgeBucket(PartitionBucket* bucket) {
- if (bucket->active_pages_head != &PartitionRootGeneric::gSeedPage) {
- for (PartitionPage* page = bucket->active_pages_head; page;
+static void PartitionPurgeBucket(internal::PartitionBucket* bucket) {
+ if (bucket->active_pages_head !=
+ internal::PartitionPage::get_sentinel_page()) {
+ for (internal::PartitionPage* page = bucket->active_pages_head; page;
page = page->next_page) {
- DCHECK(page != &PartitionRootGeneric::gSeedPage);
- (void)PartitionPurgePage(page, true);
+ DCHECK(page != internal::PartitionPage::get_sentinel_page());
+ PartitionPurgePage(page, true);
}
}
}
-void PartitionPurgeMemory(PartitionRoot* root, int flags) {
+void PartitionRoot::PurgeMemory(int flags) {
if (flags & PartitionPurgeDecommitEmptyPages)
- PartitionDecommitEmptyPages(root);
+ DecommitEmptyPages();
// We don't currently do anything for PartitionPurgeDiscardUnusedSystemPages
- // here because that flag is only useful for allocations >= system page
- // size. We only have allocations that large inside generic partitions
- // at the moment.
+ // here because that flag is only useful for allocations >= system page size.
+ // We only have allocations that large inside generic partitions at the
+ // moment.
}
-void PartitionPurgeMemoryGeneric(PartitionRootGeneric* root, int flags) {
- subtle::SpinLock::Guard guard(root->lock);
+void PartitionRootGeneric::PurgeMemory(int flags) {
+ subtle::SpinLock::Guard guard(this->lock);
if (flags & PartitionPurgeDecommitEmptyPages)
- PartitionDecommitEmptyPages(root);
+ DecommitEmptyPages();
if (flags & PartitionPurgeDiscardUnusedSystemPages) {
for (size_t i = 0; i < kGenericNumBuckets; ++i) {
- PartitionBucket* bucket = &root->buckets[i];
+ internal::PartitionBucket* bucket = &this->buckets[i];
if (bucket->slot_size >= kSystemPageSize)
PartitionPurgeBucket(bucket);
}
@@ -1266,47 +521,48 @@
}
static void PartitionDumpPageStats(PartitionBucketMemoryStats* stats_out,
- const PartitionPage* page) {
- uint16_t bucket_num_slots = PartitionBucketSlots(page->bucket);
+ internal::PartitionPage* page) {
+ uint16_t bucket_num_slots = page->bucket->get_slots_per_span();
- if (PartitionPageStateIsDecommitted(page)) {
+ if (page->is_decommitted()) {
++stats_out->num_decommitted_pages;
return;
}
- stats_out->discardable_bytes +=
- PartitionPurgePage(const_cast<PartitionPage*>(page), false);
+ stats_out->discardable_bytes += PartitionPurgePage(page, false);
- size_t raw_size = PartitionPageGetRawSize(const_cast<PartitionPage*>(page));
- if (raw_size)
+ size_t raw_size = page->get_raw_size();
+ if (raw_size) {
stats_out->active_bytes += static_cast<uint32_t>(raw_size);
- else
+ } else {
stats_out->active_bytes +=
(page->num_allocated_slots * stats_out->bucket_slot_size);
+ }
size_t page_bytes_resident =
RoundUpToSystemPage((bucket_num_slots - page->num_unprovisioned_slots) *
stats_out->bucket_slot_size);
stats_out->resident_bytes += page_bytes_resident;
- if (PartitionPageStateIsEmpty(page)) {
+ if (page->is_empty()) {
stats_out->decommittable_bytes += page_bytes_resident;
++stats_out->num_empty_pages;
- } else if (PartitionPageStateIsFull(page)) {
+ } else if (page->is_full()) {
++stats_out->num_full_pages;
} else {
- DCHECK(PartitionPageStateIsActive(page));
+ DCHECK(page->is_active());
++stats_out->num_active_pages;
}
}
static void PartitionDumpBucketStats(PartitionBucketMemoryStats* stats_out,
- const PartitionBucket* bucket) {
- DCHECK(!PartitionBucketIsDirectMapped(bucket));
+ const internal::PartitionBucket* bucket) {
+ DCHECK(!bucket->is_direct_mapped());
stats_out->is_valid = false;
- // If the active page list is empty (== &PartitionRootGeneric::gSeedPage),
- // the bucket might still need to be reported if it has a list of empty,
- // decommitted or full pages.
- if (bucket->active_pages_head == &PartitionRootGeneric::gSeedPage &&
+ // If the active page list is empty (==
+ // internal::PartitionPage::get_sentinel_page()), the bucket might still need
+ // to be reported if it has a list of empty, decommitted or full pages.
+ if (bucket->active_pages_head ==
+ internal::PartitionPage::get_sentinel_page() &&
!bucket->empty_pages_head && !bucket->decommitted_pages_head &&
!bucket->num_full_pages)
return;
@@ -1316,42 +572,41 @@
stats_out->is_direct_map = false;
stats_out->num_full_pages = static_cast<size_t>(bucket->num_full_pages);
stats_out->bucket_slot_size = bucket->slot_size;
- uint16_t bucket_num_slots = PartitionBucketSlots(bucket);
+ uint16_t bucket_num_slots = bucket->get_slots_per_span();
size_t bucket_useful_storage = stats_out->bucket_slot_size * bucket_num_slots;
- stats_out->allocated_page_size = PartitionBucketBytes(bucket);
+ stats_out->allocated_page_size = bucket->get_bytes_per_span();
stats_out->active_bytes = bucket->num_full_pages * bucket_useful_storage;
stats_out->resident_bytes =
bucket->num_full_pages * stats_out->allocated_page_size;
- for (const PartitionPage* page = bucket->empty_pages_head; page;
+ for (internal::PartitionPage* page = bucket->empty_pages_head; page;
page = page->next_page) {
- DCHECK(PartitionPageStateIsEmpty(page) ||
- PartitionPageStateIsDecommitted(page));
+ DCHECK(page->is_empty() || page->is_decommitted());
PartitionDumpPageStats(stats_out, page);
}
- for (const PartitionPage* page = bucket->decommitted_pages_head; page;
+ for (internal::PartitionPage* page = bucket->decommitted_pages_head; page;
page = page->next_page) {
- DCHECK(PartitionPageStateIsDecommitted(page));
+ DCHECK(page->is_decommitted());
PartitionDumpPageStats(stats_out, page);
}
- if (bucket->active_pages_head != &PartitionRootGeneric::gSeedPage) {
- for (const PartitionPage* page = bucket->active_pages_head; page;
+ if (bucket->active_pages_head !=
+ internal::PartitionPage::get_sentinel_page()) {
+ for (internal::PartitionPage* page = bucket->active_pages_head; page;
page = page->next_page) {
- DCHECK(page != &PartitionRootGeneric::gSeedPage);
+ DCHECK(page != internal::PartitionPage::get_sentinel_page());
PartitionDumpPageStats(stats_out, page);
}
}
}
-void PartitionDumpStatsGeneric(PartitionRootGeneric* partition,
- const char* partition_name,
- bool is_light_dump,
- PartitionStatsDumper* dumper) {
+void PartitionRootGeneric::DumpStats(const char* partition_name,
+ bool is_light_dump,
+ PartitionStatsDumper* dumper) {
PartitionMemoryStats stats = {0};
- stats.total_mmapped_bytes = partition->total_size_of_super_pages +
- partition->total_size_of_direct_mapped_pages;
- stats.total_committed_bytes = partition->total_size_of_committed_pages;
+ stats.total_mmapped_bytes =
+ this->total_size_of_super_pages + this->total_size_of_direct_mapped_pages;
+ stats.total_committed_bytes = this->total_size_of_committed_pages;
size_t direct_mapped_allocations_total_size = 0;
@@ -1368,13 +623,13 @@
PartitionBucketMemoryStats bucket_stats[kGenericNumBuckets];
size_t num_direct_mapped_allocations = 0;
{
- subtle::SpinLock::Guard guard(partition->lock);
+ subtle::SpinLock::Guard guard(this->lock);
for (size_t i = 0; i < kGenericNumBuckets; ++i) {
- const PartitionBucket* bucket = &partition->buckets[i];
+ const internal::PartitionBucket* bucket = &this->buckets[i];
// Don't report the pseudo buckets that the generic allocator sets up in
// order to preserve a fast size->bucket map (see
- // PartitionAllocGenericInit for details).
+ // PartitionRootGeneric::Init() for details).
if (!bucket->active_pages_head)
bucket_stats[i].is_valid = false;
else
@@ -1387,7 +642,7 @@
}
}
- for (PartitionDirectMapExtent *extent = partition->direct_map_list;
+ for (internal::PartitionDirectMapExtent *extent = this->direct_map_list;
extent && num_direct_mapped_allocations < kMaxReportableDirectMaps;
extent = extent->next_extent, ++num_direct_mapped_allocations) {
DCHECK(!extent->next_extent ||
@@ -1402,8 +657,8 @@
if (!is_light_dump) {
// Call |PartitionsDumpBucketStats| after collecting stats because it can
- // try to allocate using |PartitionAllocGeneric| and it can't obtain the
- // lock.
+ // try to allocate using |PartitionRootGeneric::Alloc()| and it can't
+ // obtain the lock.
for (size_t i = 0; i < kGenericNumBuckets; ++i) {
if (bucket_stats[i].is_valid)
dumper->PartitionsDumpBucketStats(partition_name, &bucket_stats[i]);
@@ -1412,16 +667,15 @@
for (size_t i = 0; i < num_direct_mapped_allocations; ++i) {
uint32_t size = direct_map_lengths[i];
- PartitionBucketMemoryStats stats;
- memset(&stats, '\0', sizeof(stats));
- stats.is_valid = true;
- stats.is_direct_map = true;
- stats.num_full_pages = 1;
- stats.allocated_page_size = size;
- stats.bucket_slot_size = size;
- stats.active_bytes = size;
- stats.resident_bytes = size;
- dumper->PartitionsDumpBucketStats(partition_name, &stats);
+ PartitionBucketMemoryStats mapped_stats = {};
+ mapped_stats.is_valid = true;
+ mapped_stats.is_direct_map = true;
+ mapped_stats.num_full_pages = 1;
+ mapped_stats.allocated_page_size = size;
+ mapped_stats.bucket_slot_size = size;
+ mapped_stats.active_bytes = size;
+ mapped_stats.resident_bytes = size;
+ dumper->PartitionsDumpBucketStats(partition_name, &mapped_stats);
}
}
@@ -1430,31 +684,46 @@
dumper->PartitionDumpTotals(partition_name, &stats);
}
-void PartitionDumpStats(PartitionRoot* partition,
- const char* partition_name,
- bool is_light_dump,
- PartitionStatsDumper* dumper) {
- static const size_t kMaxReportableBuckets = 4096 / sizeof(void*);
- PartitionBucketMemoryStats memory_stats[kMaxReportableBuckets];
- const size_t partitionNumBuckets = partition->num_buckets;
- DCHECK(partitionNumBuckets <= kMaxReportableBuckets);
-
- for (size_t i = 0; i < partitionNumBuckets; ++i)
- PartitionDumpBucketStats(&memory_stats[i], &partition->buckets()[i]);
-
- // PartitionsDumpBucketStats is called after collecting stats because it
- // can use PartitionAlloc to allocate and this can affect the statistics.
+void PartitionRoot::DumpStats(const char* partition_name,
+ bool is_light_dump,
+ PartitionStatsDumper* dumper) {
PartitionMemoryStats stats = {0};
- stats.total_mmapped_bytes = partition->total_size_of_super_pages;
- stats.total_committed_bytes = partition->total_size_of_committed_pages;
- DCHECK(!partition->total_size_of_direct_mapped_pages);
- for (size_t i = 0; i < partitionNumBuckets; ++i) {
- if (memory_stats[i].is_valid) {
- stats.total_resident_bytes += memory_stats[i].resident_bytes;
- stats.total_active_bytes += memory_stats[i].active_bytes;
- stats.total_decommittable_bytes += memory_stats[i].decommittable_bytes;
- stats.total_discardable_bytes += memory_stats[i].discardable_bytes;
- if (!is_light_dump)
+ stats.total_mmapped_bytes = this->total_size_of_super_pages;
+ stats.total_committed_bytes = this->total_size_of_committed_pages;
+ DCHECK(!this->total_size_of_direct_mapped_pages);
+
+ static constexpr size_t kMaxReportableBuckets = 4096 / sizeof(void*);
+ std::unique_ptr<PartitionBucketMemoryStats[]> memory_stats;
+ if (!is_light_dump) {
+ memory_stats = std::unique_ptr<PartitionBucketMemoryStats[]>(
+ new PartitionBucketMemoryStats[kMaxReportableBuckets]);
+ }
+
+ const size_t partition_num_buckets = this->num_buckets;
+ DCHECK(partition_num_buckets <= kMaxReportableBuckets);
+
+ for (size_t i = 0; i < partition_num_buckets; ++i) {
+ PartitionBucketMemoryStats bucket_stats = {0};
+ PartitionDumpBucketStats(&bucket_stats, &this->buckets()[i]);
+ if (bucket_stats.is_valid) {
+ stats.total_resident_bytes += bucket_stats.resident_bytes;
+ stats.total_active_bytes += bucket_stats.active_bytes;
+ stats.total_decommittable_bytes += bucket_stats.decommittable_bytes;
+ stats.total_discardable_bytes += bucket_stats.discardable_bytes;
+ }
+ if (!is_light_dump) {
+ if (bucket_stats.is_valid)
+ memory_stats[i] = bucket_stats;
+ else
+ memory_stats[i].is_valid = false;
+ }
+ }
+ if (!is_light_dump) {
+ // PartitionsDumpBucketStats is called after collecting stats because it
+ // can use PartitionRoot::Alloc() to allocate and this can affect the
+ // statistics.
+ for (size_t i = 0; i < partition_num_buckets; ++i) {
+ if (memory_stats[i].is_valid)
dumper->PartitionsDumpBucketStats(partition_name, &memory_stats[i]);
}
}
diff --git a/third_party/base/allocator/partition_allocator/partition_alloc.h b/third_party/base/allocator/partition_allocator/partition_alloc.h
index 69fba97..a80755c 100644
--- a/third_party/base/allocator/partition_allocator/partition_alloc.h
+++ b/third_party/base/allocator/partition_allocator/partition_alloc.h
@@ -2,12 +2,13 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-#ifndef BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ALLOC_H
-#define BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ALLOC_H
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ALLOC_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ALLOC_H_
// DESCRIPTION
-// partitionAlloc() / PartitionAllocGeneric() and PartitionFree() /
-// PartitionFreeGeneric() are approximately analagous to malloc() and free().
+// PartitionRoot::Alloc() / PartitionRootGeneric::Alloc() and PartitionFree() /
+// PartitionRootGeneric::Free() are approximately analagous to malloc() and
+// free().
//
// The main difference is that a PartitionRoot / PartitionRootGeneric object
// must be supplied to these functions, representing a specific "heap partition"
@@ -23,14 +24,14 @@
// PartitionRoot is really just a header adjacent to other data areas provided
// by the allocator class.
//
-// The partitionAlloc() variant of the API has the following caveats:
+// The PartitionRoot::Alloc() variant of the API has the following caveats:
// - Allocations and frees against a single partition must be single threaded.
// - Allocations must not exceed a max size, chosen at compile-time via a
// templated parameter to PartitionAllocator.
// - Allocation sizes must be aligned to the system pointer size.
// - Allocations are bucketed exactly according to size.
//
-// And for PartitionAllocGeneric():
+// And for PartitionRootGeneric::Alloc():
// - Multi-threaded use against a single partition is ok; locking is handled.
// - Allocations of any arbitrary size can be handled (subject to a limit of
// INT_MAX bytes for security reasons).
@@ -64,10 +65,17 @@
#include "build/build_config.h"
#include "third_party/base/allocator/partition_allocator/page_allocator.h"
+#include "third_party/base/allocator/partition_allocator/partition_alloc_constants.h"
+#include "third_party/base/allocator/partition_allocator/partition_bucket.h"
+#include "third_party/base/allocator/partition_allocator/partition_cookie.h"
+#include "third_party/base/allocator/partition_allocator/partition_page.h"
+#include "third_party/base/allocator/partition_allocator/partition_root_base.h"
#include "third_party/base/allocator/partition_allocator/spin_lock.h"
+#include "third_party/base/base_export.h"
#include "third_party/base/bits.h"
#include "third_party/base/compiler_specific.h"
#include "third_party/base/logging.h"
+#include "third_party/base/stl_util.h"
#include "third_party/base/sys_byteorder.h"
#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
@@ -77,289 +85,83 @@
namespace pdfium {
namespace base {
-// Allocation granularity of sizeof(void*) bytes.
-static const size_t kAllocationGranularity = sizeof(void*);
-static const size_t kAllocationGranularityMask = kAllocationGranularity - 1;
-static const size_t kBucketShift = (kAllocationGranularity == 8) ? 3 : 2;
+class PartitionStatsDumper;
-// Underlying partition storage pages are a power-of-two size. It is typical
-// for a partition page to be based on multiple system pages. Most references to
-// "page" refer to partition pages.
-// We also have the concept of "super pages" -- these are the underlying system
-// allocations we make. Super pages contain multiple partition pages inside them
-// and include space for a small amount of metadata per partition page.
-// Inside super pages, we store "slot spans". A slot span is a continguous range
-// of one or more partition pages that stores allocations of the same size.
-// Slot span sizes are adjusted depending on the allocation size, to make sure
-// the packing does not lead to unused (wasted) space at the end of the last
-// system page of the span. For our current max slot span size of 64k and other
-// constant values, we pack _all_ PartitionAllocGeneric() sizes perfectly up
-// against the end of a system page.
-#if defined(_MIPS_ARCH_LOONGSON)
-static const size_t kPartitionPageShift = 16; // 64KB
-#else
-static const size_t kPartitionPageShift = 14; // 16KB
-#endif
-static const size_t kPartitionPageSize = 1 << kPartitionPageShift;
-static const size_t kPartitionPageOffsetMask = kPartitionPageSize - 1;
-static const size_t kPartitionPageBaseMask = ~kPartitionPageOffsetMask;
-static const size_t kMaxPartitionPagesPerSlotSpan = 4;
-
-// To avoid fragmentation via never-used freelist entries, we hand out partition
-// freelist sections gradually, in units of the dominant system page size.
-// What we're actually doing is avoiding filling the full partition page (16 KB)
-// with freelist pointers right away. Writing freelist pointers will fault and
-// dirty a private page, which is very wasteful if we never actually store
-// objects there.
-static const size_t kNumSystemPagesPerPartitionPage =
- kPartitionPageSize / kSystemPageSize;
-static const size_t kMaxSystemPagesPerSlotSpan =
- kNumSystemPagesPerPartitionPage * kMaxPartitionPagesPerSlotSpan;
-
-// We reserve virtual address space in 2MB chunks (aligned to 2MB as well).
-// These chunks are called "super pages". We do this so that we can store
-// metadata in the first few pages of each 2MB aligned section. This leads to
-// a very fast free(). We specifically choose 2MB because this virtual address
-// block represents a full but single PTE allocation on ARM, ia32 and x64.
-//
-// The layout of the super page is as follows. The sizes below are the same
-// for 32 bit and 64 bit.
-//
-// | Guard page (4KB) |
-// | Metadata page (4KB) |
-// | Guard pages (8KB) |
-// | Slot span |
-// | Slot span |
-// | ... |
-// | Slot span |
-// | Guard page (4KB) |
-//
-// - Each slot span is a contiguous range of one or more PartitionPages.
-// - The metadata page has the following format. Note that the PartitionPage
-// that is not at the head of a slot span is "unused". In other words,
-// the metadata for the slot span is stored only in the first PartitionPage
-// of the slot span. Metadata accesses to other PartitionPages are
-// redirected to the first PartitionPage.
-//
-// | SuperPageExtentEntry (32B) |
-// | PartitionPage of slot span 1 (32B, used) |
-// | PartitionPage of slot span 1 (32B, unused) |
-// | PartitionPage of slot span 1 (32B, unused) |
-// | PartitionPage of slot span 2 (32B, used) |
-// | PartitionPage of slot span 3 (32B, used) |
-// | ... |
-// | PartitionPage of slot span N (32B, unused) |
-//
-// A direct mapped page has a similar layout to fake it looking like a super
-// page:
-//
-// | Guard page (4KB) |
-// | Metadata page (4KB) |
-// | Guard pages (8KB) |
-// | Direct mapped object |
-// | Guard page (4KB) |
-//
-// - The metadata page has the following layout:
-//
-// | SuperPageExtentEntry (32B) |
-// | PartitionPage (32B) |
-// | PartitionBucket (32B) |
-// | PartitionDirectMapExtent (8B) |
-static const size_t kSuperPageShift = 21; // 2MB
-static const size_t kSuperPageSize = 1 << kSuperPageShift;
-static const size_t kSuperPageOffsetMask = kSuperPageSize - 1;
-static const size_t kSuperPageBaseMask = ~kSuperPageOffsetMask;
-static const size_t kNumPartitionPagesPerSuperPage =
- kSuperPageSize / kPartitionPageSize;
-
-static const size_t kPageMetadataShift = 5; // 32 bytes per partition page.
-static const size_t kPageMetadataSize = 1 << kPageMetadataShift;
-
-// The following kGeneric* constants apply to the generic variants of the API.
-// The "order" of an allocation is closely related to the power-of-two size of
-// the allocation. More precisely, the order is the bit index of the
-// most-significant-bit in the allocation size, where the bit numbers starts
-// at index 1 for the least-significant-bit.
-// In terms of allocation sizes, order 0 covers 0, order 1 covers 1, order 2
-// covers 2->3, order 3 covers 4->7, order 4 covers 8->15.
-static const size_t kGenericMinBucketedOrder = 4; // 8 bytes.
-static const size_t kGenericMaxBucketedOrder =
- 20; // Largest bucketed order is 1<<(20-1) (storing 512KB -> almost 1MB)
-static const size_t kGenericNumBucketedOrders =
- (kGenericMaxBucketedOrder - kGenericMinBucketedOrder) + 1;
-// Eight buckets per order (for the higher orders), e.g. order 8 is 128, 144,
-// 160, ..., 240:
-static const size_t kGenericNumBucketsPerOrderBits = 3;
-static const size_t kGenericNumBucketsPerOrder =
- 1 << kGenericNumBucketsPerOrderBits;
-static const size_t kGenericNumBuckets =
- kGenericNumBucketedOrders * kGenericNumBucketsPerOrder;
-static const size_t kGenericSmallestBucket = 1
- << (kGenericMinBucketedOrder - 1);
-static const size_t kGenericMaxBucketSpacing =
- 1 << ((kGenericMaxBucketedOrder - 1) - kGenericNumBucketsPerOrderBits);
-static const size_t kGenericMaxBucketed =
- (1 << (kGenericMaxBucketedOrder - 1)) +
- ((kGenericNumBucketsPerOrder - 1) * kGenericMaxBucketSpacing);
-static const size_t kGenericMinDirectMappedDownsize =
- kGenericMaxBucketed +
- 1; // Limit when downsizing a direct mapping using realloc().
-static const size_t kGenericMaxDirectMapped = INT_MAX - kSystemPageSize;
-static const size_t kBitsPerSizeT = sizeof(void*) * CHAR_BIT;
-
-// Constants for the memory reclaim logic.
-static const size_t kMaxFreeableSpans = 16;
-
-// If the total size in bytes of allocated but not committed pages exceeds this
-// value (probably it is a "out of virtual address space" crash),
-// a special crash stack trace is generated at |partitionOutOfMemory|.
-// This is to distinguish "out of virtual address space" from
-// "out of physical memory" in crash reports.
-static const size_t kReasonableSizeOfUnusedPages = 1024 * 1024 * 1024; // 1GiB
-
-#if DCHECK_IS_ON()
-// These two byte values match tcmalloc.
-static const unsigned char kUninitializedByte = 0xAB;
-static const unsigned char kFreedByte = 0xCD;
-static const size_t kCookieSize =
- 16; // Handles alignment up to XMM instructions on Intel.
-static const unsigned char kCookieValue[kCookieSize] = {
- 0xDE, 0xAD, 0xBE, 0xEF, 0xCA, 0xFE, 0xD0, 0x0D,
- 0x13, 0x37, 0xF0, 0x05, 0xBA, 0x11, 0xAB, 0x1E};
-#endif
-
-struct PartitionBucket;
-struct PartitionRootBase;
-
-struct PartitionFreelistEntry {
- PartitionFreelistEntry* next;
-};
-
-// Some notes on page states. A page can be in one of four major states:
-// 1) Active.
-// 2) Full.
-// 3) Empty.
-// 4) Decommitted.
-// An active page has available free slots. A full page has no free slots. An
-// empty page has no free slots, and a decommitted page is an empty page that
-// had its backing memory released back to the system.
-// There are two linked lists tracking the pages. The "active page" list is an
-// approximation of a list of active pages. It is an approximation because
-// full, empty and decommitted pages may briefly be present in the list until
-// we next do a scan over it.
-// The "empty page" list is an accurate list of pages which are either empty
-// or decommitted.
-//
-// The significant page transitions are:
-// - free() will detect when a full page has a slot free()'d and immediately
-// return the page to the head of the active list.
-// - free() will detect when a page is fully emptied. It _may_ add it to the
-// empty list or it _may_ leave it on the active list until a future list scan.
-// - malloc() _may_ scan the active page list in order to fulfil the request.
-// If it does this, full, empty and decommitted pages encountered will be
-// booted out of the active list. If there are no suitable active pages found,
-// an empty or decommitted page (if one exists) will be pulled from the empty
-// list on to the active list.
-struct PartitionPage {
- PartitionFreelistEntry* freelist_head;
- PartitionPage* next_page;
- PartitionBucket* bucket;
- // Deliberately signed, 0 for empty or decommitted page, -n for full pages:
- int16_t num_allocated_slots;
- uint16_t num_unprovisioned_slots;
- uint16_t page_offset;
- int16_t empty_cache_index; // -1 if not in the empty cache.
-};
-
-struct PartitionBucket {
- PartitionPage* active_pages_head; // Accessed most in hot path => goes first.
- PartitionPage* empty_pages_head;
- PartitionPage* decommitted_pages_head;
- uint32_t slot_size;
- unsigned num_system_pages_per_slot_span : 8;
- unsigned num_full_pages : 24;
-};
-
-// An "extent" is a span of consecutive superpages. We link to the partition's
-// next extent (if there is one) at the very start of a superpage's metadata
-// area.
-struct PartitionSuperPageExtentEntry {
- PartitionRootBase* root;
- char* super_page_base;
- char* super_pages_end;
- PartitionSuperPageExtentEntry* next;
-};
-
-struct PartitionDirectMapExtent {
- PartitionDirectMapExtent* next_extent;
- PartitionDirectMapExtent* prev_extent;
- PartitionBucket* bucket;
- size_t map_size; // Mapped size, not including guard pages and meta-data.
-};
-
-struct BASE_EXPORT PartitionRootBase {
- size_t total_size_of_committed_pages;
- size_t total_size_of_super_pages;
- size_t total_size_of_direct_mapped_pages;
- // Invariant: total_size_of_committed_pages <=
- // total_size_of_super_pages +
- // total_size_of_direct_mapped_pages.
- unsigned num_buckets;
- unsigned max_allocation;
- bool initialized;
- char* next_super_page;
- char* next_partition_page;
- char* next_partition_page_end;
- PartitionSuperPageExtentEntry* current_extent;
- PartitionSuperPageExtentEntry* first_extent;
- PartitionDirectMapExtent* direct_map_list;
- PartitionPage* global_empty_page_ring[kMaxFreeableSpans];
- int16_t global_empty_page_ring_index;
- uintptr_t inverted_self;
-
- static subtle::SpinLock gInitializedLock;
- static bool gInitialized;
- // gSeedPage is used as a sentinel to indicate that there is no page
- // in the active page list. We can use nullptr, but in that case we need
- // to add a null-check branch to the hot allocation path. We want to avoid
- // that.
- static PartitionPage gSeedPage;
- static PartitionBucket gPagedBucket;
- // gOomHandlingFunction is invoked when ParitionAlloc hits OutOfMemory.
- static void (*gOomHandlingFunction)();
+enum PartitionPurgeFlags {
+ // Decommitting the ring list of empty pages is reasonably fast.
+ PartitionPurgeDecommitEmptyPages = 1 << 0,
+ // Discarding unused system pages is slower, because it involves walking all
+ // freelists in all active partition pages of all buckets >= system page
+ // size. It often frees a similar amount of memory to decommitting the empty
+ // pages, though.
+ PartitionPurgeDiscardUnusedSystemPages = 1 << 1,
};
// Never instantiate a PartitionRoot directly, instead use PartitionAlloc.
-struct PartitionRoot : public PartitionRootBase {
+struct BASE_EXPORT PartitionRoot : public internal::PartitionRootBase {
+ PartitionRoot();
+ ~PartitionRoot() override;
+ // This references the buckets OFF the edge of this struct. All uses of
+ // PartitionRoot must have the bucket array come right after.
+ //
// The PartitionAlloc templated class ensures the following is correct.
- ALWAYS_INLINE PartitionBucket* buckets() {
- return reinterpret_cast<PartitionBucket*>(this + 1);
+ ALWAYS_INLINE internal::PartitionBucket* buckets() {
+ return reinterpret_cast<internal::PartitionBucket*>(this + 1);
}
- ALWAYS_INLINE const PartitionBucket* buckets() const {
- return reinterpret_cast<const PartitionBucket*>(this + 1);
+ ALWAYS_INLINE const internal::PartitionBucket* buckets() const {
+ return reinterpret_cast<const internal::PartitionBucket*>(this + 1);
}
+
+ void Init(size_t num_buckets, size_t max_allocation);
+
+ ALWAYS_INLINE void* Alloc(size_t size, const char* type_name);
+ ALWAYS_INLINE void* AllocFlags(int flags, size_t size, const char* type_name);
+
+ void PurgeMemory(int flags);
+
+ void DumpStats(const char* partition_name,
+ bool is_light_dump,
+ PartitionStatsDumper* dumper);
};
// Never instantiate a PartitionRootGeneric directly, instead use
// PartitionAllocatorGeneric.
-struct PartitionRootGeneric : public PartitionRootBase {
+struct BASE_EXPORT PartitionRootGeneric : public internal::PartitionRootBase {
+ PartitionRootGeneric();
+ ~PartitionRootGeneric() override;
subtle::SpinLock lock;
// Some pre-computed constants.
- size_t order_index_shifts[kBitsPerSizeT + 1];
- size_t order_sub_index_masks[kBitsPerSizeT + 1];
+ size_t order_index_shifts[kBitsPerSizeT + 1] = {};
+ size_t order_sub_index_masks[kBitsPerSizeT + 1] = {};
// The bucket lookup table lets us map a size_t to a bucket quickly.
// The trailing +1 caters for the overflow case for very large allocation
// sizes. It is one flat array instead of a 2D array because in the 2D
// world, we'd need to index array[blah][max+1] which risks undefined
// behavior.
- PartitionBucket*
- bucket_lookups[((kBitsPerSizeT + 1) * kGenericNumBucketsPerOrder) + 1];
- PartitionBucket buckets[kGenericNumBuckets];
-};
+ internal::PartitionBucket*
+ bucket_lookups[((kBitsPerSizeT + 1) * kGenericNumBucketsPerOrder) + 1] =
+ {};
+ internal::PartitionBucket buckets[kGenericNumBuckets] = {};
-// Flags for PartitionAllocGenericFlags.
-enum PartitionAllocFlags {
- PartitionAllocReturnNull = 1 << 0,
+ // Public API.
+ void Init();
+
+ ALWAYS_INLINE void* Alloc(size_t size, const char* type_name);
+ ALWAYS_INLINE void* AllocFlags(int flags, size_t size, const char* type_name);
+ ALWAYS_INLINE void Free(void* ptr);
+
+ NOINLINE void* Realloc(void* ptr, size_t new_size, const char* type_name);
+ // Overload that may return nullptr if reallocation isn't possible. In this
+ // case, |ptr| remains valid.
+ NOINLINE void* TryRealloc(void* ptr, size_t new_size, const char* type_name);
+
+ ALWAYS_INLINE size_t ActualSize(size_t size);
+
+ void PurgeMemory(int flags);
+
+ void DumpStats(const char* partition_name,
+ bool is_light_dump,
+ PartitionStatsDumper* partition_stats_dumper);
};
// Struct used to retrieve total memory usage of a partition. Used by
@@ -409,58 +211,24 @@
};
BASE_EXPORT void PartitionAllocGlobalInit(void (*oom_handling_function)());
-BASE_EXPORT void PartitionAllocInit(PartitionRoot*,
- size_t num_buckets,
- size_t max_allocation);
-BASE_EXPORT void PartitionAllocGenericInit(PartitionRootGeneric*);
-
-enum PartitionPurgeFlags {
- // Decommitting the ring list of empty pages is reasonably fast.
- PartitionPurgeDecommitEmptyPages = 1 << 0,
- // Discarding unused system pages is slower, because it involves walking all
- // freelists in all active partition pages of all buckets >= system page
- // size. It often frees a similar amount of memory to decommitting the empty
- // pages, though.
- PartitionPurgeDiscardUnusedSystemPages = 1 << 1,
-};
-
-BASE_EXPORT void PartitionPurgeMemory(PartitionRoot*, int);
-BASE_EXPORT void PartitionPurgeMemoryGeneric(PartitionRootGeneric*, int);
-
-BASE_EXPORT NOINLINE void* PartitionAllocSlowPath(PartitionRootBase*,
- int,
- size_t,
- PartitionBucket*);
-BASE_EXPORT NOINLINE void PartitionFreeSlowPath(PartitionPage*);
-BASE_EXPORT NOINLINE void* PartitionReallocGenericFlags(
- PartitionRootGeneric* root,
- int flags,
- void* ptr,
- size_t new_size,
- const char* type_name);
-BASE_EXPORT NOINLINE void* PartitionReallocGeneric(PartitionRootGeneric* root,
- void* ptr,
- size_t new_size,
- const char* type_name);
-
-BASE_EXPORT void PartitionDumpStats(PartitionRoot*,
- const char* partition_name,
- bool is_light_dump,
- PartitionStatsDumper*);
-BASE_EXPORT void PartitionDumpStatsGeneric(PartitionRootGeneric*,
- const char* partition_name,
- bool is_light_dump,
- PartitionStatsDumper*);
class BASE_EXPORT PartitionAllocHooks {
public:
typedef void AllocationHook(void* address, size_t, const char* type_name);
typedef void FreeHook(void* address);
+ // To unhook, call Set*Hook with nullptr.
static void SetAllocationHook(AllocationHook* hook) {
+ // Chained allocation hooks are not supported. Registering a non-null
+ // hook when a non-null hook is already registered indicates somebody is
+ // trying to overwrite a hook.
+ CHECK(!hook || !allocation_hook_);
allocation_hook_ = hook;
}
- static void SetFreeHook(FreeHook* hook) { free_hook_ = hook; }
+ static void SetFreeHook(FreeHook* hook) {
+ CHECK(!hook || !free_hook_);
+ free_hook_ = hook;
+ }
static void AllocationHookIfEnabled(void* address,
size_t size,
@@ -496,375 +264,32 @@
static FreeHook* free_hook_;
};
-ALWAYS_INLINE PartitionFreelistEntry* PartitionFreelistMask(
- PartitionFreelistEntry* ptr) {
-// We use bswap on little endian as a fast mask for two reasons:
-// 1) If an object is freed and its vtable used where the attacker doesn't
-// get the chance to run allocations between the free and use, the vtable
-// dereference is likely to fault.
-// 2) If the attacker has a linear buffer overflow and elects to try and
-// corrupt a freelist pointer, partial pointer overwrite attacks are
-// thwarted.
-// For big endian, similar guarantees are arrived at with a negation.
-#if defined(ARCH_CPU_BIG_ENDIAN)
- uintptr_t masked = ~reinterpret_cast<uintptr_t>(ptr);
-#else
- uintptr_t masked = ByteSwapUintPtrT(reinterpret_cast<uintptr_t>(ptr));
-#endif
- return reinterpret_cast<PartitionFreelistEntry*>(masked);
+ALWAYS_INLINE void* PartitionRoot::Alloc(size_t size, const char* type_name) {
+ return AllocFlags(0, size, type_name);
}
-ALWAYS_INLINE size_t PartitionCookieSizeAdjustAdd(size_t size) {
-#if DCHECK_IS_ON()
- // Add space for cookies, checking for integer overflow. TODO(palmer):
- // Investigate the performance and code size implications of using
- // CheckedNumeric throughout PA.
- DCHECK(size + (2 * kCookieSize) > size);
- size += 2 * kCookieSize;
-#endif
- return size;
-}
-
-ALWAYS_INLINE size_t PartitionCookieSizeAdjustSubtract(size_t size) {
-#if DCHECK_IS_ON()
- // Remove space for cookies.
- DCHECK(size >= 2 * kCookieSize);
- size -= 2 * kCookieSize;
-#endif
- return size;
-}
-
-ALWAYS_INLINE void* PartitionCookieFreePointerAdjust(void* ptr) {
-#if DCHECK_IS_ON()
- // The value given to the application is actually just after the cookie.
- ptr = static_cast<char*>(ptr) - kCookieSize;
-#endif
- return ptr;
-}
-
-ALWAYS_INLINE void PartitionCookieWriteValue(void* ptr) {
-#if DCHECK_IS_ON()
- auto* cookie_ptr = reinterpret_cast<unsigned char*>(ptr);
- for (size_t i = 0; i < kCookieSize; ++i, ++cookie_ptr)
- *cookie_ptr = kCookieValue[i];
-#endif
-}
-
-ALWAYS_INLINE void PartitionCookieCheckValue(void* ptr) {
-#if DCHECK_IS_ON()
- auto* cookie_ptr = reinterpret_cast<unsigned char*>(ptr);
- for (size_t i = 0; i < kCookieSize; ++i, ++cookie_ptr)
- DCHECK(*cookie_ptr == kCookieValue[i]);
-#endif
-}
-
-ALWAYS_INLINE char* PartitionSuperPageToMetadataArea(char* ptr) {
- auto pointer_as_uint = reinterpret_cast<uintptr_t>(ptr);
- DCHECK(!(pointer_as_uint & kSuperPageOffsetMask));
- // The metadata area is exactly one system page (the guard page) into the
- // super page.
- return reinterpret_cast<char*>(pointer_as_uint + kSystemPageSize);
-}
-
-ALWAYS_INLINE PartitionPage* PartitionPointerToPageNoAlignmentCheck(void* ptr) {
- auto pointer_as_uint = reinterpret_cast<uintptr_t>(ptr);
- auto* super_page_ptr =
- reinterpret_cast<char*>(pointer_as_uint & kSuperPageBaseMask);
- uintptr_t partition_page_index =
- (pointer_as_uint & kSuperPageOffsetMask) >> kPartitionPageShift;
- // Index 0 is invalid because it is the metadata and guard area and
- // the last index is invalid because it is a guard page.
- DCHECK(partition_page_index);
- DCHECK(partition_page_index < kNumPartitionPagesPerSuperPage - 1);
- auto* page = reinterpret_cast<PartitionPage*>(
- PartitionSuperPageToMetadataArea(super_page_ptr) +
- (partition_page_index << kPageMetadataShift));
- // Partition pages in the same slot span can share the same page object.
- // Adjust for that.
- size_t delta = page->page_offset << kPageMetadataShift;
- page =
- reinterpret_cast<PartitionPage*>(reinterpret_cast<char*>(page) - delta);
- return page;
-}
-
-ALWAYS_INLINE void* PartitionPageToPointer(const PartitionPage* page) {
- auto pointer_as_uint = reinterpret_cast<uintptr_t>(page);
- uintptr_t super_page_offset = (pointer_as_uint & kSuperPageOffsetMask);
- DCHECK(super_page_offset > kSystemPageSize);
- DCHECK(super_page_offset < kSystemPageSize + (kNumPartitionPagesPerSuperPage *
- kPageMetadataSize));
- uintptr_t partition_page_index =
- (super_page_offset - kSystemPageSize) >> kPageMetadataShift;
- // Index 0 is invalid because it is the metadata area and the last index is
- // invalid because it is a guard page.
- DCHECK(partition_page_index);
- DCHECK(partition_page_index < kNumPartitionPagesPerSuperPage - 1);
- uintptr_t super_page_base = (pointer_as_uint & kSuperPageBaseMask);
- auto* ret = reinterpret_cast<void*>(
- super_page_base + (partition_page_index << kPartitionPageShift));
- return ret;
-}
-
-ALWAYS_INLINE PartitionPage* PartitionPointerToPage(void* ptr) {
- PartitionPage* page = PartitionPointerToPageNoAlignmentCheck(ptr);
- // Checks that the pointer is a multiple of bucket size.
- DCHECK(!((reinterpret_cast<uintptr_t>(ptr) -
- reinterpret_cast<uintptr_t>(PartitionPageToPointer(page))) %
- page->bucket->slot_size));
- return page;
-}
-
-ALWAYS_INLINE bool PartitionBucketIsDirectMapped(
- const PartitionBucket* bucket) {
- return !bucket->num_system_pages_per_slot_span;
-}
-
-ALWAYS_INLINE size_t PartitionBucketBytes(const PartitionBucket* bucket) {
- return bucket->num_system_pages_per_slot_span * kSystemPageSize;
-}
-
-ALWAYS_INLINE uint16_t PartitionBucketSlots(const PartitionBucket* bucket) {
- return static_cast<uint16_t>(PartitionBucketBytes(bucket) /
- bucket->slot_size);
-}
-
-ALWAYS_INLINE size_t* PartitionPageGetRawSizePtr(PartitionPage* page) {
- // For single-slot buckets which span more than one partition page, we
- // have some spare metadata space to store the raw allocation size. We
- // can use this to report better statistics.
- PartitionBucket* bucket = page->bucket;
- if (bucket->slot_size <= kMaxSystemPagesPerSlotSpan * kSystemPageSize)
- return nullptr;
-
- DCHECK((bucket->slot_size % kSystemPageSize) == 0);
- DCHECK(PartitionBucketIsDirectMapped(bucket) ||
- PartitionBucketSlots(bucket) == 1);
- page++;
- return reinterpret_cast<size_t*>(&page->freelist_head);
-}
-
-ALWAYS_INLINE size_t PartitionPageGetRawSize(PartitionPage* page) {
- size_t* raw_size_ptr = PartitionPageGetRawSizePtr(page);
- if (UNLIKELY(raw_size_ptr != nullptr))
- return *raw_size_ptr;
- return 0;
-}
-
-ALWAYS_INLINE PartitionRootBase* PartitionPageToRoot(PartitionPage* page) {
- auto* extent_entry = reinterpret_cast<PartitionSuperPageExtentEntry*>(
- reinterpret_cast<uintptr_t>(page) & kSystemPageBaseMask);
- return extent_entry->root;
-}
-
-ALWAYS_INLINE bool PartitionPointerIsValid(void* ptr) {
- PartitionPage* page = PartitionPointerToPage(ptr);
- PartitionRootBase* root = PartitionPageToRoot(page);
- return root->inverted_self == ~reinterpret_cast<uintptr_t>(root);
-}
-
-ALWAYS_INLINE void* PartitionBucketAlloc(PartitionRootBase* root,
- int flags,
- size_t size,
- PartitionBucket* bucket) {
- PartitionPage* page = bucket->active_pages_head;
- // Check that this page is neither full nor freed.
- DCHECK(page->num_allocated_slots >= 0);
- void* ret = page->freelist_head;
- if (LIKELY(ret)) {
- // If these asserts fire, you probably corrupted memory.
- DCHECK(PartitionPointerIsValid(ret));
- // All large allocations must go through the slow path to correctly
- // update the size metadata.
- DCHECK(PartitionPageGetRawSize(page) == 0);
- PartitionFreelistEntry* new_head =
- PartitionFreelistMask(static_cast<PartitionFreelistEntry*>(ret)->next);
- page->freelist_head = new_head;
- page->num_allocated_slots++;
- } else {
- ret = PartitionAllocSlowPath(root, flags, size, bucket);
- DCHECK(!ret || PartitionPointerIsValid(ret));
- }
-#if DCHECK_IS_ON()
- if (!ret)
- return nullptr;
- // Fill the uninitialized pattern, and write the cookies.
- page = PartitionPointerToPage(ret);
- size_t slot_size = page->bucket->slot_size;
- size_t raw_size = PartitionPageGetRawSize(page);
- if (raw_size) {
- DCHECK(raw_size == size);
- slot_size = raw_size;
- }
- size_t no_cookie_size = PartitionCookieSizeAdjustSubtract(slot_size);
- auto* char_ret = static_cast<char*>(ret);
- // The value given to the application is actually just after the cookie.
- ret = char_ret + kCookieSize;
- memset(ret, kUninitializedByte, no_cookie_size);
- PartitionCookieWriteValue(char_ret);
- PartitionCookieWriteValue(char_ret + kCookieSize + no_cookie_size);
-#endif
- return ret;
-}
-
-ALWAYS_INLINE void* PartitionAlloc(PartitionRoot* root,
- size_t size,
- const char* type_name) {
+ALWAYS_INLINE void* PartitionRoot::AllocFlags(int flags,
+ size_t size,
+ const char* type_name) {
#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
void* result = malloc(size);
CHECK(result);
return result;
#else
size_t requested_size = size;
- size = PartitionCookieSizeAdjustAdd(size);
- DCHECK(root->initialized);
+ size = internal::PartitionCookieSizeAdjustAdd(size);
+ DCHECK(this->initialized);
size_t index = size >> kBucketShift;
- DCHECK(index < root->num_buckets);
+ DCHECK(index < this->num_buckets);
DCHECK(size == index << kBucketShift);
- PartitionBucket* bucket = &root->buckets()[index];
- void* result = PartitionBucketAlloc(root, 0, size, bucket);
+ internal::PartitionBucket* bucket = &this->buckets()[index];
+ void* result = AllocFromBucket(bucket, flags, size);
PartitionAllocHooks::AllocationHookIfEnabled(result, requested_size,
type_name);
return result;
#endif // defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
}
-ALWAYS_INLINE void PartitionFreeWithPage(void* ptr, PartitionPage* page) {
-// If these asserts fire, you probably corrupted memory.
-#if DCHECK_IS_ON()
- size_t slot_size = page->bucket->slot_size;
- size_t raw_size = PartitionPageGetRawSize(page);
- if (raw_size)
- slot_size = raw_size;
- PartitionCookieCheckValue(ptr);
- PartitionCookieCheckValue(reinterpret_cast<char*>(ptr) + slot_size -
- kCookieSize);
- memset(ptr, kFreedByte, slot_size);
-#endif
- DCHECK(page->num_allocated_slots);
- PartitionFreelistEntry* freelist_head = page->freelist_head;
- DCHECK(!freelist_head || PartitionPointerIsValid(freelist_head));
- CHECK(ptr != freelist_head); // Catches an immediate double free.
- // Look for double free one level deeper in debug.
- DCHECK(!freelist_head || ptr != PartitionFreelistMask(freelist_head->next));
- auto* entry = static_cast<PartitionFreelistEntry*>(ptr);
- entry->next = PartitionFreelistMask(freelist_head);
- page->freelist_head = entry;
- --page->num_allocated_slots;
- if (UNLIKELY(page->num_allocated_slots <= 0)) {
- PartitionFreeSlowPath(page);
- } else {
- // All single-slot allocations must go through the slow path to
- // correctly update the size metadata.
- DCHECK(PartitionPageGetRawSize(page) == 0);
- }
-}
-
-ALWAYS_INLINE void PartitionFree(void* ptr) {
-#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
- free(ptr);
-#else
- PartitionAllocHooks::FreeHookIfEnabled(ptr);
- ptr = PartitionCookieFreePointerAdjust(ptr);
- DCHECK(PartitionPointerIsValid(ptr));
- PartitionPage* page = PartitionPointerToPage(ptr);
- PartitionFreeWithPage(ptr, page);
-#endif
-}
-
-ALWAYS_INLINE PartitionBucket* PartitionGenericSizeToBucket(
- PartitionRootGeneric* root,
- size_t size) {
- size_t order = kBitsPerSizeT - bits::CountLeadingZeroBitsSizeT(size);
- // The order index is simply the next few bits after the most significant bit.
- size_t order_index = (size >> root->order_index_shifts[order]) &
- (kGenericNumBucketsPerOrder - 1);
- // And if the remaining bits are non-zero we must bump the bucket up.
- size_t sub_order_index = size & root->order_sub_index_masks[order];
- PartitionBucket* bucket =
- root->bucket_lookups[(order << kGenericNumBucketsPerOrderBits) +
- order_index + !!sub_order_index];
- DCHECK(!bucket->slot_size || bucket->slot_size >= size);
- DCHECK(!(bucket->slot_size % kGenericSmallestBucket));
- return bucket;
-}
-
-ALWAYS_INLINE void* PartitionAllocGenericFlags(PartitionRootGeneric* root,
- int flags,
- size_t size,
- const char* type_name) {
-#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
- void* result = malloc(size);
- CHECK(result || flags & PartitionAllocReturnNull);
- return result;
-#else
- DCHECK(root->initialized);
- size_t requested_size = size;
- size = PartitionCookieSizeAdjustAdd(size);
- PartitionBucket* bucket = PartitionGenericSizeToBucket(root, size);
- void* ret = nullptr;
- {
- subtle::SpinLock::Guard guard(root->lock);
- ret = PartitionBucketAlloc(root, flags, size, bucket);
- }
- PartitionAllocHooks::AllocationHookIfEnabled(ret, requested_size, type_name);
- return ret;
-#endif
-}
-
-ALWAYS_INLINE void* PartitionAllocGeneric(PartitionRootGeneric* root,
- size_t size,
- const char* type_name) {
- return PartitionAllocGenericFlags(root, 0, size, type_name);
-}
-
-ALWAYS_INLINE void PartitionFreeGeneric(PartitionRootGeneric* root, void* ptr) {
-#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
- free(ptr);
-#else
- DCHECK(root->initialized);
-
- if (UNLIKELY(!ptr))
- return;
-
- PartitionAllocHooks::FreeHookIfEnabled(ptr);
- ptr = PartitionCookieFreePointerAdjust(ptr);
- DCHECK(PartitionPointerIsValid(ptr));
- PartitionPage* page = PartitionPointerToPage(ptr);
- {
- subtle::SpinLock::Guard guard(root->lock);
- PartitionFreeWithPage(ptr, page);
- }
-#endif
-}
-
-ALWAYS_INLINE size_t PartitionDirectMapSize(size_t size) {
- // Caller must check that the size is not above the kGenericMaxDirectMapped
- // limit before calling. This also guards against integer overflow in the
- // calculation here.
- DCHECK(size <= kGenericMaxDirectMapped);
- return (size + kSystemPageOffsetMask) & kSystemPageBaseMask;
-}
-
-ALWAYS_INLINE size_t PartitionAllocActualSize(PartitionRootGeneric* root,
- size_t size) {
-#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
- return size;
-#else
- DCHECK(root->initialized);
- size = PartitionCookieSizeAdjustAdd(size);
- PartitionBucket* bucket = PartitionGenericSizeToBucket(root, size);
- if (LIKELY(!PartitionBucketIsDirectMapped(bucket))) {
- size = bucket->slot_size;
- } else if (size > kGenericMaxDirectMapped) {
- // Too large to allocate => return the size unchanged.
- } else {
- DCHECK(bucket == &PartitionRootBase::gPagedBucket);
- size = PartitionDirectMapSize(size);
- }
- return PartitionCookieSizeAdjustSubtract(size);
-#endif
-}
-
ALWAYS_INLINE bool PartitionAllocSupportsGetSize() {
#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
return false;
@@ -877,34 +302,155 @@
// No need to lock here. Only |ptr| being freed by another thread could
// cause trouble, and the caller is responsible for that not happening.
DCHECK(PartitionAllocSupportsGetSize());
- ptr = PartitionCookieFreePointerAdjust(ptr);
- DCHECK(PartitionPointerIsValid(ptr));
- PartitionPage* page = PartitionPointerToPage(ptr);
+ ptr = internal::PartitionCookieFreePointerAdjust(ptr);
+ internal::PartitionPage* page = internal::PartitionPage::FromPointer(ptr);
+ // TODO(palmer): See if we can afford to make this a CHECK.
+ DCHECK(internal::PartitionRootBase::IsValidPage(page));
size_t size = page->bucket->slot_size;
- return PartitionCookieSizeAdjustSubtract(size);
+ return internal::PartitionCookieSizeAdjustSubtract(size);
}
-// N (or more accurately, N - sizeof(void*)) represents the largest size in
-// bytes that will be handled by a SizeSpecificPartitionAllocator.
-// Attempts to partitionAlloc() more than this amount will fail.
+ALWAYS_INLINE void PartitionFree(void* ptr) {
+#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
+ free(ptr);
+#else
+ void* original_ptr = ptr;
+ // TODO(palmer): Check ptr alignment before continuing. Shall we do the check
+ // inside PartitionCookieFreePointerAdjust?
+ PartitionAllocHooks::FreeHookIfEnabled(original_ptr);
+ ptr = internal::PartitionCookieFreePointerAdjust(ptr);
+ internal::PartitionPage* page = internal::PartitionPage::FromPointer(ptr);
+ // TODO(palmer): See if we can afford to make this a CHECK.
+ DCHECK(internal::PartitionRootBase::IsValidPage(page));
+ page->Free(ptr);
+#endif
+}
+
+ALWAYS_INLINE internal::PartitionBucket* PartitionGenericSizeToBucket(
+ PartitionRootGeneric* root,
+ size_t size) {
+ size_t order = kBitsPerSizeT - bits::CountLeadingZeroBitsSizeT(size);
+ // The order index is simply the next few bits after the most significant bit.
+ size_t order_index = (size >> root->order_index_shifts[order]) &
+ (kGenericNumBucketsPerOrder - 1);
+ // And if the remaining bits are non-zero we must bump the bucket up.
+ size_t sub_order_index = size & root->order_sub_index_masks[order];
+ internal::PartitionBucket* bucket =
+ root->bucket_lookups[(order << kGenericNumBucketsPerOrderBits) +
+ order_index + !!sub_order_index];
+ CHECK(bucket);
+ DCHECK(!bucket->slot_size || bucket->slot_size >= size);
+ DCHECK(!(bucket->slot_size % kGenericSmallestBucket));
+ return bucket;
+}
+
+ALWAYS_INLINE void* PartitionAllocGenericFlags(PartitionRootGeneric* root,
+ int flags,
+ size_t size,
+ const char* type_name) {
+ DCHECK(flags < PartitionAllocLastFlag << 1);
+
+#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
+ const bool zero_fill = flags & PartitionAllocZeroFill;
+ void* result = zero_fill ? calloc(1, size) : malloc(size);
+ CHECK(result || flags & PartitionAllocReturnNull);
+ return result;
+#else
+ DCHECK(root->initialized);
+ size_t requested_size = size;
+ size = internal::PartitionCookieSizeAdjustAdd(size);
+ internal::PartitionBucket* bucket = PartitionGenericSizeToBucket(root, size);
+ void* ret = nullptr;
+ {
+ subtle::SpinLock::Guard guard(root->lock);
+ ret = root->AllocFromBucket(bucket, flags, size);
+ }
+ PartitionAllocHooks::AllocationHookIfEnabled(ret, requested_size, type_name);
+
+ return ret;
+#endif
+}
+
+ALWAYS_INLINE void* PartitionRootGeneric::Alloc(size_t size,
+ const char* type_name) {
+ return PartitionAllocGenericFlags(this, 0, size, type_name);
+}
+
+ALWAYS_INLINE void* PartitionRootGeneric::AllocFlags(int flags,
+ size_t size,
+ const char* type_name) {
+ return PartitionAllocGenericFlags(this, flags, size, type_name);
+}
+
+ALWAYS_INLINE void PartitionRootGeneric::Free(void* ptr) {
+#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
+ free(ptr);
+#else
+ DCHECK(this->initialized);
+
+ if (UNLIKELY(!ptr))
+ return;
+
+ PartitionAllocHooks::FreeHookIfEnabled(ptr);
+ ptr = internal::PartitionCookieFreePointerAdjust(ptr);
+ internal::PartitionPage* page = internal::PartitionPage::FromPointer(ptr);
+ // TODO(palmer): See if we can afford to make this a CHECK.
+ DCHECK(IsValidPage(page));
+ {
+ subtle::SpinLock::Guard guard(this->lock);
+ page->Free(ptr);
+ }
+#endif
+}
+
+BASE_EXPORT void* PartitionReallocGenericFlags(PartitionRootGeneric* root,
+ int flags,
+ void* ptr,
+ size_t new_size,
+ const char* type_name);
+
+ALWAYS_INLINE size_t PartitionRootGeneric::ActualSize(size_t size) {
+#if defined(MEMORY_TOOL_REPLACES_ALLOCATOR)
+ return size;
+#else
+ DCHECK(this->initialized);
+ size = internal::PartitionCookieSizeAdjustAdd(size);
+ internal::PartitionBucket* bucket = PartitionGenericSizeToBucket(this, size);
+ if (LIKELY(!bucket->is_direct_mapped())) {
+ size = bucket->slot_size;
+ } else if (size > kGenericMaxDirectMapped) {
+ // Too large to allocate => return the size unchanged.
+ } else {
+ size = internal::PartitionBucket::get_direct_map_size(size);
+ }
+ return internal::PartitionCookieSizeAdjustSubtract(size);
+#endif
+}
+
template <size_t N>
class SizeSpecificPartitionAllocator {
public:
+ SizeSpecificPartitionAllocator() {
+ memset(actual_buckets_, 0,
+ sizeof(internal::PartitionBucket) * pdfium::size(actual_buckets_));
+ }
+ ~SizeSpecificPartitionAllocator() = default;
static const size_t kMaxAllocation = N - kAllocationGranularity;
static const size_t kNumBuckets = N / kAllocationGranularity;
- void init() {
- PartitionAllocInit(&partition_root_, kNumBuckets, kMaxAllocation);
- }
+ void init() { partition_root_.Init(kNumBuckets, kMaxAllocation); }
ALWAYS_INLINE PartitionRoot* root() { return &partition_root_; }
private:
PartitionRoot partition_root_;
- PartitionBucket actual_buckets_[kNumBuckets];
+ internal::PartitionBucket actual_buckets_[kNumBuckets];
};
-class PartitionAllocatorGeneric {
+class BASE_EXPORT PartitionAllocatorGeneric {
public:
- void init() { PartitionAllocGenericInit(&partition_root_); }
+ PartitionAllocatorGeneric();
+ ~PartitionAllocatorGeneric();
+
+ void init() { partition_root_.Init(); }
ALWAYS_INLINE PartitionRootGeneric* root() { return &partition_root_; }
private:
@@ -914,4 +460,4 @@
} // namespace base
} // namespace pdfium
-#endif // BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ALLOC_H
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ALLOC_H_
diff --git a/third_party/base/allocator/partition_allocator/partition_alloc_constants.h b/third_party/base/allocator/partition_allocator/partition_alloc_constants.h
new file mode 100644
index 0000000..cd9108c
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_alloc_constants.h
@@ -0,0 +1,169 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ALLOC_CONSTANTS_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ALLOC_CONSTANTS_H_
+
+#include <limits.h>
+
+#include "third_party/base/allocator/partition_allocator/page_allocator_constants.h"
+#include "third_party/base/logging.h"
+
+namespace pdfium {
+namespace base {
+
+// Allocation granularity of sizeof(void*) bytes.
+static const size_t kAllocationGranularity = sizeof(void*);
+static const size_t kAllocationGranularityMask = kAllocationGranularity - 1;
+static const size_t kBucketShift = (kAllocationGranularity == 8) ? 3 : 2;
+
+// Underlying partition storage pages are a power-of-two size. It is typical
+// for a partition page to be based on multiple system pages. Most references to
+// "page" refer to partition pages.
+// We also have the concept of "super pages" -- these are the underlying system
+// allocations we make. Super pages contain multiple partition pages inside them
+// and include space for a small amount of metadata per partition page.
+// Inside super pages, we store "slot spans". A slot span is a continguous range
+// of one or more partition pages that stores allocations of the same size.
+// Slot span sizes are adjusted depending on the allocation size, to make sure
+// the packing does not lead to unused (wasted) space at the end of the last
+// system page of the span. For our current max slot span size of 64k and other
+// constant values, we pack _all_ PartitionRootGeneric::Alloc() sizes perfectly
+// up against the end of a system page.
+#if defined(_MIPS_ARCH_LOONGSON)
+static const size_t kPartitionPageShift = 16; // 64KB
+#else
+static const size_t kPartitionPageShift = 14; // 16KB
+#endif
+static const size_t kPartitionPageSize = 1 << kPartitionPageShift;
+static const size_t kPartitionPageOffsetMask = kPartitionPageSize - 1;
+static const size_t kPartitionPageBaseMask = ~kPartitionPageOffsetMask;
+static const size_t kMaxPartitionPagesPerSlotSpan = 4;
+
+// To avoid fragmentation via never-used freelist entries, we hand out partition
+// freelist sections gradually, in units of the dominant system page size.
+// What we're actually doing is avoiding filling the full partition page (16 KB)
+// with freelist pointers right away. Writing freelist pointers will fault and
+// dirty a private page, which is very wasteful if we never actually store
+// objects there.
+static const size_t kNumSystemPagesPerPartitionPage =
+ kPartitionPageSize / kSystemPageSize;
+static const size_t kMaxSystemPagesPerSlotSpan =
+ kNumSystemPagesPerPartitionPage * kMaxPartitionPagesPerSlotSpan;
+
+// We reserve virtual address space in 2MB chunks (aligned to 2MB as well).
+// These chunks are called "super pages". We do this so that we can store
+// metadata in the first few pages of each 2MB aligned section. This leads to
+// a very fast free(). We specifically choose 2MB because this virtual address
+// block represents a full but single PTE allocation on ARM, ia32 and x64.
+//
+// The layout of the super page is as follows. The sizes below are the same
+// for 32 bit and 64 bit.
+//
+// | Guard page (4KB) |
+// | Metadata page (4KB) |
+// | Guard pages (8KB) |
+// | Slot span |
+// | Slot span |
+// | ... |
+// | Slot span |
+// | Guard page (4KB) |
+//
+// - Each slot span is a contiguous range of one or more PartitionPages.
+// - The metadata page has the following format. Note that the PartitionPage
+// that is not at the head of a slot span is "unused". In other words,
+// the metadata for the slot span is stored only in the first PartitionPage
+// of the slot span. Metadata accesses to other PartitionPages are
+// redirected to the first PartitionPage.
+//
+// | SuperPageExtentEntry (32B) |
+// | PartitionPage of slot span 1 (32B, used) |
+// | PartitionPage of slot span 1 (32B, unused) |
+// | PartitionPage of slot span 1 (32B, unused) |
+// | PartitionPage of slot span 2 (32B, used) |
+// | PartitionPage of slot span 3 (32B, used) |
+// | ... |
+// | PartitionPage of slot span N (32B, unused) |
+//
+// A direct mapped page has a similar layout to fake it looking like a super
+// page:
+//
+// | Guard page (4KB) |
+// | Metadata page (4KB) |
+// | Guard pages (8KB) |
+// | Direct mapped object |
+// | Guard page (4KB) |
+//
+// - The metadata page has the following layout:
+//
+// | SuperPageExtentEntry (32B) |
+// | PartitionPage (32B) |
+// | PartitionBucket (32B) |
+// | PartitionDirectMapExtent (8B) |
+static const size_t kSuperPageShift = 21; // 2MB
+static const size_t kSuperPageSize = 1 << kSuperPageShift;
+static const size_t kSuperPageOffsetMask = kSuperPageSize - 1;
+static const size_t kSuperPageBaseMask = ~kSuperPageOffsetMask;
+static const size_t kNumPartitionPagesPerSuperPage =
+ kSuperPageSize / kPartitionPageSize;
+
+// The following kGeneric* constants apply to the generic variants of the API.
+// The "order" of an allocation is closely related to the power-of-two size of
+// the allocation. More precisely, the order is the bit index of the
+// most-significant-bit in the allocation size, where the bit numbers starts
+// at index 1 for the least-significant-bit.
+// In terms of allocation sizes, order 0 covers 0, order 1 covers 1, order 2
+// covers 2->3, order 3 covers 4->7, order 4 covers 8->15.
+static const size_t kGenericMinBucketedOrder = 4; // 8 bytes.
+static const size_t kGenericMaxBucketedOrder =
+ 20; // Largest bucketed order is 1<<(20-1) (storing 512KB -> almost 1MB)
+static const size_t kGenericNumBucketedOrders =
+ (kGenericMaxBucketedOrder - kGenericMinBucketedOrder) + 1;
+// Eight buckets per order (for the higher orders), e.g. order 8 is 128, 144,
+// 160, ..., 240:
+static const size_t kGenericNumBucketsPerOrderBits = 3;
+static const size_t kGenericNumBucketsPerOrder =
+ 1 << kGenericNumBucketsPerOrderBits;
+static const size_t kGenericNumBuckets =
+ kGenericNumBucketedOrders * kGenericNumBucketsPerOrder;
+static const size_t kGenericSmallestBucket = 1
+ << (kGenericMinBucketedOrder - 1);
+static const size_t kGenericMaxBucketSpacing =
+ 1 << ((kGenericMaxBucketedOrder - 1) - kGenericNumBucketsPerOrderBits);
+static const size_t kGenericMaxBucketed =
+ (1 << (kGenericMaxBucketedOrder - 1)) +
+ ((kGenericNumBucketsPerOrder - 1) * kGenericMaxBucketSpacing);
+static const size_t kGenericMinDirectMappedDownsize =
+ kGenericMaxBucketed +
+ 1; // Limit when downsizing a direct mapping using realloc().
+static const size_t kGenericMaxDirectMapped =
+ (1UL << 31) + kPageAllocationGranularity; // 2 GB plus one more page.
+static const size_t kBitsPerSizeT = sizeof(void*) * CHAR_BIT;
+
+// Constant for the memory reclaim logic.
+static const size_t kMaxFreeableSpans = 16;
+
+// If the total size in bytes of allocated but not committed pages exceeds this
+// value (probably it is a "out of virtual address space" crash),
+// a special crash stack trace is generated at |PartitionOutOfMemory|.
+// This is to distinguish "out of virtual address space" from
+// "out of physical memory" in crash reports.
+static const size_t kReasonableSizeOfUnusedPages = 1024 * 1024 * 1024; // 1GB
+
+// These two byte values match tcmalloc.
+static const unsigned char kUninitializedByte = 0xAB;
+static const unsigned char kFreedByte = 0xCD;
+
+// Flags for PartitionAllocGenericFlags.
+enum PartitionAllocFlags {
+ PartitionAllocReturnNull = 1 << 0,
+ PartitionAllocZeroFill = 1 << 1,
+
+ PartitionAllocLastFlag = PartitionAllocZeroFill
+};
+
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ALLOC_CONSTANTS_H_
diff --git a/third_party/base/allocator/partition_allocator/partition_bucket.cc b/third_party/base/allocator/partition_allocator/partition_bucket.cc
new file mode 100644
index 0000000..b540adb
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_bucket.cc
@@ -0,0 +1,568 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/base/allocator/partition_allocator/partition_bucket.h"
+
+#include "build/build_config.h"
+#include "third_party/base/allocator/partition_allocator/oom.h"
+#include "third_party/base/allocator/partition_allocator/page_allocator.h"
+#include "third_party/base/allocator/partition_allocator/partition_alloc_constants.h"
+#include "third_party/base/allocator/partition_allocator/partition_direct_map_extent.h"
+#include "third_party/base/allocator/partition_allocator/partition_oom.h"
+#include "third_party/base/allocator/partition_allocator/partition_page.h"
+#include "third_party/base/allocator/partition_allocator/partition_root_base.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+namespace {
+
+ALWAYS_INLINE PartitionPage* PartitionDirectMap(PartitionRootBase* root,
+ int flags,
+ size_t raw_size) {
+ size_t size = PartitionBucket::get_direct_map_size(raw_size);
+
+ // Because we need to fake looking like a super page, we need to allocate
+ // a bunch of system pages more than "size":
+ // - The first few system pages are the partition page in which the super
+ // page metadata is stored. We fault just one system page out of a partition
+ // page sized clump.
+ // - We add a trailing guard page on 32-bit (on 64-bit we rely on the
+ // massive address space plus randomization instead).
+ size_t map_size = size + kPartitionPageSize;
+#if !defined(ARCH_CPU_64_BITS)
+ map_size += kSystemPageSize;
+#endif
+ // Round up to the allocation granularity.
+ map_size += kPageAllocationGranularityOffsetMask;
+ map_size &= kPageAllocationGranularityBaseMask;
+
+ char* ptr = reinterpret_cast<char*>(
+ AllocPages(nullptr, map_size, kSuperPageSize, PageReadWrite));
+ if (UNLIKELY(!ptr))
+ return nullptr;
+
+ size_t committed_page_size = size + kSystemPageSize;
+ root->total_size_of_direct_mapped_pages += committed_page_size;
+ root->IncreaseCommittedPages(committed_page_size);
+
+ char* slot = ptr + kPartitionPageSize;
+ CHECK(SetSystemPagesAccess(ptr + (kSystemPageSize * 2),
+ kPartitionPageSize - (kSystemPageSize * 2),
+ PageInaccessible));
+#if !defined(ARCH_CPU_64_BITS)
+ CHECK(SetSystemPagesAccess(ptr, kSystemPageSize, PageInaccessible));
+ CHECK(SetSystemPagesAccess(slot + size, kSystemPageSize, PageInaccessible));
+#endif
+
+ PartitionSuperPageExtentEntry* extent =
+ reinterpret_cast<PartitionSuperPageExtentEntry*>(
+ PartitionSuperPageToMetadataArea(ptr));
+ extent->root = root;
+ // The new structures are all located inside a fresh system page so they
+ // will all be zeroed out. These DCHECKs are for documentation.
+ DCHECK(!extent->super_page_base);
+ DCHECK(!extent->super_pages_end);
+ DCHECK(!extent->next);
+ PartitionPage* page = PartitionPage::FromPointerNoAlignmentCheck(slot);
+ PartitionBucket* bucket = reinterpret_cast<PartitionBucket*>(
+ reinterpret_cast<char*>(page) + (kPageMetadataSize * 2));
+ DCHECK(!page->next_page);
+ DCHECK(!page->num_allocated_slots);
+ DCHECK(!page->num_unprovisioned_slots);
+ DCHECK(!page->page_offset);
+ DCHECK(!page->empty_cache_index);
+ page->bucket = bucket;
+ page->freelist_head = reinterpret_cast<PartitionFreelistEntry*>(slot);
+ PartitionFreelistEntry* next_entry =
+ reinterpret_cast<PartitionFreelistEntry*>(slot);
+ next_entry->next = PartitionFreelistEntry::Transform(nullptr);
+
+ DCHECK(!bucket->active_pages_head);
+ DCHECK(!bucket->empty_pages_head);
+ DCHECK(!bucket->decommitted_pages_head);
+ DCHECK(!bucket->num_system_pages_per_slot_span);
+ DCHECK(!bucket->num_full_pages);
+ bucket->slot_size = size;
+
+ PartitionDirectMapExtent* map_extent =
+ PartitionDirectMapExtent::FromPage(page);
+ map_extent->map_size = map_size - kPartitionPageSize - kSystemPageSize;
+ map_extent->bucket = bucket;
+
+ // Maintain the doubly-linked list of all direct mappings.
+ map_extent->next_extent = root->direct_map_list;
+ if (map_extent->next_extent)
+ map_extent->next_extent->prev_extent = map_extent;
+ map_extent->prev_extent = nullptr;
+ root->direct_map_list = map_extent;
+
+ return page;
+}
+
+} // namespace
+
+// static
+PartitionBucket PartitionBucket::sentinel_bucket_;
+
+PartitionBucket* PartitionBucket::get_sentinel_bucket() {
+ return &sentinel_bucket_;
+}
+
+// TODO(ajwong): This seems to interact badly with
+// get_pages_per_slot_span() which rounds the value from this up to a
+// multiple of kNumSystemPagesPerPartitionPage (aka 4) anyways.
+// http://crbug.com/776537
+//
+// TODO(ajwong): The waste calculation seems wrong. The PTE usage should cover
+// both used and unsed pages.
+// http://crbug.com/776537
+uint8_t PartitionBucket::get_system_pages_per_slot_span() {
+ // This works out reasonably for the current bucket sizes of the generic
+ // allocator, and the current values of partition page size and constants.
+ // Specifically, we have enough room to always pack the slots perfectly into
+ // some number of system pages. The only waste is the waste associated with
+ // unfaulted pages (i.e. wasted address space).
+ // TODO: we end up using a lot of system pages for very small sizes. For
+ // example, we'll use 12 system pages for slot size 24. The slot size is
+ // so small that the waste would be tiny with just 4, or 1, system pages.
+ // Later, we can investigate whether there are anti-fragmentation benefits
+ // to using fewer system pages.
+ double best_waste_ratio = 1.0f;
+ uint16_t best_pages = 0;
+ if (this->slot_size > kMaxSystemPagesPerSlotSpan * kSystemPageSize) {
+ // TODO(ajwong): Why is there a DCHECK here for this?
+ // http://crbug.com/776537
+ DCHECK(!(this->slot_size % kSystemPageSize));
+ best_pages = static_cast<uint16_t>(this->slot_size / kSystemPageSize);
+ // TODO(ajwong): Should this be checking against
+ // kMaxSystemPagesPerSlotSpan or numeric_limits<uint8_t>::max?
+ // http://crbug.com/776537
+ CHECK(best_pages < (1 << 8));
+ return static_cast<uint8_t>(best_pages);
+ }
+ DCHECK(this->slot_size <= kMaxSystemPagesPerSlotSpan * kSystemPageSize);
+ for (uint16_t i = kNumSystemPagesPerPartitionPage - 1;
+ i <= kMaxSystemPagesPerSlotSpan; ++i) {
+ size_t page_size = kSystemPageSize * i;
+ size_t num_slots = page_size / this->slot_size;
+ size_t waste = page_size - (num_slots * this->slot_size);
+ // Leaving a page unfaulted is not free; the page will occupy an empty page
+ // table entry. Make a simple attempt to account for that.
+ //
+ // TODO(ajwong): This looks wrong. PTEs are allocated for all pages
+ // regardless of whether or not they are wasted. Should it just
+ // be waste += i * sizeof(void*)?
+ // http://crbug.com/776537
+ size_t num_remainder_pages = i & (kNumSystemPagesPerPartitionPage - 1);
+ size_t num_unfaulted_pages =
+ num_remainder_pages
+ ? (kNumSystemPagesPerPartitionPage - num_remainder_pages)
+ : 0;
+ waste += sizeof(void*) * num_unfaulted_pages;
+ double waste_ratio =
+ static_cast<double>(waste) / static_cast<double>(page_size);
+ if (waste_ratio < best_waste_ratio) {
+ best_waste_ratio = waste_ratio;
+ best_pages = i;
+ }
+ }
+ DCHECK(best_pages > 0);
+ CHECK(best_pages <= kMaxSystemPagesPerSlotSpan);
+ return static_cast<uint8_t>(best_pages);
+}
+
+void PartitionBucket::Init(uint32_t new_slot_size) {
+ slot_size = new_slot_size;
+ active_pages_head = PartitionPage::get_sentinel_page();
+ empty_pages_head = nullptr;
+ decommitted_pages_head = nullptr;
+ num_full_pages = 0;
+ num_system_pages_per_slot_span = get_system_pages_per_slot_span();
+}
+
+NOINLINE void PartitionBucket::OnFull() {
+ OOM_CRASH();
+}
+
+ALWAYS_INLINE void* PartitionBucket::AllocNewSlotSpan(
+ PartitionRootBase* root,
+ int flags,
+ uint16_t num_partition_pages) {
+ DCHECK(!(reinterpret_cast<uintptr_t>(root->next_partition_page) %
+ kPartitionPageSize));
+ DCHECK(!(reinterpret_cast<uintptr_t>(root->next_partition_page_end) %
+ kPartitionPageSize));
+ DCHECK(num_partition_pages <= kNumPartitionPagesPerSuperPage);
+ size_t total_size = kPartitionPageSize * num_partition_pages;
+ size_t num_partition_pages_left =
+ (root->next_partition_page_end - root->next_partition_page) >>
+ kPartitionPageShift;
+ if (LIKELY(num_partition_pages_left >= num_partition_pages)) {
+ // In this case, we can still hand out pages from the current super page
+ // allocation.
+ char* ret = root->next_partition_page;
+
+ // Fresh System Pages in the SuperPages are decommited. Commit them
+ // before vending them back.
+ CHECK(SetSystemPagesAccess(ret, total_size, PageReadWrite));
+
+ root->next_partition_page += total_size;
+ root->IncreaseCommittedPages(total_size);
+ return ret;
+ }
+
+ // Need a new super page. We want to allocate super pages in a continguous
+ // address region as much as possible. This is important for not causing
+ // page table bloat and not fragmenting address spaces in 32 bit
+ // architectures.
+ char* requested_address = root->next_super_page;
+ char* super_page = reinterpret_cast<char*>(AllocPages(
+ requested_address, kSuperPageSize, kSuperPageSize, PageReadWrite));
+ if (UNLIKELY(!super_page))
+ return nullptr;
+
+ root->total_size_of_super_pages += kSuperPageSize;
+ root->IncreaseCommittedPages(total_size);
+
+ // |total_size| MUST be less than kSuperPageSize - (kPartitionPageSize*2).
+ // This is a trustworthy value because num_partition_pages is not user
+ // controlled.
+ //
+ // TODO(ajwong): Introduce a DCHECK.
+ root->next_super_page = super_page + kSuperPageSize;
+ char* ret = super_page + kPartitionPageSize;
+ root->next_partition_page = ret + total_size;
+ root->next_partition_page_end = root->next_super_page - kPartitionPageSize;
+ // Make the first partition page in the super page a guard page, but leave a
+ // hole in the middle.
+ // This is where we put page metadata and also a tiny amount of extent
+ // metadata.
+ CHECK(SetSystemPagesAccess(super_page, kSystemPageSize, PageInaccessible));
+ CHECK(SetSystemPagesAccess(super_page + (kSystemPageSize * 2),
+ kPartitionPageSize - (kSystemPageSize * 2),
+ PageInaccessible));
+ // CHECK(SetSystemPagesAccess(super_page + (kSuperPageSize -
+ // kPartitionPageSize),
+ // kPartitionPageSize, PageInaccessible));
+ // All remaining slotspans for the unallocated PartitionPages inside the
+ // SuperPage are conceptually decommitted. Correctly set the state here
+ // so they do not occupy resources.
+ //
+ // TODO(ajwong): Refactor Page Allocator API so the SuperPage comes in
+ // decommited initially.
+ CHECK(SetSystemPagesAccess(super_page + kPartitionPageSize + total_size,
+ (kSuperPageSize - kPartitionPageSize - total_size),
+ PageInaccessible));
+
+ // If we were after a specific address, but didn't get it, assume that
+ // the system chose a lousy address. Here most OS'es have a default
+ // algorithm that isn't randomized. For example, most Linux
+ // distributions will allocate the mapping directly before the last
+ // successful mapping, which is far from random. So we just get fresh
+ // randomness for the next mapping attempt.
+ if (requested_address && requested_address != super_page)
+ root->next_super_page = nullptr;
+
+ // We allocated a new super page so update super page metadata.
+ // First check if this is a new extent or not.
+ PartitionSuperPageExtentEntry* latest_extent =
+ reinterpret_cast<PartitionSuperPageExtentEntry*>(
+ PartitionSuperPageToMetadataArea(super_page));
+ // By storing the root in every extent metadata object, we have a fast way
+ // to go from a pointer within the partition to the root object.
+ latest_extent->root = root;
+ // Most new extents will be part of a larger extent, and these three fields
+ // are unused, but we initialize them to 0 so that we get a clear signal
+ // in case they are accidentally used.
+ latest_extent->super_page_base = nullptr;
+ latest_extent->super_pages_end = nullptr;
+ latest_extent->next = nullptr;
+
+ PartitionSuperPageExtentEntry* current_extent = root->current_extent;
+ bool is_new_extent = (super_page != requested_address);
+ if (UNLIKELY(is_new_extent)) {
+ if (UNLIKELY(!current_extent)) {
+ DCHECK(!root->first_extent);
+ root->first_extent = latest_extent;
+ } else {
+ DCHECK(current_extent->super_page_base);
+ current_extent->next = latest_extent;
+ }
+ root->current_extent = latest_extent;
+ latest_extent->super_page_base = super_page;
+ latest_extent->super_pages_end = super_page + kSuperPageSize;
+ } else {
+ // We allocated next to an existing extent so just nudge the size up a
+ // little.
+ DCHECK(current_extent->super_pages_end);
+ current_extent->super_pages_end += kSuperPageSize;
+ DCHECK(ret >= current_extent->super_page_base &&
+ ret < current_extent->super_pages_end);
+ }
+ return ret;
+}
+
+ALWAYS_INLINE uint16_t PartitionBucket::get_pages_per_slot_span() {
+ // Rounds up to nearest multiple of kNumSystemPagesPerPartitionPage.
+ return (num_system_pages_per_slot_span +
+ (kNumSystemPagesPerPartitionPage - 1)) /
+ kNumSystemPagesPerPartitionPage;
+}
+
+ALWAYS_INLINE void PartitionBucket::InitializeSlotSpan(PartitionPage* page) {
+ // The bucket never changes. We set it up once.
+ page->bucket = this;
+ page->empty_cache_index = -1;
+
+ page->Reset();
+
+ // If this page has just a single slot, do not set up page offsets for any
+ // page metadata other than the first one. This ensures that attempts to
+ // touch invalid page metadata fail.
+ if (page->num_unprovisioned_slots == 1)
+ return;
+
+ uint16_t num_partition_pages = get_pages_per_slot_span();
+ char* page_char_ptr = reinterpret_cast<char*>(page);
+ for (uint16_t i = 1; i < num_partition_pages; ++i) {
+ page_char_ptr += kPageMetadataSize;
+ PartitionPage* secondary_page =
+ reinterpret_cast<PartitionPage*>(page_char_ptr);
+ secondary_page->page_offset = i;
+ }
+}
+
+ALWAYS_INLINE char* PartitionBucket::AllocAndFillFreelist(PartitionPage* page) {
+ DCHECK(page != PartitionPage::get_sentinel_page());
+ uint16_t num_slots = page->num_unprovisioned_slots;
+ DCHECK(num_slots);
+ // We should only get here when _every_ slot is either used or unprovisioned.
+ // (The third state is "on the freelist". If we have a non-empty freelist, we
+ // should not get here.)
+ DCHECK(num_slots + page->num_allocated_slots == this->get_slots_per_span());
+ // Similarly, make explicitly sure that the freelist is empty.
+ DCHECK(!page->freelist_head);
+ DCHECK(page->num_allocated_slots >= 0);
+
+ size_t size = this->slot_size;
+ char* base = reinterpret_cast<char*>(PartitionPage::ToPointer(page));
+ char* return_object = base + (size * page->num_allocated_slots);
+ char* first_freelist_pointer = return_object + size;
+ char* first_freelist_pointer_extent =
+ first_freelist_pointer + sizeof(PartitionFreelistEntry*);
+ // Our goal is to fault as few system pages as possible. We calculate the
+ // page containing the "end" of the returned slot, and then allow freelist
+ // pointers to be written up to the end of that page.
+ char* sub_page_limit = reinterpret_cast<char*>(
+ RoundUpToSystemPage(reinterpret_cast<size_t>(first_freelist_pointer)));
+ char* slots_limit = return_object + (size * num_slots);
+ char* freelist_limit = sub_page_limit;
+ if (UNLIKELY(slots_limit < freelist_limit))
+ freelist_limit = slots_limit;
+
+ uint16_t num_new_freelist_entries = 0;
+ if (LIKELY(first_freelist_pointer_extent <= freelist_limit)) {
+ // Only consider used space in the slot span. If we consider wasted
+ // space, we may get an off-by-one when a freelist pointer fits in the
+ // wasted space, but a slot does not.
+ // We know we can fit at least one freelist pointer.
+ num_new_freelist_entries = 1;
+ // Any further entries require space for the whole slot span.
+ num_new_freelist_entries += static_cast<uint16_t>(
+ (freelist_limit - first_freelist_pointer_extent) / size);
+ }
+
+ // We always return an object slot -- that's the +1 below.
+ // We do not neccessarily create any new freelist entries, because we cross
+ // sub page boundaries frequently for large bucket sizes.
+ DCHECK(num_new_freelist_entries + 1 <= num_slots);
+ num_slots -= (num_new_freelist_entries + 1);
+ page->num_unprovisioned_slots = num_slots;
+ page->num_allocated_slots++;
+
+ if (LIKELY(num_new_freelist_entries)) {
+ char* freelist_pointer = first_freelist_pointer;
+ PartitionFreelistEntry* entry =
+ reinterpret_cast<PartitionFreelistEntry*>(freelist_pointer);
+ page->freelist_head = entry;
+ while (--num_new_freelist_entries) {
+ freelist_pointer += size;
+ PartitionFreelistEntry* next_entry =
+ reinterpret_cast<PartitionFreelistEntry*>(freelist_pointer);
+ entry->next = PartitionFreelistEntry::Transform(next_entry);
+ entry = next_entry;
+ }
+ entry->next = PartitionFreelistEntry::Transform(nullptr);
+ } else {
+ page->freelist_head = nullptr;
+ }
+ return return_object;
+}
+
+bool PartitionBucket::SetNewActivePage() {
+ PartitionPage* page = this->active_pages_head;
+ if (page == PartitionPage::get_sentinel_page())
+ return false;
+
+ PartitionPage* next_page;
+
+ for (; page; page = next_page) {
+ next_page = page->next_page;
+ DCHECK(page->bucket == this);
+ DCHECK(page != this->empty_pages_head);
+ DCHECK(page != this->decommitted_pages_head);
+
+ if (LIKELY(page->is_active())) {
+ // This page is usable because it has freelist entries, or has
+ // unprovisioned slots we can create freelist entries from.
+ this->active_pages_head = page;
+ return true;
+ }
+
+ // Deal with empty and decommitted pages.
+ if (LIKELY(page->is_empty())) {
+ page->next_page = this->empty_pages_head;
+ this->empty_pages_head = page;
+ } else if (LIKELY(page->is_decommitted())) {
+ page->next_page = this->decommitted_pages_head;
+ this->decommitted_pages_head = page;
+ } else {
+ DCHECK(page->is_full());
+ // If we get here, we found a full page. Skip over it too, and also
+ // tag it as full (via a negative value). We need it tagged so that
+ // free'ing can tell, and move it back into the active page list.
+ page->num_allocated_slots = -page->num_allocated_slots;
+ ++this->num_full_pages;
+ // num_full_pages is a uint16_t for efficient packing so guard against
+ // overflow to be safe.
+ if (UNLIKELY(!this->num_full_pages))
+ OnFull();
+ // Not necessary but might help stop accidents.
+ page->next_page = nullptr;
+ }
+ }
+
+ this->active_pages_head = PartitionPage::get_sentinel_page();
+ return false;
+}
+
+void* PartitionBucket::SlowPathAlloc(PartitionRootBase* root,
+ int flags,
+ size_t size,
+ bool* is_already_zeroed) {
+ // The slow path is called when the freelist is empty.
+ DCHECK(!this->active_pages_head->freelist_head);
+
+ PartitionPage* new_page = nullptr;
+ *is_already_zeroed = false;
+
+ // For the PartitionRootGeneric::Alloc() API, we have a bunch of buckets
+ // marked as special cases. We bounce them through to the slow path so that
+ // we can still have a blazing fast hot path due to lack of corner-case
+ // branches.
+ //
+ // Note: The ordering of the conditionals matter! In particular,
+ // SetNewActivePage() has a side-effect even when returning
+ // false where it sweeps the active page list and may move things into
+ // the empty or decommitted lists which affects the subsequent conditional.
+ bool return_null = flags & PartitionAllocReturnNull;
+ if (UNLIKELY(this->is_direct_mapped())) {
+ DCHECK(size > kGenericMaxBucketed);
+ DCHECK(this == get_sentinel_bucket());
+ DCHECK(this->active_pages_head == PartitionPage::get_sentinel_page());
+ if (size > kGenericMaxDirectMapped) {
+ if (return_null)
+ return nullptr;
+ PartitionExcessiveAllocationSize();
+ }
+ new_page = PartitionDirectMap(root, flags, size);
+#if !defined(OS_MACOSX)
+ // Turn off the optimization to see if it helps https://crbug.com/892550.
+ *is_already_zeroed = true;
+#endif
+ } else if (LIKELY(this->SetNewActivePage())) {
+ // First, did we find an active page in the active pages list?
+ new_page = this->active_pages_head;
+ DCHECK(new_page->is_active());
+ } else if (LIKELY(this->empty_pages_head != nullptr) ||
+ LIKELY(this->decommitted_pages_head != nullptr)) {
+ // Second, look in our lists of empty and decommitted pages.
+ // Check empty pages first, which are preferred, but beware that an
+ // empty page might have been decommitted.
+ while (LIKELY((new_page = this->empty_pages_head) != nullptr)) {
+ DCHECK(new_page->bucket == this);
+ DCHECK(new_page->is_empty() || new_page->is_decommitted());
+ this->empty_pages_head = new_page->next_page;
+ // Accept the empty page unless it got decommitted.
+ if (new_page->freelist_head) {
+ new_page->next_page = nullptr;
+ break;
+ }
+ DCHECK(new_page->is_decommitted());
+ new_page->next_page = this->decommitted_pages_head;
+ this->decommitted_pages_head = new_page;
+ }
+ if (UNLIKELY(!new_page) &&
+ LIKELY(this->decommitted_pages_head != nullptr)) {
+ new_page = this->decommitted_pages_head;
+ DCHECK(new_page->bucket == this);
+ DCHECK(new_page->is_decommitted());
+ this->decommitted_pages_head = new_page->next_page;
+ void* addr = PartitionPage::ToPointer(new_page);
+ root->RecommitSystemPages(addr, new_page->bucket->get_bytes_per_span());
+ new_page->Reset();
+ // TODO(https://crbug.com/890752): Optimizing here might cause pages to
+ // not be zeroed.
+ // *is_already_zeroed = true;
+ }
+ DCHECK(new_page);
+ } else {
+ // Third. If we get here, we need a brand new page.
+ uint16_t num_partition_pages = this->get_pages_per_slot_span();
+ void* raw_pages = AllocNewSlotSpan(root, flags, num_partition_pages);
+ if (LIKELY(raw_pages != nullptr)) {
+ new_page = PartitionPage::FromPointerNoAlignmentCheck(raw_pages);
+ InitializeSlotSpan(new_page);
+ // TODO(https://crbug.com/890752): Optimizing here causes pages to not be
+ // zeroed on at least macOS.
+ // *is_already_zeroed = true;
+ }
+ }
+
+ // Bail if we had a memory allocation failure.
+ if (UNLIKELY(!new_page)) {
+ DCHECK(this->active_pages_head == PartitionPage::get_sentinel_page());
+ if (return_null)
+ return nullptr;
+ root->OutOfMemory();
+ }
+
+ // TODO(ajwong): Is there a way to avoid the reading of bucket here?
+ // It seems like in many of the conditional branches above, |this| ==
+ // |new_page->bucket|. Maybe pull this into another function?
+ PartitionBucket* bucket = new_page->bucket;
+ DCHECK(bucket != get_sentinel_bucket());
+ bucket->active_pages_head = new_page;
+ new_page->set_raw_size(size);
+
+ // If we found an active page with free slots, or an empty page, we have a
+ // usable freelist head.
+ if (LIKELY(new_page->freelist_head != nullptr)) {
+ PartitionFreelistEntry* entry = new_page->freelist_head;
+ PartitionFreelistEntry* new_head =
+ PartitionFreelistEntry::Transform(entry->next);
+ new_page->freelist_head = new_head;
+ new_page->num_allocated_slots++;
+ return entry;
+ }
+ // Otherwise, we need to build the freelist.
+ DCHECK(new_page->num_unprovisioned_slots);
+ return AllocAndFillFreelist(new_page);
+}
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
diff --git a/third_party/base/allocator/partition_allocator/partition_bucket.h b/third_party/base/allocator/partition_allocator/partition_bucket.h
new file mode 100644
index 0000000..a89099b
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_bucket.h
@@ -0,0 +1,130 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_BUCKET_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_BUCKET_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "third_party/base/allocator/partition_allocator/partition_alloc_constants.h"
+#include "third_party/base/base_export.h"
+#include "third_party/base/compiler_specific.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+struct PartitionPage;
+struct PartitionRootBase;
+
+struct PartitionBucket {
+ // Accessed most in hot path => goes first.
+ PartitionPage* active_pages_head;
+
+ PartitionPage* empty_pages_head;
+ PartitionPage* decommitted_pages_head;
+ uint32_t slot_size;
+ uint32_t num_system_pages_per_slot_span : 8;
+ uint32_t num_full_pages : 24;
+
+ // Public API.
+ void Init(uint32_t new_slot_size);
+
+ // Sets |is_already_zeroed| to true if the allocation was satisfied by
+ // requesting (a) new page(s) from the operating system, or false otherwise.
+ // This enables an optimization for when callers use |PartitionAllocZeroFill|:
+ // there is no need to call memset on fresh pages; the OS has already zeroed
+ // them. (See |PartitionRootBase::AllocFromBucket|.)
+ //
+ // Note the matching Free() functions are in PartitionPage.
+ BASE_EXPORT NOINLINE void* SlowPathAlloc(PartitionRootBase* root,
+ int flags,
+ size_t size,
+ bool* is_already_zeroed);
+
+ ALWAYS_INLINE bool is_direct_mapped() const {
+ return !num_system_pages_per_slot_span;
+ }
+ ALWAYS_INLINE size_t get_bytes_per_span() const {
+ // TODO(ajwong): Change to CheckedMul. https://crbug.com/787153
+ // https://crbug.com/680657
+ return num_system_pages_per_slot_span * kSystemPageSize;
+ }
+ ALWAYS_INLINE uint16_t get_slots_per_span() const {
+ // TODO(ajwong): Change to CheckedMul. https://crbug.com/787153
+ // https://crbug.com/680657
+ return static_cast<uint16_t>(get_bytes_per_span() / slot_size);
+ }
+
+ static ALWAYS_INLINE size_t get_direct_map_size(size_t size) {
+ // Caller must check that the size is not above the kGenericMaxDirectMapped
+ // limit before calling. This also guards against integer overflow in the
+ // calculation here.
+ DCHECK(size <= kGenericMaxDirectMapped);
+ return (size + kSystemPageOffsetMask) & kSystemPageBaseMask;
+ }
+
+ // TODO(ajwong): Can this be made private? https://crbug.com/787153
+ static PartitionBucket* get_sentinel_bucket();
+
+ // This helper function scans a bucket's active page list for a suitable new
+ // active page. When it finds a suitable new active page (one that has
+ // free slots and is not empty), it is set as the new active page. If there
+ // is no suitable new active page, the current active page is set to
+ // PartitionPage::get_sentinel_page(). As potential pages are scanned, they
+ // are tidied up according to their state. Empty pages are swept on to the
+ // empty page list, decommitted pages on to the decommitted page list and full
+ // pages are unlinked from any list.
+ //
+ // This is where the guts of the bucket maintenance is done!
+ bool SetNewActivePage();
+
+ private:
+ static void OutOfMemory(const PartitionRootBase* root);
+ static void OutOfMemoryWithLotsOfUncommitedPages();
+
+ static NOINLINE void OnFull();
+
+ // Returns a natural number of PartitionPages (calculated by
+ // get_system_pages_per_slot_span()) to allocate from the current
+ // SuperPage when the bucket runs out of slots.
+ ALWAYS_INLINE uint16_t get_pages_per_slot_span();
+
+ // Returns the number of system pages in a slot span.
+ //
+ // The calculation attemps to find the best number of System Pages to
+ // allocate for the given slot_size to minimize wasted space. It uses a
+ // heuristic that looks at number of bytes wasted after the last slot and
+ // attempts to account for the PTE usage of each System Page.
+ uint8_t get_system_pages_per_slot_span();
+
+ // Allocates a new slot span with size |num_partition_pages| from the
+ // current extent. Metadata within this slot span will be uninitialized.
+ // Returns nullptr on error.
+ ALWAYS_INLINE void* AllocNewSlotSpan(PartitionRootBase* root,
+ int flags,
+ uint16_t num_partition_pages);
+
+ // Each bucket allocates a slot span when it runs out of slots.
+ // A slot span's size is equal to get_pages_per_slot_span() number of
+ // PartitionPages. This function initializes all PartitionPage within the
+ // span to point to the first PartitionPage which holds all the metadata
+ // for the span and registers this bucket as the owner of the span. It does
+ // NOT put the slots into the bucket's freelist.
+ ALWAYS_INLINE void InitializeSlotSpan(PartitionPage* page);
+
+ // Allocates one slot from the given |page| and then adds the remainder to
+ // the current bucket. If the |page| was freshly allocated, it must have been
+ // passed through InitializeSlotSpan() first.
+ ALWAYS_INLINE char* AllocAndFillFreelist(PartitionPage* page);
+
+ static PartitionBucket sentinel_bucket_;
+};
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_BUCKET_H_
diff --git a/third_party/base/allocator/partition_allocator/partition_cookie.h b/third_party/base/allocator/partition_allocator/partition_cookie.h
new file mode 100644
index 0000000..7cf4e84
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_cookie.h
@@ -0,0 +1,72 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_COOKIE_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_COOKIE_H_
+
+#include "third_party/base/compiler_specific.h"
+#include "third_party/base/logging.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+#if DCHECK_IS_ON()
+// Handles alignment up to XMM instructions on Intel.
+static constexpr size_t kCookieSize = 16;
+
+static constexpr unsigned char kCookieValue[kCookieSize] = {
+ 0xDE, 0xAD, 0xBE, 0xEF, 0xCA, 0xFE, 0xD0, 0x0D,
+ 0x13, 0x37, 0xF0, 0x05, 0xBA, 0x11, 0xAB, 0x1E};
+#endif
+
+ALWAYS_INLINE void PartitionCookieCheckValue(void* ptr) {
+#if DCHECK_IS_ON()
+ unsigned char* cookie_ptr = reinterpret_cast<unsigned char*>(ptr);
+ for (size_t i = 0; i < kCookieSize; ++i, ++cookie_ptr)
+ DCHECK(*cookie_ptr == kCookieValue[i]);
+#endif
+}
+
+ALWAYS_INLINE size_t PartitionCookieSizeAdjustAdd(size_t size) {
+#if DCHECK_IS_ON()
+ // Add space for cookies, checking for integer overflow. TODO(palmer):
+ // Investigate the performance and code size implications of using
+ // CheckedNumeric throughout PA.
+ DCHECK(size + (2 * kCookieSize) > size);
+ size += 2 * kCookieSize;
+#endif
+ return size;
+}
+
+ALWAYS_INLINE void* PartitionCookieFreePointerAdjust(void* ptr) {
+#if DCHECK_IS_ON()
+ // The value given to the application is actually just after the cookie.
+ ptr = static_cast<char*>(ptr) - kCookieSize;
+#endif
+ return ptr;
+}
+
+ALWAYS_INLINE size_t PartitionCookieSizeAdjustSubtract(size_t size) {
+#if DCHECK_IS_ON()
+ // Remove space for cookies.
+ DCHECK(size >= 2 * kCookieSize);
+ size -= 2 * kCookieSize;
+#endif
+ return size;
+}
+
+ALWAYS_INLINE void PartitionCookieWriteValue(void* ptr) {
+#if DCHECK_IS_ON()
+ unsigned char* cookie_ptr = reinterpret_cast<unsigned char*>(ptr);
+ for (size_t i = 0; i < kCookieSize; ++i, ++cookie_ptr)
+ *cookie_ptr = kCookieValue[i];
+#endif
+}
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_COOKIE_H_
diff --git a/third_party/base/allocator/partition_allocator/partition_direct_map_extent.h b/third_party/base/allocator/partition_allocator/partition_direct_map_extent.h
new file mode 100644
index 0000000..192c5b4
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_direct_map_extent.h
@@ -0,0 +1,35 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_DIRECT_MAP_EXTENT_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_DIRECT_MAP_EXTENT_H_
+
+#include "third_party/base/allocator/partition_allocator/partition_bucket.h"
+#include "third_party/base/allocator/partition_allocator/partition_page.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+struct PartitionDirectMapExtent {
+ PartitionDirectMapExtent* next_extent;
+ PartitionDirectMapExtent* prev_extent;
+ PartitionBucket* bucket;
+ size_t map_size; // Mapped size, not including guard pages and meta-data.
+
+ ALWAYS_INLINE static PartitionDirectMapExtent* FromPage(PartitionPage* page);
+};
+
+ALWAYS_INLINE PartitionDirectMapExtent* PartitionDirectMapExtent::FromPage(
+ PartitionPage* page) {
+ DCHECK(page->bucket->is_direct_mapped());
+ return reinterpret_cast<PartitionDirectMapExtent*>(
+ reinterpret_cast<char*>(page) + 3 * kPageMetadataSize);
+}
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_DIRECT_MAP_EXTENT_H_
diff --git a/third_party/base/allocator/partition_allocator/partition_freelist_entry.h b/third_party/base/allocator/partition_allocator/partition_freelist_entry.h
new file mode 100644
index 0000000..e9f2284
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_freelist_entry.h
@@ -0,0 +1,50 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_FREELIST_ENTRY_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_FREELIST_ENTRY_H_
+
+#include <stdint.h>
+
+#include "build/build_config.h"
+#include "third_party/base/allocator/partition_allocator/partition_alloc_constants.h"
+#include "third_party/base/compiler_specific.h"
+#include "third_party/base/sys_byteorder.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+// TODO(ajwong): Introduce an EncodedFreelistEntry type and then replace
+// Transform() with Encode()/Decode() such that the API provides some static
+// type safety.
+//
+// https://crbug.com/787153
+struct PartitionFreelistEntry {
+ PartitionFreelistEntry* next;
+
+ static ALWAYS_INLINE PartitionFreelistEntry* Transform(
+ PartitionFreelistEntry* ptr) {
+// We use bswap on little endian as a fast mask for two reasons:
+// 1) If an object is freed and its vtable used where the attacker doesn't
+// get the chance to run allocations between the free and use, the vtable
+// dereference is likely to fault.
+// 2) If the attacker has a linear buffer overflow and elects to try and
+// corrupt a freelist pointer, partial pointer overwrite attacks are
+// thwarted.
+// For big endian, similar guarantees are arrived at with a negation.
+#if defined(ARCH_CPU_BIG_ENDIAN)
+ uintptr_t masked = ~reinterpret_cast<uintptr_t>(ptr);
+#else
+ uintptr_t masked = ByteSwapUintPtrT(reinterpret_cast<uintptr_t>(ptr));
+#endif
+ return reinterpret_cast<PartitionFreelistEntry*>(masked);
+ }
+};
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_FREELIST_ENTRY_H_
diff --git a/third_party/base/allocator/partition_allocator/partition_oom.cc b/third_party/base/allocator/partition_allocator/partition_oom.cc
new file mode 100644
index 0000000..a4052d1
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_oom.cc
@@ -0,0 +1,26 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/base/allocator/partition_allocator/partition_oom.h"
+
+#include "build/build_config.h"
+#include "third_party/base/allocator/partition_allocator/oom.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+void NOINLINE PartitionExcessiveAllocationSize() {
+ OOM_CRASH();
+}
+
+#if !defined(ARCH_CPU_64_BITS)
+NOINLINE void PartitionOutOfMemoryWithLotsOfUncommitedPages() {
+ OOM_CRASH();
+}
+#endif
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
diff --git a/third_party/base/allocator/partition_allocator/partition_oom.h b/third_party/base/allocator/partition_allocator/partition_oom.h
new file mode 100644
index 0000000..be43ff3
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_oom.h
@@ -0,0 +1,28 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Holds functions for generating OOM errors from PartitionAlloc. This is
+// distinct from oom.h in that it is meant only for use in PartitionAlloc.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_OOM_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_OOM_H_
+
+#include "build/build_config.h"
+#include "third_party/base/compiler_specific.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+NOINLINE void PartitionExcessiveAllocationSize();
+
+#if !defined(ARCH_CPU_64_BITS)
+NOINLINE void PartitionOutOfMemoryWithLotsOfUncommitedPages();
+#endif
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_OOM_H_
diff --git a/third_party/base/allocator/partition_allocator/partition_page.cc b/third_party/base/allocator/partition_allocator/partition_page.cc
new file mode 100644
index 0000000..3f70048
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_page.cc
@@ -0,0 +1,165 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/base/allocator/partition_allocator/partition_page.h"
+
+#include "third_party/base/allocator/partition_allocator/partition_direct_map_extent.h"
+#include "third_party/base/allocator/partition_allocator/partition_root_base.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+namespace {
+
+ALWAYS_INLINE void PartitionDirectUnmap(PartitionPage* page) {
+ PartitionRootBase* root = PartitionRootBase::FromPage(page);
+ const PartitionDirectMapExtent* extent =
+ PartitionDirectMapExtent::FromPage(page);
+ size_t unmap_size = extent->map_size;
+
+ // Maintain the doubly-linked list of all direct mappings.
+ if (extent->prev_extent) {
+ DCHECK(extent->prev_extent->next_extent == extent);
+ extent->prev_extent->next_extent = extent->next_extent;
+ } else {
+ root->direct_map_list = extent->next_extent;
+ }
+ if (extent->next_extent) {
+ DCHECK(extent->next_extent->prev_extent == extent);
+ extent->next_extent->prev_extent = extent->prev_extent;
+ }
+
+ // Add on the size of the trailing guard page and preceeding partition
+ // page.
+ unmap_size += kPartitionPageSize + kSystemPageSize;
+
+ size_t uncommitted_page_size = page->bucket->slot_size + kSystemPageSize;
+ root->DecreaseCommittedPages(uncommitted_page_size);
+ DCHECK(root->total_size_of_direct_mapped_pages >= uncommitted_page_size);
+ root->total_size_of_direct_mapped_pages -= uncommitted_page_size;
+
+ DCHECK(!(unmap_size & kPageAllocationGranularityOffsetMask));
+
+ char* ptr = reinterpret_cast<char*>(PartitionPage::ToPointer(page));
+ // Account for the mapping starting a partition page before the actual
+ // allocation address.
+ ptr -= kPartitionPageSize;
+
+ FreePages(ptr, unmap_size);
+}
+
+ALWAYS_INLINE void PartitionRegisterEmptyPage(PartitionPage* page) {
+ DCHECK(page->is_empty());
+ PartitionRootBase* root = PartitionRootBase::FromPage(page);
+
+ // If the page is already registered as empty, give it another life.
+ if (page->empty_cache_index != -1) {
+ DCHECK(page->empty_cache_index >= 0);
+ DCHECK(static_cast<unsigned>(page->empty_cache_index) < kMaxFreeableSpans);
+ DCHECK(root->global_empty_page_ring[page->empty_cache_index] == page);
+ root->global_empty_page_ring[page->empty_cache_index] = nullptr;
+ }
+
+ int16_t current_index = root->global_empty_page_ring_index;
+ PartitionPage* page_to_decommit = root->global_empty_page_ring[current_index];
+ // The page might well have been re-activated, filled up, etc. before we get
+ // around to looking at it here.
+ if (page_to_decommit)
+ page_to_decommit->DecommitIfPossible(root);
+
+ // We put the empty slot span on our global list of "pages that were once
+ // empty". thus providing it a bit of breathing room to get re-used before
+ // we really free it. This improves performance, particularly on Mac OS X
+ // which has subpar memory management performance.
+ root->global_empty_page_ring[current_index] = page;
+ page->empty_cache_index = current_index;
+ ++current_index;
+ if (current_index == kMaxFreeableSpans)
+ current_index = 0;
+ root->global_empty_page_ring_index = current_index;
+}
+
+} // namespace
+
+// static
+PartitionPage PartitionPage::sentinel_page_;
+
+PartitionPage* PartitionPage::get_sentinel_page() {
+ return &sentinel_page_;
+}
+
+void PartitionPage::FreeSlowPath() {
+ DCHECK(this != get_sentinel_page());
+ if (LIKELY(this->num_allocated_slots == 0)) {
+ // Page became fully unused.
+ if (UNLIKELY(bucket->is_direct_mapped())) {
+ PartitionDirectUnmap(this);
+ return;
+ }
+ // If it's the current active page, change it. We bounce the page to
+ // the empty list as a force towards defragmentation.
+ if (LIKELY(this == bucket->active_pages_head))
+ bucket->SetNewActivePage();
+ DCHECK(bucket->active_pages_head != this);
+
+ set_raw_size(0);
+ DCHECK(!get_raw_size());
+
+ PartitionRegisterEmptyPage(this);
+ } else {
+ DCHECK(!bucket->is_direct_mapped());
+ // Ensure that the page is full. That's the only valid case if we
+ // arrive here.
+ DCHECK(this->num_allocated_slots < 0);
+ // A transition of num_allocated_slots from 0 to -1 is not legal, and
+ // likely indicates a double-free.
+ CHECK(this->num_allocated_slots != -1);
+ this->num_allocated_slots = -this->num_allocated_slots - 2;
+ DCHECK(this->num_allocated_slots == bucket->get_slots_per_span() - 1);
+ // Fully used page became partially used. It must be put back on the
+ // non-full page list. Also make it the current page to increase the
+ // chances of it being filled up again. The old current page will be
+ // the next page.
+ DCHECK(!this->next_page);
+ if (LIKELY(bucket->active_pages_head != get_sentinel_page()))
+ this->next_page = bucket->active_pages_head;
+ bucket->active_pages_head = this;
+ --bucket->num_full_pages;
+ // Special case: for a partition page with just a single slot, it may
+ // now be empty and we want to run it through the empty logic.
+ if (UNLIKELY(this->num_allocated_slots == 0))
+ FreeSlowPath();
+ }
+}
+
+void PartitionPage::Decommit(PartitionRootBase* root) {
+ DCHECK(is_empty());
+ DCHECK(!bucket->is_direct_mapped());
+ void* addr = PartitionPage::ToPointer(this);
+ root->DecommitSystemPages(addr, bucket->get_bytes_per_span());
+
+ // We actually leave the decommitted page in the active list. We'll sweep
+ // it on to the decommitted page list when we next walk the active page
+ // list.
+ // Pulling this trick enables us to use a singly-linked page list for all
+ // cases, which is critical in keeping the page metadata structure down to
+ // 32 bytes in size.
+ freelist_head = nullptr;
+ num_unprovisioned_slots = 0;
+ DCHECK(is_decommitted());
+}
+
+void PartitionPage::DecommitIfPossible(PartitionRootBase* root) {
+ DCHECK(empty_cache_index >= 0);
+ DCHECK(static_cast<unsigned>(empty_cache_index) < kMaxFreeableSpans);
+ DCHECK(this == root->global_empty_page_ring[empty_cache_index]);
+ empty_cache_index = -1;
+ if (is_empty())
+ Decommit(root);
+}
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
diff --git a/third_party/base/allocator/partition_allocator/partition_page.h b/third_party/base/allocator/partition_allocator/partition_page.h
new file mode 100644
index 0000000..a40ff8e
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_page.h
@@ -0,0 +1,296 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_PAGE_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_PAGE_H_
+
+#include <string.h>
+
+#include "third_party/base/allocator/partition_allocator/partition_alloc_constants.h"
+#include "third_party/base/allocator/partition_allocator/partition_bucket.h"
+#include "third_party/base/allocator/partition_allocator/partition_cookie.h"
+#include "third_party/base/allocator/partition_allocator/partition_freelist_entry.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+struct PartitionRootBase;
+
+// Some notes on page states. A page can be in one of four major states:
+// 1) Active.
+// 2) Full.
+// 3) Empty.
+// 4) Decommitted.
+// An active page has available free slots. A full page has no free slots. An
+// empty page has no free slots, and a decommitted page is an empty page that
+// had its backing memory released back to the system.
+// There are two linked lists tracking the pages. The "active page" list is an
+// approximation of a list of active pages. It is an approximation because
+// full, empty and decommitted pages may briefly be present in the list until
+// we next do a scan over it.
+// The "empty page" list is an accurate list of pages which are either empty
+// or decommitted.
+//
+// The significant page transitions are:
+// - free() will detect when a full page has a slot free()'d and immediately
+// return the page to the head of the active list.
+// - free() will detect when a page is fully emptied. It _may_ add it to the
+// empty list or it _may_ leave it on the active list until a future list scan.
+// - malloc() _may_ scan the active page list in order to fulfil the request.
+// If it does this, full, empty and decommitted pages encountered will be
+// booted out of the active list. If there are no suitable active pages found,
+// an empty or decommitted page (if one exists) will be pulled from the empty
+// list on to the active list.
+//
+// TODO(ajwong): Evaluate if this should be named PartitionSlotSpanMetadata or
+// similar. If so, all uses of the term "page" in comments, member variables,
+// local variables, and documentation that refer to this concept should be
+// updated.
+struct PartitionPage {
+ PartitionFreelistEntry* freelist_head;
+ PartitionPage* next_page;
+ PartitionBucket* bucket;
+ // Deliberately signed, 0 for empty or decommitted page, -n for full pages:
+ int16_t num_allocated_slots;
+ uint16_t num_unprovisioned_slots;
+ uint16_t page_offset;
+ int16_t empty_cache_index; // -1 if not in the empty cache.
+
+ // Public API
+
+ // Note the matching Alloc() functions are in PartitionPage.
+ BASE_EXPORT NOINLINE void FreeSlowPath();
+ ALWAYS_INLINE void Free(void* ptr);
+
+ void Decommit(PartitionRootBase* root);
+ void DecommitIfPossible(PartitionRootBase* root);
+
+ // Pointer manipulation functions. These must be static as the input |page|
+ // pointer may be the result of an offset calculation and therefore cannot
+ // be trusted. The objective of these functions is to sanitize this input.
+ ALWAYS_INLINE static void* ToPointer(const PartitionPage* page);
+ ALWAYS_INLINE static PartitionPage* FromPointerNoAlignmentCheck(void* ptr);
+ ALWAYS_INLINE static PartitionPage* FromPointer(void* ptr);
+
+ ALWAYS_INLINE const size_t* get_raw_size_ptr() const;
+ ALWAYS_INLINE size_t* get_raw_size_ptr() {
+ return const_cast<size_t*>(
+ const_cast<const PartitionPage*>(this)->get_raw_size_ptr());
+ }
+
+ ALWAYS_INLINE size_t get_raw_size() const;
+ ALWAYS_INLINE void set_raw_size(size_t size);
+
+ ALWAYS_INLINE void Reset();
+
+ // TODO(ajwong): Can this be made private? https://crbug.com/787153
+ BASE_EXPORT static PartitionPage* get_sentinel_page();
+
+ // Page State accessors.
+ // Note that it's only valid to call these functions on pages found on one of
+ // the page lists. Specifically, you can't call these functions on full pages
+ // that were detached from the active list.
+ //
+ // This restriction provides the flexibity for some of the status fields to
+ // be repurposed when a page is taken off a list. See the negation of
+ // |num_allocated_slots| when a full page is removed from the active list
+ // for an example of such repurposing.
+ ALWAYS_INLINE bool is_active() const;
+ ALWAYS_INLINE bool is_full() const;
+ ALWAYS_INLINE bool is_empty() const;
+ ALWAYS_INLINE bool is_decommitted() const;
+
+ private:
+ // g_sentinel_page is used as a sentinel to indicate that there is no page
+ // in the active page list. We can use nullptr, but in that case we need
+ // to add a null-check branch to the hot allocation path. We want to avoid
+ // that.
+ //
+ // Note, this declaration is kept in the header as opposed to an anonymous
+ // namespace so the getter can be fully inlined.
+ static PartitionPage sentinel_page_;
+};
+static_assert(sizeof(PartitionPage) <= kPageMetadataSize,
+ "PartitionPage must be able to fit in a metadata slot");
+
+ALWAYS_INLINE char* PartitionSuperPageToMetadataArea(char* ptr) {
+ uintptr_t pointer_as_uint = reinterpret_cast<uintptr_t>(ptr);
+ DCHECK(!(pointer_as_uint & kSuperPageOffsetMask));
+ // The metadata area is exactly one system page (the guard page) into the
+ // super page.
+ return reinterpret_cast<char*>(pointer_as_uint + kSystemPageSize);
+}
+
+ALWAYS_INLINE PartitionPage* PartitionPage::FromPointerNoAlignmentCheck(
+ void* ptr) {
+ uintptr_t pointer_as_uint = reinterpret_cast<uintptr_t>(ptr);
+ char* super_page_ptr =
+ reinterpret_cast<char*>(pointer_as_uint & kSuperPageBaseMask);
+ uintptr_t partition_page_index =
+ (pointer_as_uint & kSuperPageOffsetMask) >> kPartitionPageShift;
+ // Index 0 is invalid because it is the metadata and guard area and
+ // the last index is invalid because it is a guard page.
+ DCHECK(partition_page_index);
+ DCHECK(partition_page_index < kNumPartitionPagesPerSuperPage - 1);
+ PartitionPage* page = reinterpret_cast<PartitionPage*>(
+ PartitionSuperPageToMetadataArea(super_page_ptr) +
+ (partition_page_index << kPageMetadataShift));
+ // Partition pages in the same slot span can share the same page object.
+ // Adjust for that.
+ size_t delta = page->page_offset << kPageMetadataShift;
+ page =
+ reinterpret_cast<PartitionPage*>(reinterpret_cast<char*>(page) - delta);
+ return page;
+}
+
+// Resturns start of the slot span for the PartitionPage.
+ALWAYS_INLINE void* PartitionPage::ToPointer(const PartitionPage* page) {
+ uintptr_t pointer_as_uint = reinterpret_cast<uintptr_t>(page);
+
+ uintptr_t super_page_offset = (pointer_as_uint & kSuperPageOffsetMask);
+
+ // A valid |page| must be past the first guard System page and within
+ // the following metadata region.
+ DCHECK(super_page_offset > kSystemPageSize);
+ // Must be less than total metadata region.
+ DCHECK(super_page_offset < kSystemPageSize + (kNumPartitionPagesPerSuperPage *
+ kPageMetadataSize));
+ uintptr_t partition_page_index =
+ (super_page_offset - kSystemPageSize) >> kPageMetadataShift;
+ // Index 0 is invalid because it is the superpage extent metadata and the
+ // last index is invalid because the whole PartitionPage is set as guard
+ // pages for the metadata region.
+ DCHECK(partition_page_index);
+ DCHECK(partition_page_index < kNumPartitionPagesPerSuperPage - 1);
+ uintptr_t super_page_base = (pointer_as_uint & kSuperPageBaseMask);
+ void* ret = reinterpret_cast<void*>(
+ super_page_base + (partition_page_index << kPartitionPageShift));
+ return ret;
+}
+
+ALWAYS_INLINE PartitionPage* PartitionPage::FromPointer(void* ptr) {
+ PartitionPage* page = PartitionPage::FromPointerNoAlignmentCheck(ptr);
+ // Checks that the pointer is a multiple of bucket size.
+ DCHECK(!((reinterpret_cast<uintptr_t>(ptr) -
+ reinterpret_cast<uintptr_t>(PartitionPage::ToPointer(page))) %
+ page->bucket->slot_size));
+ return page;
+}
+
+ALWAYS_INLINE const size_t* PartitionPage::get_raw_size_ptr() const {
+ // For single-slot buckets which span more than one partition page, we
+ // have some spare metadata space to store the raw allocation size. We
+ // can use this to report better statistics.
+ if (bucket->slot_size <= kMaxSystemPagesPerSlotSpan * kSystemPageSize)
+ return nullptr;
+
+ DCHECK((bucket->slot_size % kSystemPageSize) == 0);
+ DCHECK(bucket->is_direct_mapped() || bucket->get_slots_per_span() == 1);
+
+ const PartitionPage* the_next_page = this + 1;
+ return reinterpret_cast<const size_t*>(&the_next_page->freelist_head);
+}
+
+ALWAYS_INLINE size_t PartitionPage::get_raw_size() const {
+ const size_t* ptr = get_raw_size_ptr();
+ if (UNLIKELY(ptr != nullptr))
+ return *ptr;
+ return 0;
+}
+
+ALWAYS_INLINE void PartitionPage::Free(void* ptr) {
+ size_t slot_size = this->bucket->slot_size;
+ const size_t raw_size = get_raw_size();
+ if (raw_size) {
+ slot_size = raw_size;
+ }
+
+#if DCHECK_IS_ON()
+ // If these asserts fire, you probably corrupted memory.
+ PartitionCookieCheckValue(ptr);
+ PartitionCookieCheckValue(reinterpret_cast<char*>(ptr) + slot_size -
+ kCookieSize);
+
+ memset(ptr, kFreedByte, slot_size);
+#endif
+
+ DCHECK(this->num_allocated_slots);
+ // TODO(palmer): See if we can afford to make this a CHECK.
+ // FIX FIX FIX
+ // DCHECK(!freelist_head || PartitionRootBase::IsValidPage(
+ // PartitionPage::FromPointer(freelist_head)));
+ CHECK(ptr != freelist_head); // Catches an immediate double free.
+ // Look for double free one level deeper in debug.
+ DCHECK(!freelist_head || ptr != internal::PartitionFreelistEntry::Transform(
+ freelist_head->next));
+ internal::PartitionFreelistEntry* entry =
+ static_cast<internal::PartitionFreelistEntry*>(ptr);
+ entry->next = internal::PartitionFreelistEntry::Transform(freelist_head);
+ freelist_head = entry;
+ --this->num_allocated_slots;
+ if (UNLIKELY(this->num_allocated_slots <= 0)) {
+ FreeSlowPath();
+ } else {
+ // All single-slot allocations must go through the slow path to
+ // correctly update the size metadata.
+ DCHECK(get_raw_size() == 0);
+ }
+}
+
+ALWAYS_INLINE bool PartitionPage::is_active() const {
+ DCHECK(this != get_sentinel_page());
+ DCHECK(!page_offset);
+ return (num_allocated_slots > 0 &&
+ (freelist_head || num_unprovisioned_slots));
+}
+
+ALWAYS_INLINE bool PartitionPage::is_full() const {
+ DCHECK(this != get_sentinel_page());
+ DCHECK(!page_offset);
+ bool ret = (num_allocated_slots == bucket->get_slots_per_span());
+ if (ret) {
+ DCHECK(!freelist_head);
+ DCHECK(!num_unprovisioned_slots);
+ }
+ return ret;
+}
+
+ALWAYS_INLINE bool PartitionPage::is_empty() const {
+ DCHECK(this != get_sentinel_page());
+ DCHECK(!page_offset);
+ return (!num_allocated_slots && freelist_head);
+}
+
+ALWAYS_INLINE bool PartitionPage::is_decommitted() const {
+ DCHECK(this != get_sentinel_page());
+ DCHECK(!page_offset);
+ bool ret = (!num_allocated_slots && !freelist_head);
+ if (ret) {
+ DCHECK(!num_unprovisioned_slots);
+ DCHECK(empty_cache_index == -1);
+ }
+ return ret;
+}
+
+ALWAYS_INLINE void PartitionPage::set_raw_size(size_t size) {
+ size_t* raw_size_ptr = get_raw_size_ptr();
+ if (UNLIKELY(raw_size_ptr != nullptr))
+ *raw_size_ptr = size;
+}
+
+ALWAYS_INLINE void PartitionPage::Reset() {
+ DCHECK(this->is_decommitted());
+
+ num_unprovisioned_slots = bucket->get_slots_per_span();
+ DCHECK(num_unprovisioned_slots);
+
+ next_page = nullptr;
+}
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_PAGE_H_
diff --git a/third_party/base/allocator/partition_allocator/partition_root_base.cc b/third_party/base/allocator/partition_allocator/partition_root_base.cc
new file mode 100644
index 0000000..3136584
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_root_base.cc
@@ -0,0 +1,42 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/base/allocator/partition_allocator/partition_root_base.h"
+
+#include "build/build_config.h"
+#include "third_party/base/allocator/partition_allocator/oom.h"
+#include "third_party/base/allocator/partition_allocator/partition_oom.h"
+#include "third_party/base/allocator/partition_allocator/partition_page.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+NOINLINE void PartitionRootBase::OutOfMemory() {
+#if !defined(ARCH_CPU_64_BITS)
+ // Check whether this OOM is due to a lot of super pages that are allocated
+ // but not committed, probably due to http://crbug.com/421387.
+ if (total_size_of_super_pages + total_size_of_direct_mapped_pages -
+ total_size_of_committed_pages >
+ kReasonableSizeOfUnusedPages) {
+ PartitionOutOfMemoryWithLotsOfUncommitedPages();
+ }
+#endif
+ if (PartitionRootBase::gOomHandlingFunction)
+ (*PartitionRootBase::gOomHandlingFunction)();
+ OOM_CRASH();
+}
+
+void PartitionRootBase::DecommitEmptyPages() {
+ for (size_t i = 0; i < kMaxFreeableSpans; ++i) {
+ internal::PartitionPage* page = global_empty_page_ring[i];
+ if (page)
+ page->DecommitIfPossible(this);
+ global_empty_page_ring[i] = nullptr;
+ }
+}
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
diff --git a/third_party/base/allocator/partition_allocator/partition_root_base.h b/third_party/base/allocator/partition_allocator/partition_root_base.h
new file mode 100644
index 0000000..e4f7228
--- /dev/null
+++ b/third_party/base/allocator/partition_allocator/partition_root_base.h
@@ -0,0 +1,195 @@
+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ROOT_BASE_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ROOT_BASE_H_
+
+#include "build/build_config.h"
+#include "third_party/base/allocator/partition_allocator/page_allocator.h"
+#include "third_party/base/allocator/partition_allocator/partition_alloc_constants.h"
+#include "third_party/base/allocator/partition_allocator/partition_bucket.h"
+#include "third_party/base/allocator/partition_allocator/partition_direct_map_extent.h"
+#include "third_party/base/allocator/partition_allocator/partition_page.h"
+
+namespace pdfium {
+namespace base {
+namespace internal {
+
+struct PartitionPage;
+struct PartitionRootBase;
+
+// An "extent" is a span of consecutive superpages. We link to the partition's
+// next extent (if there is one) to the very start of a superpage's metadata
+// area.
+struct PartitionSuperPageExtentEntry {
+ PartitionRootBase* root;
+ char* super_page_base;
+ char* super_pages_end;
+ PartitionSuperPageExtentEntry* next;
+};
+static_assert(
+ sizeof(PartitionSuperPageExtentEntry) <= kPageMetadataSize,
+ "PartitionSuperPageExtentEntry must be able to fit in a metadata slot");
+
+struct BASE_EXPORT PartitionRootBase {
+ PartitionRootBase();
+ virtual ~PartitionRootBase();
+ size_t total_size_of_committed_pages = 0;
+ size_t total_size_of_super_pages = 0;
+ size_t total_size_of_direct_mapped_pages = 0;
+ // Invariant: total_size_of_committed_pages <=
+ // total_size_of_super_pages +
+ // total_size_of_direct_mapped_pages.
+ unsigned num_buckets = 0;
+ unsigned max_allocation = 0;
+ bool initialized = false;
+ char* next_super_page = nullptr;
+ char* next_partition_page = nullptr;
+ char* next_partition_page_end = nullptr;
+ PartitionSuperPageExtentEntry* current_extent = nullptr;
+ PartitionSuperPageExtentEntry* first_extent = nullptr;
+ PartitionDirectMapExtent* direct_map_list = nullptr;
+ PartitionPage* global_empty_page_ring[kMaxFreeableSpans] = {};
+ int16_t global_empty_page_ring_index = 0;
+ uintptr_t inverted_self = 0;
+
+ // Public API
+
+ // Allocates out of the given bucket. Properly, this function should probably
+ // be in PartitionBucket, but because the implementation needs to be inlined
+ // for performance, and because it needs to inspect PartitionPage,
+ // it becomes impossible to have it in PartitionBucket as this causes a
+ // cyclical dependency on PartitionPage function implementations.
+ //
+ // Moving it a layer lower couples PartitionRootBase and PartitionBucket, but
+ // preserves the layering of the includes.
+ //
+ // Note the matching Free() functions are in PartitionPage.
+ ALWAYS_INLINE void* AllocFromBucket(PartitionBucket* bucket,
+ int flags,
+ size_t size);
+
+ ALWAYS_INLINE static bool IsValidPage(PartitionPage* page);
+ ALWAYS_INLINE static PartitionRootBase* FromPage(PartitionPage* page);
+
+ // gOomHandlingFunction is invoked when PartitionAlloc hits OutOfMemory.
+ static void (*gOomHandlingFunction)();
+ NOINLINE void OutOfMemory();
+
+ ALWAYS_INLINE void IncreaseCommittedPages(size_t len);
+ ALWAYS_INLINE void DecreaseCommittedPages(size_t len);
+ ALWAYS_INLINE void DecommitSystemPages(void* address, size_t length);
+ ALWAYS_INLINE void RecommitSystemPages(void* address, size_t length);
+
+ void DecommitEmptyPages();
+};
+
+ALWAYS_INLINE void* PartitionRootBase::AllocFromBucket(PartitionBucket* bucket,
+ int flags,
+ size_t size) {
+ bool zero_fill = flags & PartitionAllocZeroFill;
+ bool is_already_zeroed = false;
+
+ PartitionPage* page = bucket->active_pages_head;
+ // Check that this page is neither full nor freed.
+ DCHECK(page->num_allocated_slots >= 0);
+ void* ret = page->freelist_head;
+ if (LIKELY(ret != 0)) {
+ // If these DCHECKs fire, you probably corrupted memory. TODO(palmer): See
+ // if we can afford to make these CHECKs.
+ DCHECK(PartitionRootBase::IsValidPage(page));
+
+ // All large allocations must go through the slow path to correctly update
+ // the size metadata.
+ DCHECK(page->get_raw_size() == 0);
+ internal::PartitionFreelistEntry* new_head =
+ internal::PartitionFreelistEntry::Transform(
+ static_cast<internal::PartitionFreelistEntry*>(ret)->next);
+ page->freelist_head = new_head;
+ page->num_allocated_slots++;
+ } else {
+ ret = bucket->SlowPathAlloc(this, flags, size, &is_already_zeroed);
+ // TODO(palmer): See if we can afford to make this a CHECK.
+ DCHECK(!ret ||
+ PartitionRootBase::IsValidPage(PartitionPage::FromPointer(ret)));
+ }
+
+#if DCHECK_IS_ON()
+ if (!ret) {
+ return nullptr;
+ }
+
+ page = PartitionPage::FromPointer(ret);
+ // TODO(ajwong): Can |page->bucket| ever not be |this|? If not, can this just
+ // be bucket->slot_size?
+ size_t new_slot_size = page->bucket->slot_size;
+ size_t raw_size = page->get_raw_size();
+ if (raw_size) {
+ DCHECK(raw_size == size);
+ new_slot_size = raw_size;
+ }
+ size_t no_cookie_size = PartitionCookieSizeAdjustSubtract(new_slot_size);
+ char* char_ret = static_cast<char*>(ret);
+ // The value given to the application is actually just after the cookie.
+ ret = char_ret + kCookieSize;
+
+ // Fill the region kUninitializedByte or 0, and surround it with 2 cookies.
+ PartitionCookieWriteValue(char_ret);
+ if (!zero_fill) {
+ memset(ret, kUninitializedByte, no_cookie_size);
+ } else if (!is_already_zeroed) {
+ memset(ret, 0, no_cookie_size);
+ }
+ PartitionCookieWriteValue(char_ret + kCookieSize + no_cookie_size);
+#else
+ if (ret && zero_fill && !is_already_zeroed) {
+ memset(ret, 0, size);
+ }
+#endif
+
+ return ret;
+}
+
+ALWAYS_INLINE bool PartitionRootBase::IsValidPage(PartitionPage* page) {
+ PartitionRootBase* root = PartitionRootBase::FromPage(page);
+ return root->inverted_self == ~reinterpret_cast<uintptr_t>(root);
+}
+
+ALWAYS_INLINE PartitionRootBase* PartitionRootBase::FromPage(
+ PartitionPage* page) {
+ PartitionSuperPageExtentEntry* extent_entry =
+ reinterpret_cast<PartitionSuperPageExtentEntry*>(
+ reinterpret_cast<uintptr_t>(page) & kSystemPageBaseMask);
+ return extent_entry->root;
+}
+
+ALWAYS_INLINE void PartitionRootBase::IncreaseCommittedPages(size_t len) {
+ total_size_of_committed_pages += len;
+ DCHECK(total_size_of_committed_pages <=
+ total_size_of_super_pages + total_size_of_direct_mapped_pages);
+}
+
+ALWAYS_INLINE void PartitionRootBase::DecreaseCommittedPages(size_t len) {
+ total_size_of_committed_pages -= len;
+ DCHECK(total_size_of_committed_pages <=
+ total_size_of_super_pages + total_size_of_direct_mapped_pages);
+}
+
+ALWAYS_INLINE void PartitionRootBase::DecommitSystemPages(void* address,
+ size_t length) {
+ ::pdfium::base::DecommitSystemPages(address, length);
+ DecreaseCommittedPages(length);
+}
+
+ALWAYS_INLINE void PartitionRootBase::RecommitSystemPages(void* address,
+ size_t length) {
+ CHECK(::pdfium::base::RecommitSystemPages(address, length, PageReadWrite));
+ IncreaseCommittedPages(length);
+}
+
+} // namespace internal
+} // namespace base
+} // namespace pdfium
+
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_PARTITION_ROOT_BASE_H_
diff --git a/third_party/base/allocator/partition_allocator/spin_lock.cc b/third_party/base/allocator/partition_allocator/spin_lock.cc
index 8d7151a..4205583 100644
--- a/third_party/base/allocator/partition_allocator/spin_lock.cc
+++ b/third_party/base/allocator/partition_allocator/spin_lock.cc
@@ -4,9 +4,12 @@
#include "third_party/base/allocator/partition_allocator/spin_lock.h"
+#include "build/build_config.h"
+#include "third_party/base/logging.h"
+
#if defined(OS_WIN)
#include <windows.h>
-#elif defined(OS_POSIX)
+#elif defined(OS_POSIX) || defined(OS_FUCHSIA)
#include <sched.h>
#endif
@@ -21,9 +24,12 @@
// you really should be using a proper lock (such as |base::Lock|)rather than
// these spinlocks.
#if defined(OS_WIN)
+
#define YIELD_PROCESSOR YieldProcessor()
#define YIELD_THREAD SwitchToThread()
-#elif defined(COMPILER_GCC) || defined(__clang__)
+
+#elif defined(OS_POSIX) || defined(OS_FUCHSIA)
+
#if defined(ARCH_CPU_X86_64) || defined(ARCH_CPU_X86)
#define YIELD_PROCESSOR __asm__ __volatile__("pause")
#elif (defined(ARCH_CPU_ARMEL) && __ARM_ARCH >= 6) || defined(ARCH_CPU_ARM64)
@@ -37,22 +43,26 @@
// Don't bother doing using .word here since r2 is the lowest supported mips64
// that Chromium supports.
#define YIELD_PROCESSOR __asm__ __volatile__("pause")
-#endif
-#endif
+#elif defined(ARCH_CPU_PPC64_FAMILY)
+#define YIELD_PROCESSOR __asm__ __volatile__("or 31,31,31")
+#elif defined(ARCH_CPU_S390_FAMILY)
+// just do nothing
+#define YIELD_PROCESSOR ((void)0)
+#endif // ARCH
#ifndef YIELD_PROCESSOR
#warning "Processor yield not supported on this architecture."
#define YIELD_PROCESSOR ((void)0)
#endif
-#ifndef YIELD_THREAD
-#if defined(OS_POSIX)
#define YIELD_THREAD sched_yield()
-#else
+
+#else // Other OS
+
#warning "Thread yield not supported on this OS."
#define YIELD_THREAD ((void)0)
-#endif
-#endif
+
+#endif // OS_WIN
namespace pdfium {
namespace base {
@@ -63,6 +73,9 @@
// critical section defaults, and various other recommendations.
// TODO(jschuh): Further tuning may be warranted.
static const int kYieldProcessorTries = 1000;
+ // The value of |kYieldThreadTries| is completely made up.
+ static const int kYieldThreadTries = 10;
+ int yield_thread_count = 0;
do {
do {
for (int count = 0; count < kYieldProcessorTries; ++count) {
@@ -73,8 +86,17 @@
return;
}
- // Give the OS a chance to schedule something on this core.
- YIELD_THREAD;
+ if (yield_thread_count < kYieldThreadTries) {
+ ++yield_thread_count;
+ // Give the OS a chance to schedule something on this core.
+ YIELD_THREAD;
+ } else {
+ // At this point, it's likely that the lock is held by a lower priority
+ // thread that is unavailable to finish its work because of higher
+ // priority threads spinning here. Sleeping should ensure that they make
+ // progress.
+ NOTREACHED();
+ }
} while (lock_.load(std::memory_order_relaxed));
} while (UNLIKELY(lock_.exchange(true, std::memory_order_acquire)));
}
diff --git a/third_party/base/allocator/partition_allocator/spin_lock.h b/third_party/base/allocator/partition_allocator/spin_lock.h
index 7a42a29..5613fd1 100644
--- a/third_party/base/allocator/partition_allocator/spin_lock.h
+++ b/third_party/base/allocator/partition_allocator/spin_lock.h
@@ -2,8 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
-#ifndef BASE_ALLOCATOR_PARTITION_ALLOCATOR_SPIN_LOCK_H
-#define BASE_ALLOCATOR_PARTITION_ALLOCATOR_SPIN_LOCK_H
+#ifndef THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_SPIN_LOCK_H_
+#define THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_SPIN_LOCK_H_
#include <atomic>
#include <memory>
@@ -17,16 +17,14 @@
// intended only for very short duration locks and assume a system with multiple
// cores. For any potentially longer wait you should use a real lock, such as
// |base::Lock|.
-//
-// |SpinLock|s MUST be globals. Using them as (e.g.) struct/class members will
-// result in an uninitialized lock, which is dangerously incorrect.
-
namespace pdfium {
namespace base {
namespace subtle {
-class SpinLock {
+class BASE_EXPORT SpinLock {
public:
+ constexpr SpinLock() = default;
+ ~SpinLock() = default;
using Guard = std::lock_guard<SpinLock>;
ALWAYS_INLINE void lock() {
@@ -42,13 +40,13 @@
private:
// This is called if the initial attempt to acquire the lock fails. It's
// slower, but has a much better scheduling and power consumption behavior.
- BASE_EXPORT void LockSlow();
+ void LockSlow();
- std::atomic_int lock_;
+ std::atomic_int lock_{0};
};
} // namespace subtle
} // namespace base
} // namespace pdfium
-#endif // BASE_ALLOCATOR_PARTITION_ALLOCATOR_SPIN_LOCK_H
+#endif // THIRD_PARTY_BASE_ALLOCATOR_PARTITION_ALLOCATOR_SPIN_LOCK_H_
diff --git a/third_party/base/stl_util.h b/third_party/base/stl_util.h
index ab3ac60..8163b73 100644
--- a/third_party/base/stl_util.h
+++ b/third_party/base/stl_util.h
@@ -16,6 +16,18 @@
namespace pdfium {
+// C++11 implementation of C++17's std::size():
+// http://en.cppreference.com/w/cpp/iterator/size
+template <typename Container>
+constexpr auto size(const Container& c) -> decltype(c.size()) {
+ return c.size();
+}
+
+template <typename T, size_t N>
+constexpr size_t size(const T (&array)[N]) noexcept {
+ return N;
+}
+
// Test to see if a set, map, hash_set or hash_map contains a particular key.
// Returns true if the key is in the collection.
template <typename Collection, typename Key>
