DCScan: Implement DCScan based on userfaultfd
This is a version of DCScan based on userfaultfd and existing safepoint infrastructure. The basic algorithm is: 1) as soon as the scanner starts scanning a slot span, it protects it entirely; 2) when mutator hits the fault, the handler is called. This handler synchronously enters the safepoint. Other mutators hitting the fault will be blocked waiting for the handler to finish scanning; 3) at the end of scanning all slot spans are unprotected again. Bug: 1206047 Change-Id: I7f116d4cbdbd69ad9cd201aaa19ae620d4ed722d Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2875430 Commit-Queue: Anton Bikineev <bikineev@chromium.org> Auto-Submit: Anton Bikineev <bikineev@chromium.org> Reviewed-by: Will Harris <wfh@chromium.org> Reviewed-by: Kentaro Hara <haraken@chromium.org> Reviewed-by: Michael Lippautz <mlippautz@chromium.org> Owners-Override: Kentaro Hara <haraken@chromium.org> Cr-Commit-Position: refs/heads/master@{#884767}
This commit is contained in:

committed by
Chromium LUCI CQ

parent
e92ad0319f
commit
6b5ab9570d
@ -1937,6 +1937,8 @@ component("base") {
|
||||
"allocator/partition_allocator/starscan/starscan_fwd.h",
|
||||
"allocator/partition_allocator/starscan/stats_collector.cc",
|
||||
"allocator/partition_allocator/starscan/stats_collector.h",
|
||||
"allocator/partition_allocator/starscan/write_protector.cc",
|
||||
"allocator/partition_allocator/starscan/write_protector.h",
|
||||
"allocator/partition_allocator/thread_cache.cc",
|
||||
"allocator/partition_allocator/thread_cache.h",
|
||||
"allocator/partition_allocator/yield_processor.h",
|
||||
|
@ -167,7 +167,7 @@ BASE_EXPORT void ConfigurePartitionRefCountSupport(bool enable_ref_count);
|
||||
#endif
|
||||
|
||||
#if BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && PA_ALLOW_PCSCAN
|
||||
BASE_EXPORT void EnablePCScan();
|
||||
BASE_EXPORT void EnablePCScan(bool dcscan);
|
||||
#endif
|
||||
|
||||
} // namespace allocator
|
||||
|
@ -484,8 +484,10 @@ void ConfigurePartitionRefCountSupport(bool enable_ref_count) {
|
||||
#endif // BUILDFLAG(ENABLE_RUNTIME_BACKUP_REF_PTR_CONTROL)
|
||||
|
||||
#if PA_ALLOW_PCSCAN
|
||||
void EnablePCScan() {
|
||||
internal::PCScan::Initialize();
|
||||
void EnablePCScan(bool dcscan) {
|
||||
internal::PCScan::Initialize(
|
||||
dcscan ? internal::PCScan::WantedWriteProtectionMode::kEnabled
|
||||
: internal::PCScan::WantedWriteProtectionMode::kDisabled);
|
||||
internal::PCScan::RegisterScannableRoot(Allocator());
|
||||
if (Allocator() != AlignedAllocator())
|
||||
internal::PCScan::RegisterScannableRoot(AlignedAllocator());
|
||||
|
@ -34,6 +34,12 @@ static_assert(sizeof(void*) != 8, "");
|
||||
#define PA_STARSCAN_NEON_SUPPORTED
|
||||
#endif
|
||||
|
||||
#if defined(PA_HAS_64_BITS_POINTERS) && \
|
||||
(defined(OS_LINUX) || defined(OS_ANDROID))
|
||||
// TODO(bikineev): Enable for ChromeOS.
|
||||
#define PA_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED
|
||||
#endif
|
||||
|
||||
// POSIX is not only UNIX, e.g. macOS and other OSes. We do use Linux-specific
|
||||
// features such as futex(2).
|
||||
#if defined(OS_LINUX) || defined(OS_CHROMEOS) || defined(OS_ANDROID)
|
||||
|
@ -54,5 +54,8 @@ const Feature kPartitionAllocPCScanStackScanning {
|
||||
#endif // defined(PA_PCSCAN_STACK_SUPPORTED)
|
||||
};
|
||||
|
||||
const Feature kPartitionAllocDCScan{"PartitionAllocDCScan",
|
||||
FEATURE_DISABLED_BY_DEFAULT};
|
||||
|
||||
} // namespace features
|
||||
} // namespace base
|
||||
|
@ -28,6 +28,7 @@ extern const BASE_EXPORT Feature kPartitionAllocLargeThreadCacheSize;
|
||||
|
||||
extern const BASE_EXPORT Feature kPartitionAllocPCScanMUAwareScheduler;
|
||||
extern const BASE_EXPORT Feature kPartitionAllocPCScanStackScanning;
|
||||
extern const BASE_EXPORT Feature kPartitionAllocDCScan;
|
||||
|
||||
extern const BASE_EXPORT Feature kPartitionAllocLazyCommit;
|
||||
|
||||
|
@ -9,8 +9,8 @@
|
||||
namespace base {
|
||||
namespace internal {
|
||||
|
||||
void PCScan::Initialize() {
|
||||
PCScanInternal::Instance().Initialize();
|
||||
void PCScan::Initialize(WantedWriteProtectionMode wpmode) {
|
||||
PCScanInternal::Instance().Initialize(wpmode);
|
||||
}
|
||||
|
||||
void PCScan::RegisterScannableRoot(Root* root) {
|
||||
@ -68,8 +68,8 @@ void PCScan::UninitForTesting() {
|
||||
ReinitPCScanMetadataAllocatorForTesting(); // IN-TEST
|
||||
}
|
||||
|
||||
void PCScan::ReinitForTesting() {
|
||||
PCScanInternal::Instance().ReinitForTesting(); // IN-TEST
|
||||
void PCScan::ReinitForTesting(WantedWriteProtectionMode wpmode) {
|
||||
PCScanInternal::Instance().ReinitForTesting(wpmode); // IN-TEST
|
||||
}
|
||||
|
||||
void PCScan::FinishScanForTesting() {
|
||||
|
@ -52,11 +52,18 @@ class BASE_EXPORT PCScan final {
|
||||
kEager,
|
||||
};
|
||||
|
||||
// Based on the provided mode, PCScan will try to use a certain
|
||||
// WriteProtector, if supported by the system.
|
||||
enum class WantedWriteProtectionMode : uint8_t {
|
||||
kDisabled,
|
||||
kEnabled,
|
||||
};
|
||||
|
||||
PCScan(const PCScan&) = delete;
|
||||
PCScan& operator=(const PCScan&) = delete;
|
||||
|
||||
// Initializes PCScan and prepares internal data structures.
|
||||
static void Initialize();
|
||||
static void Initialize(WantedWriteProtectionMode);
|
||||
|
||||
// Registers a root for scanning.
|
||||
static void RegisterScannableRoot(Root* root);
|
||||
@ -132,7 +139,7 @@ class BASE_EXPORT PCScan final {
|
||||
static void FinishScanForTesting();
|
||||
|
||||
// Reinitialize internal structures (e.g. card table).
|
||||
static void ReinitForTesting();
|
||||
static void ReinitForTesting(WantedWriteProtectionMode);
|
||||
|
||||
size_t epoch() const { return scheduler_.epoch(); }
|
||||
|
||||
|
@ -22,6 +22,7 @@
|
||||
#include "base/allocator/partition_allocator/partition_address_space.h"
|
||||
#include "base/allocator/partition_allocator/partition_alloc.h"
|
||||
#include "base/allocator/partition_allocator/partition_alloc_check.h"
|
||||
#include "base/allocator/partition_allocator/partition_alloc_config.h"
|
||||
#include "base/allocator/partition_allocator/partition_alloc_constants.h"
|
||||
#include "base/allocator/partition_allocator/partition_alloc_features.h"
|
||||
#include "base/allocator/partition_allocator/partition_page.h"
|
||||
@ -303,14 +304,11 @@ void PCScanSnapshot::Take(size_t pcscan_epoch) {
|
||||
typename Root::ScopedGuard guard(root->lock_);
|
||||
|
||||
// Take a snapshot of all super pages and scannable slot spans.
|
||||
// TODO(bikineev): Consider making current_extent lock-free and moving it
|
||||
// to the concurrent thread.
|
||||
for (auto* super_page_extent = root->first_extent; super_page_extent;
|
||||
super_page_extent = super_page_extent->next) {
|
||||
for (char* super_page = super_page_extent->super_page_base;
|
||||
super_page != super_page_extent->super_pages_end;
|
||||
super_page += kSuperPageSize) {
|
||||
// TODO(bikineev): Consider following freelists instead of slot spans.
|
||||
const size_t visited_slot_spans = IterateSlotSpans<ThreadSafe>(
|
||||
super_page, true /*with_quarantine*/,
|
||||
[this](SlotSpan* slot_span) -> bool {
|
||||
@ -424,11 +422,13 @@ class PCScanTask final : public base::RefCountedThreadSafe<PCScanTask>,
|
||||
}
|
||||
}
|
||||
~SyncScope() {
|
||||
// First, notify other scanning threads that this thread is done.
|
||||
// First, notify the scanning thread that this thread is done.
|
||||
NotifyThreads();
|
||||
// Then, unprotect all scanned pages, if needed.
|
||||
task_.UnprotectPartitions();
|
||||
if (context == Context::kScanner) {
|
||||
// The scanner thread must wait here until all safepoints leave.
|
||||
// Otherwise, sweeping may free a page that can be accessed by a
|
||||
// Otherwise, sweeping may free a page that can later be accessed by a
|
||||
// descheduled mutator.
|
||||
WaitForOtherThreads();
|
||||
}
|
||||
@ -491,6 +491,9 @@ class PCScanTask final : public base::RefCountedThreadSafe<PCScanTask>,
|
||||
// Clear quarantined objects and prepare card table for fast lookup
|
||||
void ClearQuarantinedObjectsAndPrepareCardTable();
|
||||
|
||||
// Unprotect all slot spans from all partitions.
|
||||
void UnprotectPartitions();
|
||||
|
||||
// Sweeps (frees) unreachable quarantined entries. Returns the size of swept
|
||||
// objects.
|
||||
void SweepQuarantine();
|
||||
@ -507,6 +510,8 @@ class PCScanTask final : public base::RefCountedThreadSafe<PCScanTask>,
|
||||
std::mutex mutex_;
|
||||
std::condition_variable condvar_;
|
||||
std::atomic<size_t> number_of_scanning_threads_{0u};
|
||||
// We can unprotect only once to reduce context-switches.
|
||||
std::once_flag unprotect_once_flag_;
|
||||
PCScan& pcscan_;
|
||||
};
|
||||
|
||||
@ -608,6 +613,19 @@ void PCScanTask::ClearQuarantinedObjectsAndPrepareCardTable() {
|
||||
});
|
||||
}
|
||||
|
||||
void PCScanTask::UnprotectPartitions() {
|
||||
std::call_once(unprotect_once_flag_, [this] {
|
||||
auto& pcscan = PCScanInternal::Instance();
|
||||
const auto unprotect = [&pcscan](const auto& slot_span) {
|
||||
pcscan.UnprotectPages(
|
||||
reinterpret_cast<uintptr_t>(slot_span.begin),
|
||||
(slot_span.end - slot_span.begin) * sizeof(uintptr_t));
|
||||
};
|
||||
snapshot_.large_scan_areas_worklist().VisitNonConcurrently(unprotect);
|
||||
snapshot_.scan_areas_worklist().VisitNonConcurrently(unprotect);
|
||||
});
|
||||
}
|
||||
|
||||
class PCScanScanLoop final : public ScanLoop<PCScanScanLoop> {
|
||||
friend class ScanLoop<PCScanScanLoop>;
|
||||
|
||||
@ -696,7 +714,11 @@ void PCScanTask::ScanPartitions() {
|
||||
// is scanned contains quarantined objects.
|
||||
PCScanSnapshot::LargeScanAreasWorklist::RandomizedView large_scan_areas(
|
||||
snapshot_.large_scan_areas_worklist());
|
||||
large_scan_areas.Visit([this, &scan_loop](auto scan_area) {
|
||||
auto& pcscan = PCScanInternal::Instance();
|
||||
large_scan_areas.Visit([this, &scan_loop, &pcscan](auto scan_area) {
|
||||
// Protect slot span before scanning it.
|
||||
pcscan.ProtectPages(reinterpret_cast<uintptr_t>(scan_area.begin),
|
||||
(scan_area.end - scan_area.begin) * sizeof(uintptr_t));
|
||||
// The bitmap is (a) always guaranteed to exist and (b) the same for all
|
||||
// objects in a given slot span.
|
||||
// TODO(chromium:1129751): Check mutator bitmap as well if performance
|
||||
@ -722,7 +744,10 @@ void PCScanTask::ScanPartitions() {
|
||||
// Scan areas with regular size slots.
|
||||
PCScanSnapshot::ScanAreasWorklist::RandomizedView scan_areas(
|
||||
snapshot_.scan_areas_worklist());
|
||||
scan_areas.Visit([&scan_loop](auto scan_area) {
|
||||
scan_areas.Visit([&scan_loop, &pcscan](auto scan_area) {
|
||||
// Protect slot span before scanning it.
|
||||
pcscan.ProtectPages(reinterpret_cast<uintptr_t>(scan_area.begin),
|
||||
(scan_area.end - scan_area.begin) * sizeof(uintptr_t));
|
||||
scan_loop.Run(scan_area.begin, scan_area.end);
|
||||
});
|
||||
|
||||
@ -970,10 +995,18 @@ PCScanInternal::PCScanInternal() : simd_support_(DetectSimdSupport()) {}
|
||||
|
||||
PCScanInternal::~PCScanInternal() = default;
|
||||
|
||||
void PCScanInternal::Initialize() {
|
||||
void PCScanInternal::Initialize(PCScan::WantedWriteProtectionMode wpmode) {
|
||||
PA_DCHECK(!is_initialized_);
|
||||
CommitCardTable();
|
||||
PCScan::SetClearType(PCScan::ClearType::kLazy);
|
||||
#if defined(PA_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED)
|
||||
if (wpmode == PCScan::WantedWriteProtectionMode::kEnabled)
|
||||
write_protector_ = std::make_unique<UserFaultFDWriteProtector>();
|
||||
else
|
||||
write_protector_ = std::make_unique<NoWriteProtector>();
|
||||
#else
|
||||
write_protector_ = std::make_unique<NoWriteProtector>();
|
||||
#endif // defined(PA_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED)
|
||||
PCScan::SetClearType(write_protector_->SupportedClearType());
|
||||
is_initialized_ = true;
|
||||
}
|
||||
|
||||
@ -1137,6 +1170,23 @@ void* PCScanInternal::GetCurrentThreadStackTop() const {
|
||||
return it != stack_tops_.end() ? it->second : nullptr;
|
||||
}
|
||||
|
||||
void PCScanInternal::ProtectPages(uintptr_t begin, size_t size) {
|
||||
// Slot-span sizes are multiple of system page size. However, the ranges that
|
||||
// are recorded are not, since in the snapshot we only record the used
|
||||
// payload. Therefore we align up the incoming range by 4k. The unused part of
|
||||
// slot-spans doesn't need to be protected (the allocator will enter the
|
||||
// safepoint before trying to allocate from it).
|
||||
PA_DCHECK(write_protector_.get());
|
||||
write_protector_->ProtectPages(
|
||||
begin, (size + SystemPageSize() - 1) & ~(SystemPageSize() - 1));
|
||||
}
|
||||
|
||||
void PCScanInternal::UnprotectPages(uintptr_t begin, size_t size) {
|
||||
PA_DCHECK(write_protector_.get());
|
||||
write_protector_->UnprotectPages(
|
||||
begin, (size + SystemPageSize() - 1) & ~(SystemPageSize() - 1));
|
||||
}
|
||||
|
||||
void PCScanInternal::ClearRootsForTesting() {
|
||||
// Set all roots as non-scannable and non-quarantinable.
|
||||
for (auto* root : scannable_roots_) {
|
||||
@ -1150,9 +1200,9 @@ void PCScanInternal::ClearRootsForTesting() {
|
||||
nonscannable_roots_.ClearForTesting(); // IN-TEST
|
||||
}
|
||||
|
||||
void PCScanInternal::ReinitForTesting() {
|
||||
void PCScanInternal::ReinitForTesting(PCScan::WantedWriteProtectionMode mode) {
|
||||
is_initialized_ = false;
|
||||
Initialize();
|
||||
Initialize(mode);
|
||||
}
|
||||
|
||||
void PCScanInternal::FinishScanForTesting() {
|
||||
|
@ -12,6 +12,7 @@
|
||||
#include "base/allocator/partition_allocator/starscan/metadata_allocator.h"
|
||||
#include "base/allocator/partition_allocator/starscan/pcscan.h"
|
||||
#include "base/allocator/partition_allocator/starscan/starscan_fwd.h"
|
||||
#include "base/allocator/partition_allocator/starscan/write_protector.h"
|
||||
#include "base/memory/scoped_refptr.h"
|
||||
#include "base/no_destructor.h"
|
||||
|
||||
@ -69,7 +70,7 @@ class PCScanInternal final {
|
||||
|
||||
~PCScanInternal();
|
||||
|
||||
void Initialize();
|
||||
void Initialize(PCScan::WantedWriteProtectionMode);
|
||||
bool is_initialized() const { return is_initialized_; }
|
||||
|
||||
void PerformScan(PCScan::InvocationMode);
|
||||
@ -107,9 +108,12 @@ class PCScanInternal final {
|
||||
|
||||
void* GetCurrentThreadStackTop() const;
|
||||
|
||||
void ClearRootsForTesting(); // IN-TEST
|
||||
void ReinitForTesting(); // IN-TEST
|
||||
void FinishScanForTesting(); // IN-TEST
|
||||
void ProtectPages(uintptr_t begin, size_t size);
|
||||
void UnprotectPages(uintptr_t begin, size_t size);
|
||||
|
||||
void ClearRootsForTesting(); // IN-TEST
|
||||
void ReinitForTesting(PCScan::WantedWriteProtectionMode); // IN-TEST
|
||||
void FinishScanForTesting(); // IN-TEST
|
||||
|
||||
private:
|
||||
friend base::NoDestructor<PCScanInternal>;
|
||||
@ -138,6 +142,8 @@ class PCScanInternal final {
|
||||
const char* process_name_ = nullptr;
|
||||
const SimdSupport simd_support_;
|
||||
|
||||
std::unique_ptr<WriteProtector> write_protector_;
|
||||
|
||||
bool is_initialized_ = false;
|
||||
};
|
||||
|
||||
|
@ -44,7 +44,7 @@ class PCScanTest : public testing::Test {
|
||||
PartitionAllocGlobalInit([](size_t) { LOG(FATAL) << "Out of memory"; });
|
||||
// Previous test runs within the same process decommit GigaCage, therefore
|
||||
// we need to make sure that the card table is recommitted for each run.
|
||||
PCScan::ReinitForTesting();
|
||||
PCScan::ReinitForTesting(PCScan::WantedWriteProtectionMode::kDisabled);
|
||||
allocator_.init({PartitionOptions::AlignedAlloc::kAllowed,
|
||||
PartitionOptions::ThreadCache::kDisabled,
|
||||
PartitionOptions::Quarantine::kAllowed,
|
||||
|
@ -60,11 +60,21 @@ class RacefulWorklist {
|
||||
|
||||
void Push(const T& t) { data_.push_back(Node(t)); }
|
||||
|
||||
template <typename Function>
|
||||
void VisitNonConcurrently(Function) const;
|
||||
|
||||
private:
|
||||
Underlying data_;
|
||||
std::atomic<bool> fully_visited_{false};
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
template <typename Function>
|
||||
void RacefulWorklist<T>::VisitNonConcurrently(Function f) const {
|
||||
for (const auto& t : data_)
|
||||
f(t.value);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
template <typename Function>
|
||||
void RacefulWorklist<T>::RandomizedView::Visit(Function f) {
|
||||
|
135
base/allocator/partition_allocator/starscan/write_protector.cc
Normal file
135
base/allocator/partition_allocator/starscan/write_protector.cc
Normal file
@ -0,0 +1,135 @@
|
||||
// Copyright (c) 2021 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#include "base/allocator/partition_allocator/starscan/write_protector.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
|
||||
#include "base/allocator/partition_allocator/address_pool_manager.h"
|
||||
#include "base/allocator/partition_allocator/partition_address_space.h"
|
||||
#include "base/allocator/partition_allocator/partition_alloc_check.h"
|
||||
#include "base/logging.h"
|
||||
#include "base/posix/eintr_wrapper.h"
|
||||
#include "base/threading/platform_thread.h"
|
||||
#include "build/build_config.h"
|
||||
|
||||
#if defined(PA_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED)
|
||||
#include <fcntl.h>
|
||||
#include <linux/userfaultfd.h>
|
||||
#include <poll.h>
|
||||
#include <sys/ioctl.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <sys/types.h>
|
||||
#endif // defined(PA_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED)
|
||||
|
||||
namespace base {
|
||||
namespace internal {
|
||||
|
||||
PCScan::ClearType NoWriteProtector::SupportedClearType() const {
|
||||
return PCScan::ClearType::kLazy;
|
||||
}
|
||||
|
||||
#if defined(PA_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED)
|
||||
namespace {
|
||||
void UserFaultFDThread(int uffd) {
|
||||
PA_DCHECK(-1 != uffd);
|
||||
|
||||
static constexpr char kThreadName[] = "PCScanPFHandler";
|
||||
base::PlatformThread::SetName(kThreadName);
|
||||
|
||||
while (true) {
|
||||
// Pool on the uffd descriptor for page fault events.
|
||||
pollfd pollfd{.fd = uffd, .events = POLLIN};
|
||||
const int nready = HANDLE_EINTR(poll(&pollfd, 1, -1));
|
||||
PA_CHECK(-1 != nready);
|
||||
|
||||
// Get page fault info.
|
||||
uffd_msg msg;
|
||||
const int nread = HANDLE_EINTR(read(uffd, &msg, sizeof(msg)));
|
||||
PA_CHECK(0 != nread);
|
||||
|
||||
// We only expect page faults.
|
||||
PA_DCHECK(UFFD_EVENT_PAGEFAULT == msg.event);
|
||||
// We have subscribed only to wp-fault events.
|
||||
PA_DCHECK(UFFD_PAGEFAULT_FLAG_WP & msg.arg.pagefault.flags);
|
||||
|
||||
// Enter the safepoint. Concurrent faulted writes will wait until safepoint
|
||||
// finishes.
|
||||
PCScan::JoinScanIfNeeded();
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
UserFaultFDWriteProtector::UserFaultFDWriteProtector()
|
||||
: uffd_(syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK)) {
|
||||
if (uffd_ == -1) {
|
||||
LOG(WARNING) << "userfaultfd is not supported by the current kernel";
|
||||
return;
|
||||
}
|
||||
|
||||
PA_PCHECK(-1 != uffd_);
|
||||
|
||||
uffdio_api uffdio_api;
|
||||
uffdio_api.api = UFFD_API;
|
||||
uffdio_api.features = 0;
|
||||
PA_CHECK(-1 != ioctl(uffd_, UFFDIO_API, &uffdio_api));
|
||||
PA_CHECK(UFFD_API == uffdio_api.api);
|
||||
|
||||
// Register the giga-cage to listen uffd events.
|
||||
struct uffdio_register uffdio_register;
|
||||
uffdio_register.range.start = PartitionAddressSpace::BRPPoolBase();
|
||||
uffdio_register.range.len = AddressPoolManager::kBRPPoolMaxSize;
|
||||
uffdio_register.mode = UFFDIO_REGISTER_MODE_WP;
|
||||
PA_CHECK(-1 != ioctl(uffd_, UFFDIO_REGISTER, &uffdio_register));
|
||||
|
||||
// Start uffd thread.
|
||||
std::thread(UserFaultFDThread, uffd_).detach();
|
||||
}
|
||||
|
||||
namespace {
|
||||
enum class UserFaultFDWPMode {
|
||||
kProtect,
|
||||
kUnprotect,
|
||||
};
|
||||
|
||||
void UserFaultFDWPSet(int uffd,
|
||||
uintptr_t begin,
|
||||
size_t length,
|
||||
UserFaultFDWPMode mode) {
|
||||
PA_DCHECK(0 == (begin % SystemPageSize()));
|
||||
PA_DCHECK(0 == (length % SystemPageSize()));
|
||||
|
||||
uffdio_writeprotect wp;
|
||||
wp.range.start = begin;
|
||||
wp.range.len = length;
|
||||
wp.mode =
|
||||
(mode == UserFaultFDWPMode::kProtect) ? UFFDIO_WRITEPROTECT_MODE_WP : 0;
|
||||
PA_PCHECK(-1 != ioctl(uffd, UFFDIO_WRITEPROTECT, &wp));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void UserFaultFDWriteProtector::ProtectPages(uintptr_t begin, size_t length) {
|
||||
if (IsSupported())
|
||||
UserFaultFDWPSet(uffd_, begin, length, UserFaultFDWPMode::kProtect);
|
||||
}
|
||||
|
||||
void UserFaultFDWriteProtector::UnprotectPages(uintptr_t begin, size_t length) {
|
||||
if (IsSupported())
|
||||
UserFaultFDWPSet(uffd_, begin, length, UserFaultFDWPMode::kUnprotect);
|
||||
}
|
||||
|
||||
PCScan::ClearType UserFaultFDWriteProtector::SupportedClearType() const {
|
||||
return IsSupported() ? PCScan::ClearType::kEager : PCScan::ClearType::kLazy;
|
||||
}
|
||||
|
||||
bool UserFaultFDWriteProtector::IsSupported() const {
|
||||
return uffd_ != -1;
|
||||
}
|
||||
|
||||
#endif // defined(PA_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED)
|
||||
|
||||
} // namespace internal
|
||||
} // namespace base
|
@ -0,0 +1,64 @@
|
||||
// Copyright (c) 2021 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#ifndef BASE_ALLOCATOR_PARTITION_ALLOCATOR_STARSCAN_WRITE_PROTECTOR_H_
|
||||
#define BASE_ALLOCATOR_PARTITION_ALLOCATOR_STARSCAN_WRITE_PROTECTOR_H_
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <mutex>
|
||||
|
||||
#include "base/allocator/partition_allocator/starscan/metadata_allocator.h"
|
||||
#include "base/allocator/partition_allocator/starscan/pcscan.h"
|
||||
#include "base/allocator/partition_allocator/starscan/raceful_worklist.h"
|
||||
#include "build/build_config.h"
|
||||
|
||||
namespace base {
|
||||
namespace internal {
|
||||
|
||||
// Interface for page protection/unprotection. This is used in DCScan to catch
|
||||
// concurrent mutator writes. Protection is done when the scanner starts
|
||||
// scanning a range. Unprotection happens at the end of the scanning phase.
|
||||
class WriteProtector : public AllocatedOnPCScanMetadataPartition {
|
||||
public:
|
||||
virtual ~WriteProtector() = default;
|
||||
|
||||
virtual void ProtectPages(uintptr_t begin, size_t length) = 0;
|
||||
virtual void UnprotectPages(uintptr_t begin, size_t length) = 0;
|
||||
|
||||
virtual PCScan::ClearType SupportedClearType() const = 0;
|
||||
};
|
||||
|
||||
class NoWriteProtector final : public WriteProtector {
|
||||
public:
|
||||
void ProtectPages(uintptr_t, size_t) final {}
|
||||
void UnprotectPages(uintptr_t, size_t) final {}
|
||||
PCScan::ClearType SupportedClearType() const final;
|
||||
};
|
||||
|
||||
#if defined(PA_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED)
|
||||
class UserFaultFDWriteProtector final : public WriteProtector {
|
||||
public:
|
||||
UserFaultFDWriteProtector();
|
||||
|
||||
UserFaultFDWriteProtector(const UserFaultFDWriteProtector&) = delete;
|
||||
UserFaultFDWriteProtector& operator=(const UserFaultFDWriteProtector&) =
|
||||
delete;
|
||||
|
||||
void ProtectPages(uintptr_t, size_t) final;
|
||||
void UnprotectPages(uintptr_t, size_t) final;
|
||||
|
||||
PCScan::ClearType SupportedClearType() const final;
|
||||
|
||||
private:
|
||||
bool IsSupported() const;
|
||||
|
||||
const int uffd_ = 0;
|
||||
};
|
||||
#endif // defined(PA_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED)
|
||||
|
||||
} // namespace internal
|
||||
} // namespace base
|
||||
|
||||
#endif // BASE_ALLOCATOR_PARTITION_ALLOCATOR_STARSCAN_WRITE_PROTECTOR_H_
|
@ -55,7 +55,7 @@ bool EnablePCScanForMallocPartitionsIfNeeded() {
|
||||
#if BUILDFLAG(USE_PARTITION_ALLOC_AS_MALLOC) && PA_ALLOW_PCSCAN
|
||||
DCHECK(base::FeatureList::GetInstance());
|
||||
if (base::FeatureList::IsEnabled(base::features::kPartitionAllocPCScan)) {
|
||||
base::allocator::EnablePCScan();
|
||||
base::allocator::EnablePCScan(/*dcscan*/ false);
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
@ -67,7 +67,13 @@ bool EnablePCScanForMallocPartitionsInBrowserProcessIfNeeded() {
|
||||
DCHECK(base::FeatureList::GetInstance());
|
||||
if (base::FeatureList::IsEnabled(
|
||||
base::features::kPartitionAllocPCScanBrowserOnly)) {
|
||||
base::allocator::EnablePCScan();
|
||||
const bool dcscan_wanted =
|
||||
base::FeatureList::IsEnabled(base::features::kPartitionAllocDCScan);
|
||||
#if !defined(PA_STARSCAN_UFFD_WRITE_PROTECTOR_SUPPORTED)
|
||||
CHECK(!dcscan_wanted)
|
||||
<< "DCScan is currently only supported on Linux based systems";
|
||||
#endif
|
||||
base::allocator::EnablePCScan(dcscan_wanted);
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
Reference in New Issue
Block a user