0

[IC] Implement image cache to control storage classes

Bug: 882967
Change-Id: I114ea70a7f5976d8600f81be22b0096606964dac
Reviewed-on: https://chromium-review.googlesource.com/1225274
Commit-Queue: Brandon Wylie <wylieb@chromium.org>
Reviewed-by: Filip Gorski <fgorski@chromium.org>
Reviewed-by: Gabriel Charette <gab@chromium.org>
Reviewed-by: Dominic Battré <battre@chromium.org>
Reviewed-by: Sky Malice <skym@chromium.org>
Cr-Commit-Position: refs/heads/master@{#592592}
This commit is contained in:
Brandon Wylie
2018-09-19 23:03:37 +00:00
committed by Commit Bot
parent d0ef0ba10e
commit 6961a2ee42
6 changed files with 442 additions and 13 deletions
chrome/browser/prefs
components/image_fetcher/core/storage

@ -84,6 +84,7 @@
#include "components/feature_engagement/buildflags.h"
#include "components/flags_ui/pref_service_flags_storage.h"
#include "components/gcm_driver/gcm_channel_status_syncer.h"
#include "components/image_fetcher/core/storage/image_cache.h"
#include "components/invalidation/impl/per_user_topic_registration_manager.h"
#include "components/language/content/browser/geo_language_provider.h"
#include "components/metrics/metrics_pref_names.h"
@ -549,6 +550,7 @@ void RegisterProfilePrefs(user_prefs::PrefRegistrySyncable* registry) {
DocumentProvider::RegisterProfilePrefs(registry);
DownloadPrefs::RegisterProfilePrefs(registry);
HostContentSettingsMap::RegisterProfilePrefs(registry);
image_fetcher::ImageCache::RegisterProfilePrefs(registry);
ImportantSitesUtil::RegisterProfilePrefs(registry);
IncognitoModePrefs::RegisterProfilePrefs(registry);
MediaCaptureDevicesDispatcher::RegisterProfilePrefs(registry);

@ -16,7 +16,9 @@ static_library("storage") {
]
deps = [
"proto",
"//components/base32",
"//components/leveldb_proto",
"//components/prefs",
"//net",
]
public_deps = [
@ -27,6 +29,7 @@ static_library("storage") {
source_set("storage_unit_tests") {
testonly = true
sources = [
"image_cache_unittest.cc",
"image_data_store_disk_unittest.cc",
"image_metadata_store_leveldb_unittest.cc",
]
@ -36,6 +39,7 @@ source_set("storage_unit_tests") {
"//base",
"//base/test:test_support",
"//components/leveldb_proto:test_support",
"//components/prefs:test_support",
"//testing/gmock",
"//testing/gtest:gtest",
]

@ -1,3 +1,5 @@
include_rules = [
"+components/base32",
"+components/leveldb_proto",
"+components/prefs",
]

@ -6,28 +6,179 @@
#include <utility>
#include "base/bind.h"
#include "base/sha1.h"
#include "base/task/post_task.h"
#include "base/task/task_traits.h"
#include "base/threading/sequenced_task_runner_handle.h"
#include "base/time/clock.h"
#include "base/time/time.h"
#include "components/base32/base32.h"
#include "components/image_fetcher/core/storage/image_data_store.h"
#include "components/image_fetcher/core/storage/image_metadata_store.h"
#include "components/prefs/pref_registry_simple.h"
#include "components/prefs/pref_service.h"
namespace {
constexpr char kPrefLastEvictionKey[] =
"cached_image_fetcher_last_eviction_time";
// TODO(wylieb): Control these parameters server-side.
constexpr size_t kCacheMaxSize = 64 * 1024 * 1024; // 64mb.
// Cache items are allowed to live for the given amount of days.
constexpr size_t kCacheItemsTimeToLiveDays = 7;
constexpr size_t kImageCacheEvictionIntervalHours = 24;
constexpr size_t kImageCacheEvictionDelayMinutes = 5;
std::string HashUrlToKey(const std::string& input) {
return base32::Base32Encode(base::SHA1HashString(input));
}
} // namespace
namespace image_fetcher {
// TODO(wylieb): Implement method stubs.
// TODO(wylieb): Queue requests made before storage is ready.
// static
void ImageCache::RegisterProfilePrefs(PrefRegistrySimple* registry) {
registry->RegisterTimePref(kPrefLastEvictionKey, base::Time());
}
ImageCache::ImageCache(std::unique_ptr<ImageDataStore> data_store,
std::unique_ptr<ImageMetadataStore> metadata_store)
: data_store_(std::move(data_store)),
std::unique_ptr<ImageMetadataStore> metadata_store,
PrefService* pref_service,
base::Clock* clock,
scoped_refptr<base::SequencedTaskRunner> task_runner)
: initialization_attempted_(false),
data_store_(std::move(data_store)),
metadata_store_(std::move(metadata_store)),
pref_service_(pref_service),
clock_(clock),
task_runner_(task_runner),
weak_ptr_factory_(this) {}
ImageCache::~ImageCache() = default;
void ImageCache::SaveImage(const std::string& url,
const std::string& image_data) {}
void ImageCache::SaveImage(std::string url, std::string image_data) {
// If the image data is larger than the cache's max size, bail out.
if (image_data.length() > kCacheMaxSize) {
return;
}
void ImageCache::LoadImage(const std::string& url, ImageDataCallback callback) {
base::OnceClosure request =
base::BindOnce(&ImageCache::SaveImageImpl, weak_ptr_factory_.GetWeakPtr(),
url, std::move(image_data));
QueueOrStartRequest(std::move(request));
}
void ImageCache::DeleteImage(const std::string& url) {}
void ImageCache::LoadImage(std::string url, ImageDataCallback callback) {
base::OnceClosure request =
base::BindOnce(&ImageCache::LoadImageImpl, weak_ptr_factory_.GetWeakPtr(),
url, std::move(callback));
QueueOrStartRequest(std::move(request));
}
void ImageCache::DeleteImage(std::string url) {
base::OnceClosure request = base::BindOnce(
&ImageCache::DeleteImageImpl, weak_ptr_factory_.GetWeakPtr(), url);
QueueOrStartRequest(std::move(request));
}
void ImageCache::QueueOrStartRequest(base::OnceClosure request) {
if (!AreAllDependenciesInitialized()) {
queued_requests_.push_back(std::move(request));
MaybeStartInitialization();
return;
}
// Post task for fairness with tasks that may be queued.
task_runner_->PostTask(FROM_HERE, std::move(request));
}
void ImageCache::MaybeStartInitialization() {
if (initialization_attempted_) {
return;
}
initialization_attempted_ = true;
data_store_->Initialize(base::BindOnce(&ImageCache::OnDependencyInitialized,
weak_ptr_factory_.GetWeakPtr()));
metadata_store_->Initialize(base::BindOnce(
&ImageCache::OnDependencyInitialized, weak_ptr_factory_.GetWeakPtr()));
}
bool ImageCache::AreAllDependenciesInitialized() const {
return data_store_->IsInitialized() && metadata_store_->IsInitialized();
}
void ImageCache::OnDependencyInitialized() {
if (!AreAllDependenciesInitialized()) {
return;
}
// Everything is initialized, take care of the queued requests.
for (base::OnceClosure& request : queued_requests_) {
task_runner_->PostTask(FROM_HERE, std::move(request));
}
queued_requests_.clear();
// TODO(wylieb): Consider delaying eviction as new requests come in via
// seperate weak pointers.
// Once all the queued requests are taken care of, run eviction.
task_runner_->PostDelayedTask(
FROM_HERE,
base::BindOnce(&ImageCache::RunEviction, weak_ptr_factory_.GetWeakPtr()),
base::TimeDelta::FromMinutes(kImageCacheEvictionDelayMinutes));
}
void ImageCache::SaveImageImpl(const std::string& url, std::string image_data) {
std::string key = HashUrlToKey(url);
// TODO(wylieb): Run eviction if size is larger than desired size.
size_t length = image_data.length();
data_store_->SaveImage(key, std::move(image_data));
metadata_store_->SaveImageMetadata(key, length);
}
void ImageCache::LoadImageImpl(const std::string& url,
ImageDataCallback callback) {
std::string key = HashUrlToKey(url);
data_store_->LoadImage(key, std::move(callback));
metadata_store_->UpdateImageMetadata(key);
}
void ImageCache::DeleteImageImpl(const std::string& url) {
std::string key = HashUrlToKey(url);
data_store_->DeleteImage(key);
metadata_store_->DeleteImageMetadata(key);
}
// TODO(wylieb): Support an eviction and reconciliation routine.
void ImageCache::RunEviction() {
base::Time last_eviction_time = pref_service_->GetTime(kPrefLastEvictionKey);
// If we've already garbage collected in the past interval, bail out.
if (last_eviction_time >
clock_->Now() -
base::TimeDelta::FromHours(kImageCacheEvictionIntervalHours)) {
return;
}
base::Time eviction_time = clock_->Now();
pref_service_->SetTime(kPrefLastEvictionKey, eviction_time);
metadata_store_->EvictImageMetadata(
eviction_time - base::TimeDelta::FromDays(kCacheItemsTimeToLiveDays),
kCacheMaxSize,
base::BindOnce(&ImageCache::OnKeysEvicted,
weak_ptr_factory_.GetWeakPtr()));
}
void ImageCache::OnKeysEvicted(std::vector<std::string> keys) {
for (const std::string& key : keys) {
data_store_->DeleteImage(key);
}
}
} // namespace image_fetcher

@ -11,6 +11,14 @@
#include "base/memory/weak_ptr.h"
#include "components/image_fetcher/core/storage/image_store_types.h"
class PrefRegistrySimple;
class PrefService;
namespace base {
class Clock;
class SequencedTaskRunner;
} // namespace base
namespace image_fetcher {
class ImageDataStore;
@ -19,23 +27,62 @@ class ImageMetadataStore;
// Persist image meta/data via the given implementations of ImageDataStore and
// ImageMetadataStore.
class ImageCache {
public:
static void RegisterProfilePrefs(PrefRegistrySimple* registry);
ImageCache(std::unique_ptr<ImageDataStore> data_storage,
std::unique_ptr<ImageMetadataStore> metadata_storage);
std::unique_ptr<ImageMetadataStore> metadata_storage,
PrefService* pref_service,
base::Clock* clock,
scoped_refptr<base::SequencedTaskRunner> task_runner);
~ImageCache();
// Adds or updates the image data for the |url|. If the class hasn't been
// initialized yet, the call is queued.
void SaveImage(const std::string& url, const std::string& image_data);
void SaveImage(std::string url, std::string image_data);
// Loads the image data for the |url| and passes it to |callback|.
void LoadImage(const std::string& url, ImageDataCallback callback);
// Loads the image data for the |url| and passes it to |callback|. If there's
// no image in the cache, then an empty string is returned.
void LoadImage(std::string url, ImageDataCallback callback);
// Deletes the image data for the |url|.
void DeleteImage(const std::string& url);
void DeleteImage(std::string url);
private:
friend class ImageCacheTest;
// Queue or start |request| depending if the cache is initialized.
void QueueOrStartRequest(base::OnceClosure request);
// Start initializing the stores if it hasn't been started already.
void MaybeStartInitialization();
// Returns true iff both the stores have been initialized.
bool AreAllDependenciesInitialized() const;
// Receives callbacks when stores are initialized.
void OnDependencyInitialized();
// Saves the |image_data| for |url|.
void SaveImageImpl(const std::string& url, std::string image_data);
// Loads the data for |url|, calls the user back before updating metadata.
void LoadImageImpl(const std::string& url, ImageDataCallback callback);
// Deletes the data for |url|.
void DeleteImageImpl(const std::string& url);
// Runs eviction on the data stores.
void RunEviction();
// Deletes the given keys from the data store.
void OnKeysEvicted(std::vector<std::string> keys);
bool initialization_attempted_;
std::vector<base::OnceClosure> queued_requests_;
std::unique_ptr<ImageDataStore> data_store_;
std::unique_ptr<ImageMetadataStore> metadata_store_;
PrefService* pref_service_;
// Owned by the service which instantiates this.
base::Clock* clock_;
scoped_refptr<base::SequencedTaskRunner> task_runner_;
base::WeakPtrFactory<ImageCache> weak_ptr_factory_;

@ -0,0 +1,223 @@
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "components/image_fetcher/core/storage/image_cache.h"
#include "base/files/scoped_temp_dir.h"
#include "base/run_loop.h"
#include "base/test/scoped_task_environment.h"
#include "base/test/simple_test_clock.h"
#include "base/threading/sequenced_task_runner_handle.h"
#include "components/image_fetcher/core/storage/image_data_store_disk.h"
#include "components/image_fetcher/core/storage/image_metadata_store_leveldb.h"
#include "components/image_fetcher/core/storage/proto/cached_image_metadata.pb.h"
#include "components/leveldb_proto/testing/fake_db.h"
#include "components/prefs/testing_pref_service.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
using leveldb_proto::test::FakeDB;
using testing::Mock;
namespace image_fetcher {
namespace {
constexpr char kImageUrl[] = "http://gstatic.img.com/foo.jpg";
constexpr char kImageData[] = "data";
} // namespace
class ImageCacheTest : public testing::Test {
public:
ImageCacheTest() {}
void SetUp() override { ASSERT_TRUE(temp_dir_.CreateUniqueTempDir()); }
void CreateImageCache() {
clock_.SetNow(base::Time());
auto db =
std::make_unique<FakeDB<CachedImageMetadataProto>>(&metadata_store_);
db_ = db.get();
auto metadata_store = std::make_unique<ImageMetadataStoreLevelDB>(
base::FilePath(), std::move(db), &clock_);
auto data_store = std::make_unique<ImageDataStoreDisk>(
temp_dir_.GetPath(), base::SequencedTaskRunnerHandle::Get());
ImageCache::RegisterProfilePrefs(test_prefs_.registry());
image_cache_ = std::make_unique<ImageCache>(
std::move(data_store), std::move(metadata_store), &test_prefs_, &clock_,
base::SequencedTaskRunnerHandle::Get());
}
void InitializeImageCache() {
image_cache_->MaybeStartInitialization();
db_->InitCallback(true);
RunUntilIdle();
}
void PrepareImageCache() {
CreateImageCache();
InitializeImageCache();
image_cache()->SaveImage(kImageUrl, kImageData);
RunUntilIdle();
}
bool IsCacheInitialized() {
return image_cache()->AreAllDependenciesInitialized();
}
void RunCacheEviction(bool success) {
image_cache()->RunEviction();
if (success) {
db_->LoadCallback(true);
db_->UpdateCallback(true);
}
RunUntilIdle();
}
void RunUntilIdle() { base::RunLoop().RunUntilIdle(); }
TestingPrefServiceSimple* prefs() { return &test_prefs_; }
base::SimpleTestClock* clock() { return &clock_; }
ImageCache* image_cache() { return image_cache_.get(); }
FakeDB<CachedImageMetadataProto>* db() { return db_; }
MOCK_METHOD1(DataCallback, void(std::string));
private:
std::unique_ptr<ImageCache> image_cache_;
base::SimpleTestClock clock_;
TestingPrefServiceSimple test_prefs_;
base::ScopedTempDir temp_dir_;
FakeDB<CachedImageMetadataProto>* db_;
std::map<std::string, CachedImageMetadataProto> metadata_store_;
base::test::ScopedTaskEnvironment scoped_task_environment_;
DISALLOW_COPY_AND_ASSIGN(ImageCacheTest);
};
TEST_F(ImageCacheTest, SanityTest) {
CreateImageCache();
InitializeImageCache();
image_cache()->SaveImage(kImageUrl, kImageData);
RunUntilIdle();
EXPECT_CALL(*this, DataCallback(kImageData));
image_cache()->LoadImage(
kImageUrl,
base::BindOnce(&ImageCacheTest::DataCallback, base::Unretained(this)));
RunUntilIdle();
image_cache()->DeleteImage(kImageUrl);
RunUntilIdle();
EXPECT_CALL(*this, DataCallback(std::string()));
image_cache()->LoadImage(
kImageUrl,
base::BindOnce(&ImageCacheTest::DataCallback, base::Unretained(this)));
RunUntilIdle();
}
TEST_F(ImageCacheTest, SaveCallsInitialization) {
CreateImageCache();
ASSERT_FALSE(IsCacheInitialized());
image_cache()->SaveImage(kImageUrl, kImageData);
db()->InitCallback(true);
RunUntilIdle();
ASSERT_TRUE(IsCacheInitialized());
}
TEST_F(ImageCacheTest, Save) {
CreateImageCache();
InitializeImageCache();
image_cache()->SaveImage(kImageUrl, kImageData);
RunUntilIdle();
EXPECT_CALL(*this, DataCallback(kImageData));
image_cache()->LoadImage(
kImageUrl,
base::BindOnce(&ImageCacheTest::DataCallback, base::Unretained(this)));
RunUntilIdle();
}
TEST_F(ImageCacheTest, Load) {
PrepareImageCache();
EXPECT_CALL(*this, DataCallback(kImageData));
image_cache()->LoadImage(
kImageUrl,
base::BindOnce(&ImageCacheTest::DataCallback, base::Unretained(this)));
RunUntilIdle();
}
TEST_F(ImageCacheTest, Delete) {
PrepareImageCache();
EXPECT_CALL(*this, DataCallback(kImageData));
image_cache()->LoadImage(
kImageUrl,
base::BindOnce(&ImageCacheTest::DataCallback, base::Unretained(this)));
RunUntilIdle();
image_cache()->DeleteImage(kImageUrl);
RunUntilIdle();
EXPECT_CALL(*this, DataCallback(std::string()));
image_cache()->LoadImage(
kImageUrl,
base::BindOnce(&ImageCacheTest::DataCallback, base::Unretained(this)));
RunUntilIdle();
}
TEST_F(ImageCacheTest, Eviction) {
PrepareImageCache();
clock()->SetNow(clock()->Now() + base::TimeDelta::FromDays(7));
RunCacheEviction(/* success */ true);
EXPECT_CALL(*this, DataCallback(std::string()));
image_cache()->LoadImage(
kImageUrl,
base::BindOnce(&ImageCacheTest::DataCallback, base::Unretained(this)));
RunUntilIdle();
}
TEST_F(ImageCacheTest, EvictionTooSoon) {
PrepareImageCache();
clock()->SetNow(clock()->Now() + base::TimeDelta::FromDays(6));
RunCacheEviction(/* success */ true);
EXPECT_CALL(*this, DataCallback(kImageData));
image_cache()->LoadImage(
kImageUrl,
base::BindOnce(&ImageCacheTest::DataCallback, base::Unretained(this)));
RunUntilIdle();
}
TEST_F(ImageCacheTest, EvictionWhenEvictionAlreadyPerformed) {
PrepareImageCache();
prefs()->SetTime("cached_image_fetcher_last_eviction_time", clock()->Now());
clock()->SetNow(clock()->Now() + base::TimeDelta::FromHours(23));
RunCacheEviction(/* success */ false);
EXPECT_CALL(*this, DataCallback(kImageData));
image_cache()->LoadImage(
kImageUrl,
base::BindOnce(&ImageCacheTest::DataCallback, base::Unretained(this)));
RunUntilIdle();
}
} // namespace image_fetcher