// Copyright 2021 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// How dynamic assets are handled:
//
// Terminology:
// "Position" - A physical location on the screen where a dynamic asset appears.
// Its identifier is arbitrary, opaque, and embedded in the dynamic asset's id.
//
// "Index"- There shall be 1 or more assets assigned to each position. For
// example, if an animation has a cross-fade transition from image 1 to image 2,
// there may be 2 dynamic assets in the animation that share the same position.
// However, their indices will be different. Example:
// "_CrOS_Photo_PositionA_1" (Index 1 Position A)
// "_CrOS_Photo_PositionA_2" (Index 2 Position A)
// ...
//
// Now we'll step through an example with 2 positions and 2 assets per position:
// "_CrOS_Photo_PositionA_1"
// "_CrOS_Photo_PositionA_2"
// "_CrOS_Photo_PositionB_1"
// "_CrOS_Photo_PositionB_2"
//
// On the very first frame, the provider assigns a topic to each asset in the
// model using all topics available in the model:
// "_CrOS_Photo_PositionA_1" -> "TopicA"
// "_CrOS_Photo_PositionA_2" -> "TopicB"
// "_CrOS_Photo_PositionB_1" -> "TopicC"
// "_CrOS_Photo_PositionB_2" -> "TopicD"
//
// At the start of each new animation cycle, the provider first "rotates" the
// topics at each position. The topic previously assigned to asset with index
// <i + 1> is now assigned to asset with index <i> for all <i> at a given
// position. After rotation, the asset with the highest index at each position
// is left without an assigned topic:
// "_CrOS_Photo_PositionA_1" -> "TopicB"
// "_CrOS_Photo_PositionA_2" -> ???
// "_CrOS_Photo_PositionB_1" -> "TopicD"
// "_CrOS_Photo_PositionB_2" -> ???
//
// The provider then pulls the latest 2 topics from the model (since there are
// 2 assets left now without an assigned topic), and assigns a new topic to
// those assets.
// "_CrOS_Photo_PositionA_1" -> "TopicB"
// "_CrOS_Photo_PositionA_2" -> "TopicE" (new)
// "_CrOS_Photo_PositionB_1" -> "TopicD"
// "_CrOS_Photo_PositionB_2" -> "TopicF" (new)
//
// The process above repeats for each new animation cycle. Note the process
// generalizes to the simplest case where there is only 1 assigned topic per
// position. Rotation will just leave all assets in the animation without an
// assigned topic.
#include "ash/ambient/model/ambient_animation_photo_provider.h"
#include <algorithm>
#include <functional>
#include <iterator>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include "ash/ambient/metrics/ambient_metrics.h"
#include "ash/ambient/resources/ambient_animation_resource_constants.h"
#include "ash/ambient/resources/ambient_animation_static_resources.h"
#include "ash/ambient/util/ambient_util.h"
#include "ash/utility/cropping_util.h"
#include "ash/utility/lottie_util.h"
#include "base/check.h"
#include "base/functional/bind.h"
#include "base/functional/callback.h"
#include "base/logging.h"
#include "base/memory/scoped_refptr.h"
#include "base/notreached.h"
#include "base/numerics/ranges.h"
#include "base/rand_util.h"
#include "cc/paint/paint_flags.h"
#include "cc/paint/skottie_frame_data.h"
#include "ui/gfx/geometry/size.h"
#include "ui/gfx/image/image_skia.h"
#include "ui/gfx/image/image_skia_rep.h"
namespace ash {
namespace {
// TODO(esum): Experiment with different filter qualities for different asset
// types. Thus far, "high" quality has a large impact on performance;
// the frame rate is cut in half due to the increased computational
// complexity. "Medium" quality is the best compromise so far with little to
// no visible difference from "high" quality while maintaining close to 60
// fps.
constexpr cc::PaintFlags::FilterQuality kFilterQuality =
cc::PaintFlags::FilterQuality::kMedium;
cc::SkottieFrameData BuildSkottieFrameData(const gfx::ImageSkia& image,
float scale_factor) {
DCHECK(!image.isNull());
const gfx::ImageSkiaRep& image_rep = image.GetRepresentation(scale_factor);
DCHECK(!image_rep.is_null());
DCHECK(image_rep.has_paint_image());
return {
/*image=*/image_rep.paint_image(),
/*quality=*/kFilterQuality,
};
}
bool IsPortrait(const gfx::Size& size) {
DCHECK(!size.IsEmpty());
return size.height() > size.width();
}
// Provides images for dynamic assets based on the following UX requirements:
// * Make a best effort to assign portrait images to portrait assets and same
// for landscape.
// * If there are less topics available than the number of dynamic assets in
// the animation, the available photos should be evenly distributed and
// duplicated among the assets. For example, if there are 2 topics available
// and 6 dynamic assets, each topic should appear 3 times.
// * The photos should be shuffled among the assets between animation cycles.
class DynamicImageProvider {
public:
using TopicReferenceVector =
std::vector<std::reference_wrapper<const PhotoWithDetails>>;
explicit DynamicImageProvider(TopicReferenceVector all_available_topics) {
DCHECK(!all_available_topics.empty())
<< "Animation should not have started rendering without any decoded "
"photos in the model.";
base::RandomShuffle(all_available_topics.begin(),
all_available_topics.end());
for (auto& topic_ref : all_available_topics) {
// Note the AmbientPhotoConfig for animations states that topics from IMAX
// containing primary and related photos should be split into 2. So the
// related photo should always be null (hence no point in reading it).
DCHECK(!topic_ref.get().photo.isNull());
if (IsPortrait(topic_ref.get().photo.size())) {
portrait_set_.topics.push_back(std::move(topic_ref));
} else {
landscape_set_.topics.push_back(std::move(topic_ref));
}
}
}
const PhotoWithDetails& GetTopicForAssetSize(
const std::optional<gfx::Size>& asset_size) {
const PhotoWithDetails* topic = nullptr;
// If the |asset_size| is unavailable, this is unexpected but not fatal. The
// choice to default to portrait is arbitrary.
if (!asset_size || IsPortrait(*asset_size)) {
topic = GetNextTopic(/*primary_topic_set=*/portrait_set_,
/*secondary_topic_set=*/landscape_set_);
} else {
topic = GetNextTopic(/*primary_topic_set=*/landscape_set_,
/*secondary_topic_set=*/portrait_set_);
}
DCHECK(topic);
TryResetCurrentTopicIndices();
return *topic;
}
private:
struct TopicSet {
// Not mutated after DynamicImageProvider's constructor.
TopicReferenceVector topics;
// Incremented each time a topic is picked from the set and loops back to
// 0 when all topics from all TopicSets have been exhausted.
size_t current_topic_idx = 0;
};
static const PhotoWithDetails* GetNextTopicFromTopicSet(TopicSet& topic_set) {
if (topic_set.current_topic_idx >= topic_set.topics.size())
return nullptr;
const PhotoWithDetails* topic =
&topic_set.topics[topic_set.current_topic_idx].get();
++topic_set.current_topic_idx;
return topic;
}
static const PhotoWithDetails* GetNextTopic(TopicSet& primary_topic_set,
TopicSet& secondary_topic_set) {
const PhotoWithDetails* topic = GetNextTopicFromTopicSet(primary_topic_set);
return topic ? topic : GetNextTopicFromTopicSet(secondary_topic_set);
}
void TryResetCurrentTopicIndices() {
// Once all available topics have been exhausted, reset the
// |current_topic_idx| for each TopicSet to start "fresh" again.
if (landscape_set_.current_topic_idx >= landscape_set_.topics.size() &&
portrait_set_.current_topic_idx >= portrait_set_.topics.size()) {
landscape_set_.current_topic_idx = 0;
portrait_set_.current_topic_idx = 0;
}
}
TopicSet landscape_set_;
TopicSet portrait_set_;
};
} // namespace
class AmbientAnimationPhotoProvider::StaticImageAssetImpl
: public cc::SkottieFrameDataProvider::ImageAsset {
public:
StaticImageAssetImpl(std::string_view asset_id,
const AmbientAnimationStaticResources& static_resources)
: image_(static_resources.GetStaticImageAsset(asset_id)) {
DCHECK(!IsCustomizableLottieId(asset_id));
DCHECK(!image_.isNull())
<< "Static image asset " << asset_id << " is unknown.";
DVLOG(1) << "Loaded static asset " << asset_id;
}
cc::SkottieFrameData GetFrameData(float t, float scale_factor) override {
if (!enabled_)
return cc::SkottieFrameData();
if (!current_frame_data_.image ||
current_frame_data_scale_factor_ != scale_factor) {
current_frame_data_ = BuildSkottieFrameData(image_, scale_factor);
current_frame_data_scale_factor_ = scale_factor;
}
return current_frame_data_;
}
bool enabled() const { return enabled_; }
void set_enabled(bool enabled) { enabled_ = enabled; }
private:
// Private destructor since cc::SkottieFrameDataProvider::ImageAsset is a
// ref-counted API.
~StaticImageAssetImpl() override = default;
const gfx::ImageSkia image_;
cc::SkottieFrameData current_frame_data_;
float current_frame_data_scale_factor_ = 0;
bool enabled_ = true;
};
class AmbientAnimationPhotoProvider::DynamicImageAssetImpl
: public cc::SkottieFrameDataProvider::ImageAsset {
public:
DynamicImageAssetImpl(
std::string_view asset_id,
std::optional<gfx::Size> size,
const base::WeakPtr<AmbientAnimationPhotoProvider>& provider)
: asset_id_(asset_id), size_(std::move(size)), provider_(provider) {
DCHECK(provider_);
if (!ambient::util::ParseDynamicLottieAssetId(asset_id, parsed_asset_id_)) {
LOG(DFATAL) << "Animation file is invalid. Failed to parse dynamic "
"image asset id "
<< asset_id;
}
if (!size_)
DLOG(ERROR) << "Dimensions unavailable for dynamic asset " << asset_id_;
}
cc::SkottieFrameData GetFrameData(float t, float scale_factor) override {
DVLOG(4) << "GetFrameData for asset " << asset_id_ << " time " << t;
bool is_first_rendered_frame =
last_observed_animation_timestamp_ == kAnimationTimestampInvalid;
// The animation frame timestamp units are dictated by Skottie and are
// irrelevant here. The timestamp for each individual asset is monotonically
// increasing until the animation loops back to the beginning, indicating
// the start of a new cycle.
bool is_starting_new_cycle = t < last_observed_animation_timestamp_;
last_observed_animation_timestamp_ = t;
if (is_first_rendered_frame || is_starting_new_cycle) {
DVLOG(4) << "Returning new image for dynamic asset " << asset_id_;
if (provider_) {
current_topic_ = provider_->GenerateNextTopicForDynamicAsset(*this);
// Force |current_frame_data_| to be reset below.
current_frame_data_scale_factor_ = kImageScaleFactorInvalid;
} else {
// If this corner case does somehow happen, it will only be for a brief
// period when the animation is being torn down.
DVLOG(1) << "AmbientAnimationPhotoProvider has been destroyed. Cannot "
"refresh images.";
}
} else {
DVLOG(4) << "No update required to dynamic asset at this time";
}
SetCurrentFrameDataForScale(scale_factor);
return current_frame_data_;
}
PhotoWithDetails ExtractAssignedTopic() {
current_frame_data_ = cc::SkottieFrameData();
current_frame_data_scale_factor_ = kImageScaleFactorInvalid;
return std::move(current_topic_);
}
bool HasAssignedTopic() const { return !current_topic_.photo.isNull(); }
const std::optional<gfx::Size>& size() const { return size_; }
const std::string& asset_id() const { return asset_id_; }
const ambient::util::ParsedDynamicAssetId& parsed_asset_id() const {
return parsed_asset_id_;
}
const std::string& position_id() const {
return parsed_asset_id_.position_id;
}
int idx() const { return parsed_asset_id_.idx; }
private:
static constexpr float kAnimationTimestampInvalid = -1.f;
static constexpr float kImageScaleFactorInvalid = 0.f;
// Private destructor since cc::SkottieFrameDataProvider::ImageAsset is a
// ref-counted API.
~DynamicImageAssetImpl() override = default;
void SetCurrentFrameDataForScale(float scale_factor) {
static constexpr float kScaleFactorEpsilon = 0.01f;
DCHECK(!current_topic_.photo.isNull());
if (current_frame_data_scale_factor_ != kImageScaleFactorInvalid &&
base::IsApproximatelyEqual(current_frame_data_scale_factor_,
scale_factor, kScaleFactorEpsilon)) {
DVLOG(4) << "Current frame data already matches target scale.";
return;
}
// First load the closest image representation from the source at the target
// |scale_factor|, then crop the image representation to the asset's aspect
// ratio.
const gfx::ImageSkiaRep& image_rep =
current_topic_.photo.GetRepresentation(scale_factor);
DCHECK(!image_rep.is_null());
cc::PaintImage paint_image;
if (size_) {
// Crop the image such that it exactly matches this asset's aspect ratio.
// Skottie will handle rescaling the image to the exact desired
// dimensions farther down the pipeline.
SkBitmap cropped_bitmap = CenterCropImage(image_rep.GetBitmap(), *size_);
// Prevents a deep copy in PaintImage::CreateFromBitmap().
cropped_bitmap.setImmutable();
paint_image = cc::PaintImage::CreateFromBitmap(std::move(cropped_bitmap));
} else {
DLOG(ERROR) << "Dynamic asset " << asset_id_
<< " missing dimensions in lottie file";
DCHECK(image_rep.has_paint_image());
paint_image = image_rep.paint_image();
}
current_frame_data_.image = std::move(paint_image);
current_frame_data_.quality = kFilterQuality;
current_frame_data_scale_factor_ = scale_factor;
}
const std::string asset_id_;
ambient::util::ParsedDynamicAssetId parsed_asset_id_;
const std::optional<gfx::Size> size_;
const base::WeakPtr<AmbientAnimationPhotoProvider> provider_;
// Last animation frame timestamp that was observed.
float last_observed_animation_timestamp_ = kAnimationTimestampInvalid;
cc::SkottieFrameData current_frame_data_;
float current_frame_data_scale_factor_ = kImageScaleFactorInvalid;
// The original topic off of which |current_frame_data_| was built. May have
// multiple scale representations in its image in the event that a different
// scale factor is required while rendering.
PhotoWithDetails current_topic_;
};
bool AmbientAnimationPhotoProvider::OrderDynamicAssetsByIdx::operator()(
const scoped_refptr<DynamicImageAssetImpl>& asset_l,
const scoped_refptr<DynamicImageAssetImpl>& asset_r) const {
DCHECK(asset_l);
DCHECK(asset_r);
return asset_l->idx() < asset_r->idx();
}
AmbientAnimationPhotoProvider::AmbientAnimationPhotoProvider(
const AmbientAnimationStaticResources* static_resources,
const AmbientBackendModel* backend_model)
: static_resources_(static_resources),
backend_model_(backend_model),
weak_factory_(this) {
DCHECK(static_resources_);
DCHECK(backend_model_);
}
AmbientAnimationPhotoProvider::~AmbientAnimationPhotoProvider() = default;
scoped_refptr<cc::SkottieFrameDataProvider::ImageAsset>
AmbientAnimationPhotoProvider::LoadImageAsset(
std::string_view asset_id,
const base::FilePath& resource_path,
const std::optional<gfx::Size>& size) {
// Note in practice, all of the image assets are loaded one time by Skottie
// when the animation is initially loaded. So the set of assets does not
// change once the animation starts rendering.
if (IsCustomizableLottieId(asset_id)) {
auto dynamic_asset = base::MakeRefCounted<DynamicImageAssetImpl>(
asset_id, size, weak_factory_.GetWeakPtr());
dynamic_assets_per_position_[dynamic_asset->position_id()].insert(
dynamic_asset);
++total_num_dynamic_assets_;
return dynamic_asset;
} else {
// For static assets, the |size| isn't needed. It should match the size of
// the image loaded from animation's |static_resources_| since that is the
// very image created by UX when the animation was built.
auto static_asset = base::MakeRefCounted<StaticImageAssetImpl>(
asset_id, *static_resources_);
const auto hash_id = cc::HashSkottieResourceId(asset_id);
static_assets_[hash_id] = static_asset;
if (hash_id ==
cc::HashSkottieResourceId(ambient::resources::kTreeShadowAssetId)) {
static_asset->set_enabled(enable_tree_shadow_);
}
return static_asset;
}
}
void AmbientAnimationPhotoProvider::AddObserver(Observer* obs) {
observers_.AddObserver(obs);
}
void AmbientAnimationPhotoProvider::RemoveObserver(Observer* obs) {
observers_.RemoveObserver(obs);
}
bool AmbientAnimationPhotoProvider::ToggleStaticImageAsset(
cc::SkottieResourceIdHash asset_id,
bool enabled) {
auto iter = static_assets_.find(asset_id);
if (iter == static_assets_.end()) {
// When the view is first created, all assets might not be loaded yet. Store
// the `enabled` state to apply on the tree shadow asset when it is loaded.
enable_tree_shadow_ = enabled;
} else {
iter->second->set_enabled(enabled);
}
return true;
}
// Invoked whenever an asset detects a new animation cycle has started. In
// practice, there may be multiple dynamic assets in an animation. So the
// first asset that detects a new animation cycle (which is arbitrary), will
// cause the provider internally to find a new topic for *all* dynamic assets in
// the animation. The provider then returns the first asset's assigned topic and
// saves the other N - 1 assets' topics, marking them as pending. When the other
// N - 1 assets call GenerateNextTopicForDynamicAsset() shortly after, the
// provider simply retrieves the corresponding pending topic until the set of
// pending topics is empty. This process then repeats at the start of the next
// animation cycle.
PhotoWithDetails
AmbientAnimationPhotoProvider::GenerateNextTopicForDynamicAsset(
const DynamicImageAssetImpl& target_asset) {
DVLOG(4) << __func__;
PhotoWithDetails topic_for_target_asset =
ExtractPendingTopicForDynamicAsset(target_asset);
if (!topic_for_target_asset.photo.isNull()) {
return topic_for_target_asset;
}
DCHECK(pending_dynamic_asset_topics_.empty())
<< "All pending topics should have been returned before the first frame "
"of each animation cycle.";
RotateDynamicAssetTopics();
DynamicImageProvider image_provider(GetTopicsToChooseFrom());
for (const auto& [_, dynamic_asset_set] : dynamic_assets_per_position_) {
for (const auto& dynamic_asset : dynamic_asset_set) {
bool asset_already_has_assigned_topic =
pending_dynamic_asset_topics_.contains(dynamic_asset.get());
if (asset_already_has_assigned_topic)
continue;
pending_dynamic_asset_topics_.emplace(
dynamic_asset.get(),
image_provider.GetTopicForAssetSize(dynamic_asset->size()));
}
}
NotifyObserverOfNewTopics();
topic_for_target_asset = ExtractPendingTopicForDynamicAsset(target_asset);
DCHECK(!topic_for_target_asset.photo.isNull())
<< "GenerateNextTopicForDynamicAsset() for unknown asset "
<< target_asset.asset_id();
return topic_for_target_asset;
}
PhotoWithDetails
AmbientAnimationPhotoProvider::ExtractPendingTopicForDynamicAsset(
const DynamicImageAssetImpl& asset) {
auto pending_topic_iter = pending_dynamic_asset_topics_.find(&asset);
if (pending_topic_iter == pending_dynamic_asset_topics_.end()) {
return PhotoWithDetails();
} else {
PhotoWithDetails pending_topic = std::move(pending_topic_iter->second);
pending_dynamic_asset_topics_.erase(pending_topic_iter);
return pending_topic;
}
}
// For each position, the topic assigned to asset with index <i + 1> gets
// assigned to the asset with index <i>. Ultimately, the asset with the highest
// index at each position is left without an assigned topic. See comments at
// the top of the file for an example.
void AmbientAnimationPhotoProvider::RotateDynamicAssetTopics() {
for (const auto& [_, dynamic_asset_set] : dynamic_assets_per_position_) {
DCHECK(!dynamic_asset_set.empty());
auto current_asset = dynamic_asset_set.begin();
auto next_asset = std::next(current_asset);
for (; next_asset != dynamic_asset_set.end();
++current_asset, ++next_asset) {
// HasAssignedTopic() should only be false on the very first frame.
if ((*next_asset)->HasAssignedTopic()) {
pending_dynamic_asset_topics_[current_asset->get()] =
(*next_asset)->ExtractAssignedTopic();
}
}
}
}
std::vector<std::reference_wrapper<const PhotoWithDetails>>
AmbientAnimationPhotoProvider::GetTopicsToChooseFrom() const {
const base::circular_deque<PhotoWithDetails>& all_available_topics =
backend_model_->all_decoded_topics();
size_t num_assets_without_assigned_topic =
total_num_dynamic_assets_ - pending_dynamic_asset_topics_.size();
// Clamp |num_assets_without_assigned_topic| in case the controller is having
// a hard time preparing new topics (ex: network congestion) and there are
// minimal topics in the model (1 is the bare minimum).
size_t num_available_topics = all_available_topics.size();
size_t num_topics_to_choose_from =
std::min(num_assets_without_assigned_topic, num_available_topics);
// |all_available_topics| is ordered from least recent to most recent, so
// choose from the topics beginning at the end of the queue.
std::vector<std::reference_wrapper<const PhotoWithDetails>>
topics_to_choose_from;
auto range_begin = all_available_topics.rbegin();
auto range_end = range_begin + num_topics_to_choose_from;
for (auto topic_iter = range_begin; topic_iter != range_end; ++topic_iter) {
topics_to_choose_from.push_back(std::cref(*topic_iter));
}
return topics_to_choose_from;
}
void AmbientAnimationPhotoProvider::NotifyObserverOfNewTopics() {
base::flat_map<ambient::util::ParsedDynamicAssetId,
std::reference_wrapper<const PhotoWithDetails>>
new_topics;
for (const auto& [asset, topic] : pending_dynamic_asset_topics_) {
new_topics.emplace(asset->parsed_asset_id(), std::cref(topic));
}
for (Observer& obs : observers_) {
obs.OnDynamicImageAssetsRefreshed(new_topics);
}
}
} // namespace ash
Codebase Browser
chromium