#include "ash/ambient/model/ambient_animation_photo_provider.h"
#include <algorithm>
#include <functional>
#include <iterator>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include "ash/ambient/metrics/ambient_metrics.h"
#include "ash/ambient/resources/ambient_animation_resource_constants.h"
#include "ash/ambient/resources/ambient_animation_static_resources.h"
#include "ash/ambient/util/ambient_util.h"
#include "ash/utility/cropping_util.h"
#include "ash/utility/lottie_util.h"
#include "base/check.h"
#include "base/functional/bind.h"
#include "base/functional/callback.h"
#include "base/logging.h"
#include "base/memory/scoped_refptr.h"
#include "base/notreached.h"
#include "base/numerics/ranges.h"
#include "base/rand_util.h"
#include "cc/paint/paint_flags.h"
#include "cc/paint/skottie_frame_data.h"
#include "ui/gfx/geometry/size.h"
#include "ui/gfx/image/image_skia.h"
#include "ui/gfx/image/image_skia_rep.h"
namespace ash {
namespace {
constexpr cc::PaintFlags::FilterQuality kFilterQuality =
cc::PaintFlags::FilterQuality::kMedium;
cc::SkottieFrameData BuildSkottieFrameData(const gfx::ImageSkia& image,
float scale_factor) {
DCHECK(!image.isNull());
const gfx::ImageSkiaRep& image_rep = image.GetRepresentation(scale_factor);
DCHECK(!image_rep.is_null());
DCHECK(image_rep.has_paint_image());
return {
image_rep.paint_image(),
kFilterQuality,
};
}
bool IsPortrait(const gfx::Size& size) {
DCHECK(!size.IsEmpty());
return size.height() > size.width();
}
class DynamicImageProvider {
public:
using TopicReferenceVector =
std::vector<std::reference_wrapper<const PhotoWithDetails>>;
explicit DynamicImageProvider(TopicReferenceVector all_available_topics) {
DCHECK(!all_available_topics.empty())
<< "Animation should not have started rendering without any decoded "
"photos in the model.";
base::RandomShuffle(all_available_topics.begin(),
all_available_topics.end());
for (auto& topic_ref : all_available_topics) {
DCHECK(!topic_ref.get().photo.isNull());
if (IsPortrait(topic_ref.get().photo.size())) {
portrait_set_.topics.push_back(std::move(topic_ref));
} else {
landscape_set_.topics.push_back(std::move(topic_ref));
}
}
}
const PhotoWithDetails& GetTopicForAssetSize(
const std::optional<gfx::Size>& asset_size) {
const PhotoWithDetails* topic = nullptr;
if (!asset_size || IsPortrait(*asset_size)) {
topic = GetNextTopic(portrait_set_,
landscape_set_);
} else {
topic = GetNextTopic(landscape_set_,
portrait_set_);
}
DCHECK(topic);
TryResetCurrentTopicIndices();
return *topic;
}
private:
struct TopicSet {
TopicReferenceVector topics;
size_t current_topic_idx = 0;
};
static const PhotoWithDetails* GetNextTopicFromTopicSet(TopicSet& topic_set) {
if (topic_set.current_topic_idx >= topic_set.topics.size())
return nullptr;
const PhotoWithDetails* topic =
&topic_set.topics[topic_set.current_topic_idx].get();
++topic_set.current_topic_idx;
return topic;
}
static const PhotoWithDetails* GetNextTopic(TopicSet& primary_topic_set,
TopicSet& secondary_topic_set) {
const PhotoWithDetails* topic = GetNextTopicFromTopicSet(primary_topic_set);
return topic ? topic : GetNextTopicFromTopicSet(secondary_topic_set);
}
void TryResetCurrentTopicIndices() {
if (landscape_set_.current_topic_idx >= landscape_set_.topics.size() &&
portrait_set_.current_topic_idx >= portrait_set_.topics.size()) {
landscape_set_.current_topic_idx = 0;
portrait_set_.current_topic_idx = 0;
}
}
TopicSet landscape_set_;
TopicSet portrait_set_;
};
}
class AmbientAnimationPhotoProvider::StaticImageAssetImpl
: public cc::SkottieFrameDataProvider::ImageAsset {
public:
StaticImageAssetImpl(std::string_view asset_id,
const AmbientAnimationStaticResources& static_resources)
: image_(static_resources.GetStaticImageAsset(asset_id)) {
DCHECK(!IsCustomizableLottieId(asset_id));
DCHECK(!image_.isNull())
<< "Static image asset " << asset_id << " is unknown.";
DVLOG(1) << "Loaded static asset " << asset_id;
}
cc::SkottieFrameData GetFrameData(float t, float scale_factor) override {
if (!enabled_)
return cc::SkottieFrameData();
if (!current_frame_data_.image ||
current_frame_data_scale_factor_ != scale_factor) {
current_frame_data_ = BuildSkottieFrameData(image_, scale_factor);
current_frame_data_scale_factor_ = scale_factor;
}
return current_frame_data_;
}
bool enabled() const { return enabled_; }
void set_enabled(bool enabled) { enabled_ = enabled; }
private:
~StaticImageAssetImpl() override = default;
const gfx::ImageSkia image_;
cc::SkottieFrameData current_frame_data_;
float current_frame_data_scale_factor_ = 0;
bool enabled_ = true;
};
class AmbientAnimationPhotoProvider::DynamicImageAssetImpl
: public cc::SkottieFrameDataProvider::ImageAsset {
public:
DynamicImageAssetImpl(
std::string_view asset_id,
std::optional<gfx::Size> size,
const base::WeakPtr<AmbientAnimationPhotoProvider>& provider)
: asset_id_(asset_id), size_(std::move(size)), provider_(provider) {
DCHECK(provider_);
if (!ambient::util::ParseDynamicLottieAssetId(asset_id, parsed_asset_id_)) {
LOG(DFATAL) << "Animation file is invalid. Failed to parse dynamic "
"image asset id "
<< asset_id;
}
if (!size_)
DLOG(ERROR) << "Dimensions unavailable for dynamic asset " << asset_id_;
}
cc::SkottieFrameData GetFrameData(float t, float scale_factor) override {
DVLOG(4) << "GetFrameData for asset " << asset_id_ << " time " << t;
bool is_first_rendered_frame =
last_observed_animation_timestamp_ == kAnimationTimestampInvalid;
bool is_starting_new_cycle = t < last_observed_animation_timestamp_;
last_observed_animation_timestamp_ = t;
if (is_first_rendered_frame || is_starting_new_cycle) {
DVLOG(4) << "Returning new image for dynamic asset " << asset_id_;
if (provider_) {
current_topic_ = provider_->GenerateNextTopicForDynamicAsset(*this);
current_frame_data_scale_factor_ = kImageScaleFactorInvalid;
} else {
DVLOG(1) << "AmbientAnimationPhotoProvider has been destroyed. Cannot "
"refresh images.";
}
} else {
DVLOG(4) << "No update required to dynamic asset at this time";
}
SetCurrentFrameDataForScale(scale_factor);
return current_frame_data_;
}
PhotoWithDetails ExtractAssignedTopic() {
current_frame_data_ = cc::SkottieFrameData();
current_frame_data_scale_factor_ = kImageScaleFactorInvalid;
return std::move(current_topic_);
}
bool HasAssignedTopic() const { return !current_topic_.photo.isNull(); }
const std::optional<gfx::Size>& size() const { return size_; }
const std::string& asset_id() const { return asset_id_; }
const ambient::util::ParsedDynamicAssetId& parsed_asset_id() const {
return parsed_asset_id_;
}
const std::string& position_id() const {
return parsed_asset_id_.position_id;
}
int idx() const { return parsed_asset_id_.idx; }
private:
static constexpr float kAnimationTimestampInvalid = -1.f;
static constexpr float kImageScaleFactorInvalid = 0.f;
~DynamicImageAssetImpl() override = default;
void SetCurrentFrameDataForScale(float scale_factor) {
static constexpr float kScaleFactorEpsilon = 0.01f;
DCHECK(!current_topic_.photo.isNull());
if (current_frame_data_scale_factor_ != kImageScaleFactorInvalid &&
base::IsApproximatelyEqual(current_frame_data_scale_factor_,
scale_factor, kScaleFactorEpsilon)) {
DVLOG(4) << "Current frame data already matches target scale.";
return;
}
const gfx::ImageSkiaRep& image_rep =
current_topic_.photo.GetRepresentation(scale_factor);
DCHECK(!image_rep.is_null());
cc::PaintImage paint_image;
if (size_) {
SkBitmap cropped_bitmap = CenterCropImage(image_rep.GetBitmap(), *size_);
cropped_bitmap.setImmutable();
paint_image = cc::PaintImage::CreateFromBitmap(std::move(cropped_bitmap));
} else {
DLOG(ERROR) << "Dynamic asset " << asset_id_
<< " missing dimensions in lottie file";
DCHECK(image_rep.has_paint_image());
paint_image = image_rep.paint_image();
}
current_frame_data_.image = std::move(paint_image);
current_frame_data_.quality = kFilterQuality;
current_frame_data_scale_factor_ = scale_factor;
}
const std::string asset_id_;
ambient::util::ParsedDynamicAssetId parsed_asset_id_;
const std::optional<gfx::Size> size_;
const base::WeakPtr<AmbientAnimationPhotoProvider> provider_;
float last_observed_animation_timestamp_ = kAnimationTimestampInvalid;
cc::SkottieFrameData current_frame_data_;
float current_frame_data_scale_factor_ = kImageScaleFactorInvalid;
PhotoWithDetails current_topic_;
};
bool AmbientAnimationPhotoProvider::OrderDynamicAssetsByIdx::operator()(
const scoped_refptr<DynamicImageAssetImpl>& asset_l,
const scoped_refptr<DynamicImageAssetImpl>& asset_r) const {
DCHECK(asset_l);
DCHECK(asset_r);
return asset_l->idx() < asset_r->idx();
}
AmbientAnimationPhotoProvider::AmbientAnimationPhotoProvider(
const AmbientAnimationStaticResources* static_resources,
const AmbientBackendModel* backend_model)
: static_resources_(static_resources),
backend_model_(backend_model),
weak_factory_(this) {
DCHECK(static_resources_);
DCHECK(backend_model_);
}
AmbientAnimationPhotoProvider::~AmbientAnimationPhotoProvider() = default;
scoped_refptr<cc::SkottieFrameDataProvider::ImageAsset>
AmbientAnimationPhotoProvider::LoadImageAsset(
std::string_view asset_id,
const base::FilePath& resource_path,
const std::optional<gfx::Size>& size) {
if (IsCustomizableLottieId(asset_id)) {
auto dynamic_asset = base::MakeRefCounted<DynamicImageAssetImpl>(
asset_id, size, weak_factory_.GetWeakPtr());
dynamic_assets_per_position_[dynamic_asset->position_id()].insert(
dynamic_asset);
++total_num_dynamic_assets_;
return dynamic_asset;
} else {
auto static_asset = base::MakeRefCounted<StaticImageAssetImpl>(
asset_id, *static_resources_);
const auto hash_id = cc::HashSkottieResourceId(asset_id);
static_assets_[hash_id] = static_asset;
if (hash_id ==
cc::HashSkottieResourceId(ambient::resources::kTreeShadowAssetId)) {
static_asset->set_enabled(enable_tree_shadow_);
}
return static_asset;
}
}
void AmbientAnimationPhotoProvider::AddObserver(Observer* obs) {
observers_.AddObserver(obs);
}
void AmbientAnimationPhotoProvider::RemoveObserver(Observer* obs) {
observers_.RemoveObserver(obs);
}
bool AmbientAnimationPhotoProvider::ToggleStaticImageAsset(
cc::SkottieResourceIdHash asset_id,
bool enabled) {
auto iter = static_assets_.find(asset_id);
if (iter == static_assets_.end()) {
enable_tree_shadow_ = enabled;
} else {
iter->second->set_enabled(enabled);
}
return true;
}
PhotoWithDetails
AmbientAnimationPhotoProvider::GenerateNextTopicForDynamicAsset(
const DynamicImageAssetImpl& target_asset) {
DVLOG(4) << __func__;
PhotoWithDetails topic_for_target_asset =
ExtractPendingTopicForDynamicAsset(target_asset);
if (!topic_for_target_asset.photo.isNull()) {
return topic_for_target_asset;
}
DCHECK(pending_dynamic_asset_topics_.empty())
<< "All pending topics should have been returned before the first frame "
"of each animation cycle.";
RotateDynamicAssetTopics();
DynamicImageProvider image_provider(GetTopicsToChooseFrom());
for (const auto& [_, dynamic_asset_set] : dynamic_assets_per_position_) {
for (const auto& dynamic_asset : dynamic_asset_set) {
bool asset_already_has_assigned_topic =
pending_dynamic_asset_topics_.contains(dynamic_asset.get());
if (asset_already_has_assigned_topic)
continue;
pending_dynamic_asset_topics_.emplace(
dynamic_asset.get(),
image_provider.GetTopicForAssetSize(dynamic_asset->size()));
}
}
NotifyObserverOfNewTopics();
topic_for_target_asset = ExtractPendingTopicForDynamicAsset(target_asset);
DCHECK(!topic_for_target_asset.photo.isNull())
<< "GenerateNextTopicForDynamicAsset() for unknown asset "
<< target_asset.asset_id();
return topic_for_target_asset;
}
PhotoWithDetails
AmbientAnimationPhotoProvider::ExtractPendingTopicForDynamicAsset(
const DynamicImageAssetImpl& asset) {
auto pending_topic_iter = pending_dynamic_asset_topics_.find(&asset);
if (pending_topic_iter == pending_dynamic_asset_topics_.end()) {
return PhotoWithDetails();
} else {
PhotoWithDetails pending_topic = std::move(pending_topic_iter->second);
pending_dynamic_asset_topics_.erase(pending_topic_iter);
return pending_topic;
}
}
void AmbientAnimationPhotoProvider::RotateDynamicAssetTopics() {
for (const auto& [_, dynamic_asset_set] : dynamic_assets_per_position_) {
DCHECK(!dynamic_asset_set.empty());
auto current_asset = dynamic_asset_set.begin();
auto next_asset = std::next(current_asset);
for (; next_asset != dynamic_asset_set.end();
++current_asset, ++next_asset) {
if ((*next_asset)->HasAssignedTopic()) {
pending_dynamic_asset_topics_[current_asset->get()] =
(*next_asset)->ExtractAssignedTopic();
}
}
}
}
std::vector<std::reference_wrapper<const PhotoWithDetails>>
AmbientAnimationPhotoProvider::GetTopicsToChooseFrom() const {
const base::circular_deque<PhotoWithDetails>& all_available_topics =
backend_model_->all_decoded_topics();
size_t num_assets_without_assigned_topic =
total_num_dynamic_assets_ - pending_dynamic_asset_topics_.size();
size_t num_available_topics = all_available_topics.size();
size_t num_topics_to_choose_from =
std::min(num_assets_without_assigned_topic, num_available_topics);
std::vector<std::reference_wrapper<const PhotoWithDetails>>
topics_to_choose_from;
auto range_begin = all_available_topics.rbegin();
auto range_end = range_begin + num_topics_to_choose_from;
for (auto topic_iter = range_begin; topic_iter != range_end; ++topic_iter) {
topics_to_choose_from.push_back(std::cref(*topic_iter));
}
return topics_to_choose_from;
}
void AmbientAnimationPhotoProvider::NotifyObserverOfNewTopics() {
base::flat_map<ambient::util::ParsedDynamicAssetId,
std::reference_wrapper<const PhotoWithDetails>>
new_topics;
for (const auto& [asset, topic] : pending_dynamic_asset_topics_) {
new_topics.emplace(asset->parsed_asset_id(), std::cref(topic));
}
for (Observer& obs : observers_) {
obs.OnDynamicImageAssetsRefreshed(new_topics);
}
}
}