* Copyright (C) 2022 Huawei Device Co., Ltd.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "image_receiver.h"
#include "image_log.h"
#include "image_packer.h"
#include "image_source.h"
#include "image_utils.h"
#include "image_receiver_buffer_processor.h"
#include "image_receiver_manager.h"
#include "sync_fence.h"
#undef LOG_DOMAIN
#define LOG_DOMAIN LOG_TAG_DOMAIN_ID_IMAGE
#undef LOG_TAG
#define LOG_TAG "imageReceiver"
namespace OHOS {
namespace Media {
ImageReceiver::~ImageReceiver()
{
std::lock_guard<std::mutex> guard(imageReceiverMutex_);
if (receiverConsumerSurface_ != nullptr) {
receiverConsumerSurface_->UnregisterConsumerListener();
}
SurfaceUtils* utils = SurfaceUtils::GetInstance();
if (receiverProducerSurface_ != nullptr && utils != nullptr) {
utils->Remove(receiverProducerSurface_->GetUniqueId());
}
receiverConsumerSurface_ = nullptr;
receiverProducerSurface_ = nullptr;
iraContext_ = nullptr;
surfaceBufferAvaliableListener_ = nullptr;
surfaceBufferAvaliableArriveListener_ = nullptr;
bufferProcessor_ = nullptr;
}
enum class Mode {
MODE_PREVIEW = 0,
MODE_PHOTO
};
int64_t PackImage(int &fd, std::unique_ptr<PixelMap> pixelMap)
{
IMAGE_LOGD("PackImage");
ImagePacker imagePacker;
PackOption option;
option.format = ImageReceiver::OPTION_FORMAT;
option.quality = ImageReceiver::OPTION_QUALITY;
option.numberHint = ImageReceiver::OPTION_NUMBERHINT;
std::set<std::string> formats;
bool cond = pixelMap == nullptr;
CHECK_ERROR_RETURN_RET_LOG(cond, 0, "pixelMap is nullptr");
uint32_t ret = imagePacker.GetSupportedFormats(formats);
CHECK_ERROR_RETURN_RET_LOG(ret != SUCCESS, 0,
"image packer get supported format failed, ret=%{public}u.", ret);
IMAGE_LOGD("SUCCESS");
imagePacker.StartPacking(fd, option);
imagePacker.AddImage(*pixelMap);
int64_t packedSize = 0;
imagePacker.FinalizePacking(packedSize);
IMAGE_LOGD("packedSize=%{public}lld.", static_cast<long long>(packedSize));
IMAGE_LOGD("packedSize=%{public}lld.", static_cast<long long>(packedSize));
return packedSize;
}
std::unique_ptr<PixelMap> ImageReceiver::getSurfacePixelMap(InitializationOptions initializationOpts)
{
std::lock_guard<std::mutex> guard(imageReceiverMutex_);
bool cond = iraContext_ == nullptr || iraContext_->currentBuffer_ == nullptr;
CHECK_ERROR_RETURN_RET_LOG(cond, nullptr, "getSurfacePixelMap: iraContext_ or currentBuffer_ is nullptr");
uint32_t *addr = reinterpret_cast<uint32_t *>(iraContext_->currentBuffer_->GetVirAddr());
if (addr == nullptr) {
IMAGE_LOGE("getSurfacePixelMap: GetVirAddr() returned nullptr");
return nullptr;
}
uint32_t size = iraContext_->currentBuffer_->GetSize();
return PixelMap::Create(addr, size, initializationOpts);
}
static int32_t SaveSTP(uint32_t *buffer,
uint32_t bufferSize,
int &fd,
InitializationOptions initializationOpts)
{
int64_t errorCode = -1;
std::unique_ptr<PixelMap> pixelMap = PixelMap::Create(buffer, bufferSize, initializationOpts);
if (pixelMap.get() != nullptr) {
ImageInfo imageInfo;
pixelMap->GetImageInfo(imageInfo);
IMAGE_LOGD("create pixel map imageInfo.size.width=%{public}u.", imageInfo.size.width);
} else {
IMAGE_LOGE("pixelMap.get() == nullptr");
return ERR_MEDIA_INVALID_VALUE;
}
ImagePacker imagePacker;
errorCode = PackImage(fd, std::move(pixelMap));
if (errorCode > 0) {
errorCode = SUCCESS;
} else {
errorCode = ERR_MEDIA_INVALID_VALUE;
}
return errorCode;
}
int32_t ImageReceiver::SaveBufferAsImageInner(int &fd, OHOS::sptr<OHOS::SurfaceBuffer> buffer,
InitializationOptions initializationOpts)
{
int32_t errorcode = 0;
if (buffer != nullptr) {
uint32_t *addr = reinterpret_cast<uint32_t *>(buffer->GetVirAddr());
int32_t size = buffer->GetSize();
errorcode = SaveSTP(addr, static_cast<uint32_t>(size), fd, initializationOpts);
if (iraContext_ != nullptr && iraContext_->GetReceiverBufferConsumer() != nullptr) {
iraContext_->GetReceiverBufferConsumer()->ReleaseBuffer(buffer, -1);
} else {
IMAGE_LOGD("iraContext_ or GetReceiverBufferConsumer() is nullptr");
}
} else {
IMAGE_LOGD("SaveBufferAsImage buffer == nullptr");
}
return errorcode;
}
int32_t ImageReceiver::SaveBufferAsImage(int &fd,
OHOS::sptr<OHOS::SurfaceBuffer> buffer,
InitializationOptions initializationOpts)
{
std::lock_guard<std::mutex> guard(imageReceiverMutex_);
CHECK_ERROR_RETURN_RET_LOG(iraContext_ == nullptr, ERR_MEDIA_INVALID_VALUE,
"SaveBufferAsImage: iraContext_ is nullptr");
return SaveBufferAsImageInner(fd, buffer, initializationOpts);
}
int32_t ImageReceiver::SaveBufferAsImage(int &fd,
InitializationOptions initializationOpts)
{
std::lock_guard<std::mutex> guard(imageReceiverMutex_);
CHECK_ERROR_RETURN_RET_LOG(iraContext_ == nullptr, ERR_MEDIA_INVALID_VALUE,
"SaveBufferAsImage: iraContext_ is nullptr");
if (iraContext_->currentBuffer_ != nullptr) {
return SaveBufferAsImageInner(fd, iraContext_->currentBuffer_, initializationOpts);
}
IMAGE_LOGD("iraContext_->currentBuffer_ is nullptr");
return 0;
}
void ImageReceiver::ReleaseBuffer(OHOS::sptr<OHOS::SurfaceBuffer> &buffer) __attribute__((no_sanitize("cfi")))
{
std::lock_guard<std::mutex> guard(imageReceiverMutex_);
if (buffer != nullptr) {
if (iraContext_ != nullptr) {
auto listenerConsumerSurface = iraContext_->GetReceiverBufferConsumer();
if (listenerConsumerSurface != nullptr) {
listenerConsumerSurface->ReleaseBuffer(buffer, -1);
} else {
IMAGE_LOGD("listenerConsumerSurface == nullptr");
}
if (buffer == iraContext_->currentBuffer_) {
iraContext_->currentBuffer_ = nullptr;
}
} else {
IMAGE_LOGD("iraContext_ == nullptr");
}
buffer = nullptr;
}
}
void ImageReceiverSurfaceListener ::OnBufferAvailable()
{
IMAGE_LOGD("OnBufferAvailable");
auto ir = ir_.lock();
if (ir && ir->surfaceBufferAvaliableListener_ != nullptr) {
ir->surfaceBufferAvaliableListener_->OnSurfaceBufferAvaliable();
}
if (ir && ir->surfaceBufferAvaliableArriveListener_ != nullptr) {
ir->surfaceBufferAvaliableArriveListener_->OnSurfaceBufferAvaliable();
}
}
std::shared_ptr<ImageReceiverContext> ImageReceiverContext ::CreateImageReceiverContext()
{
std::shared_ptr<ImageReceiverContext> irc = std::make_shared<ImageReceiverContext>();
return irc;
}
sptr<Surface> ImageReceiver::getSurfaceById(std::string id)
{
ImageReceiverManager& imageReceiverManager = ImageReceiverManager::getInstance();
sptr<Surface> surface = imageReceiverManager.getSurfaceByKeyId(id);
IMAGE_LOGD("getSurfaceById");
return surface;
}
static std::shared_ptr<ImageReceiver> CreateImageReceiverInner(ImageReceiverOptions &options)
{
std::shared_ptr<ImageReceiver> iva = std::make_shared<ImageReceiver>();
iva->iraContext_ = ImageReceiverContext::CreateImageReceiverContext();
iva->receiverConsumerSurface_ = IConsumerSurface::Create();
if (iva->receiverConsumerSurface_ == nullptr) {
IMAGE_LOGD("SurfaceAsConsumer is nullptr");
return nullptr;
}
iva->receiverConsumerSurface_->SetDefaultWidthAndHeight(options.width, options.height);
iva->receiverConsumerSurface_->SetQueueSize(options.capacity);
iva->receiverConsumerSurface_->SetDefaultUsage(BUFFER_USAGE_CPU_READ);
auto p = iva->receiverConsumerSurface_->GetProducer();
iva->receiverProducerSurface_ = Surface::CreateSurfaceAsProducer(p);
CHECK_DEBUG_RETURN_RET_LOG(iva->receiverProducerSurface_ == nullptr, nullptr,
"SurfaceAsProducer is nullptr");
SurfaceUtils* utils = SurfaceUtils::GetInstance();
if (utils != nullptr) {
utils->Add(iva->receiverProducerSurface_->GetUniqueId(), iva->receiverProducerSurface_);
}
iva->iraContext_->SetReceiverBufferConsumer(iva->receiverConsumerSurface_);
iva->iraContext_->SetReceiverBufferProducer(iva->receiverProducerSurface_);
iva->iraContext_->SetWidth(options.width);
iva->iraContext_->SetHeight(options.height);
iva->iraContext_->SetFormat(options.format);
iva->iraContext_->SetCapicity(options.capacity);
ImageReceiverManager& imageReceiverManager = ImageReceiverManager::getInstance();
std::string receiverKey = imageReceiverManager.SaveImageReceiver(iva);
iva->iraContext_->SetReceiverKey(receiverKey);
sptr<ImageReceiverSurfaceListener> listener = new ImageReceiverSurfaceListener();
listener->ir_ = iva;
iva->receiverConsumerSurface_->RegisterConsumerListener((sptr<IBufferConsumerListener> &)listener);
return iva;
}
std::shared_ptr<ImageReceiver> ImageReceiver::CreateImageReceiver(int32_t width,
int32_t height,
int32_t format,
int32_t capacity)
{
ImageReceiverOptions options{width, height, capacity, format};
return CreateImageReceiverInner(options);
}
std::shared_ptr<ImageReceiver> ImageReceiver::CreateImageReceiver(ImageReceiverOptions &options)
{
return CreateImageReceiverInner(options);
}
static void DumpSurfaceBufferFromImageReceiver(OHOS::sptr<OHOS::SurfaceBuffer> buffer, const std::string suffix = "")
{
CHECK_ERROR_RETURN(!buffer);
std::string fileSuffix = suffix + "-format-" + std::to_string(buffer->GetFormat()) + "-Width-"
+ std::to_string(buffer->GetWidth()) + "-Height-" + std::to_string(buffer->GetHeight()) + "-Stride-"
+ std::to_string(buffer->GetStride());
ImageUtils::DumpDataIfDumpEnabled(reinterpret_cast<const char*>(buffer->GetVirAddr()), buffer->GetSize(),
fileSuffix);
}
OHOS::sptr<OHOS::SurfaceBuffer> ImageReceiver::ReadNextImage(int64_t ×tamp)
{
std::lock_guard<std::mutex> guard(imageReceiverMutex_);
CHECK_ERROR_RETURN_RET_LOG(iraContext_ == nullptr, nullptr, "ReadNextImage: iraContext_ is nullptr");
sptr<SyncFence> flushFence = SyncFence::InvalidFence();
OHOS::Rect damage = {};
OHOS::sptr<OHOS::SurfaceBuffer> buffer;
sptr<IConsumerSurface> listenerConsumerSurface = iraContext_->GetReceiverBufferConsumer();
CHECK_ERROR_RETURN_RET_LOG(listenerConsumerSurface == nullptr, nullptr,
"ReadNextImage: GetReceiverBufferConsumer() is nullptr");
SurfaceError surfaceError = listenerConsumerSurface->AcquireBuffer(buffer, flushFence, timestamp, damage);
if (surfaceError == SURFACE_ERROR_OK) {
uint32_t timeout = 3000;
if (flushFence->IsValid() && flushFence->Wait(timeout) != EOK) {
IMAGE_LOGI("%{public}s buffer is not ready, try to get next buffer", __func__);
listenerConsumerSurface->ReleaseBuffer(buffer, -1);
return nullptr;
}
DumpSurfaceBufferFromImageReceiver(buffer, "imageReceiver");
iraContext_->currentBuffer_ = buffer;
} else {
IMAGE_LOGD("buffer is null");
}
IMAGE_LOGD("[ImageReceiver] ReadNextImage %{public}lld", static_cast<long long>(timestamp));
return iraContext_->GetCurrentBuffer();
}
OHOS::sptr<OHOS::SurfaceBuffer> ImageReceiver::ReadNextImage()
{
int64_t timestamp = 0;
return ReadNextImage(timestamp);
}
OHOS::sptr<OHOS::SurfaceBuffer> ImageReceiver::ReadLastImage(int64_t ×tamp)
{
std::lock_guard<std::mutex> guard(imageReceiverMutex_);
CHECK_ERROR_RETURN_RET_LOG(iraContext_ == nullptr, nullptr, "ReadLastImage: iraContext_ is nullptr");
sptr<SyncFence> flushFence = SyncFence::InvalidFence();
OHOS::Rect damage = {};
OHOS::sptr<OHOS::SurfaceBuffer> buffer;
OHOS::sptr<OHOS::SurfaceBuffer> bufferBefore;
sptr<IConsumerSurface> listenerConsumerSurface = iraContext_->GetReceiverBufferConsumer();
CHECK_ERROR_RETURN_RET_LOG(listenerConsumerSurface == nullptr, nullptr,
"ReadLastImage: GetReceiverBufferConsumer() is nullptr");
SurfaceError surfaceError = listenerConsumerSurface->AcquireBuffer(buffer, flushFence, timestamp, damage);
while (surfaceError == SURFACE_ERROR_OK) {
uint32_t timeout = 3000;
if (flushFence->IsValid() && flushFence->Wait(timeout) != EOK) {
IMAGE_LOGI("%{public}s buffer is not ready, try to get next buffer", __func__);
listenerConsumerSurface->ReleaseBuffer(buffer, -1);
surfaceError = listenerConsumerSurface->AcquireBuffer(buffer, flushFence, timestamp, damage);
continue;
}
bufferBefore = buffer;
surfaceError = listenerConsumerSurface->AcquireBuffer(buffer, flushFence, timestamp, damage);
DumpSurfaceBufferFromImageReceiver(bufferBefore, "imageReceiver") ;
}
iraContext_->currentBuffer_ = bufferBefore;
IMAGE_LOGD("[ImageReceiver] ReadLastImage %{public}lld", static_cast<long long>(timestamp));
return iraContext_->GetCurrentBuffer();
}
OHOS::sptr<OHOS::SurfaceBuffer> ImageReceiver::ReadLastImage()
{
int64_t timestamp = 0;
return ReadLastImage(timestamp);
}
sptr<Surface> ImageReceiver::GetReceiverSurface()
{
std::lock_guard<std::mutex> guard(imageReceiverMutex_);
if (iraContext_ != nullptr) {
return iraContext_->GetReceiverBufferProducer();
}
return nullptr;
}
std::shared_ptr<IBufferProcessor> ImageReceiver::GetBufferProcessor()
{
if (bufferProcessor_ == nullptr) {
bufferProcessor_ = std::make_shared<ImageReceiverBufferProcessor>(this);
}
return bufferProcessor_;
}
std::shared_ptr<NativeImage> ImageReceiver::NextNativeImage()
{
CHECK_ERROR_RETURN_RET(GetBufferProcessor() == nullptr, nullptr);
int64_t timestamp = 0;
auto surfaceBuffer = ReadNextImage(timestamp);
if (surfaceBuffer == nullptr) {
return nullptr;
}
return std::make_shared<NativeImage>(surfaceBuffer, GetBufferProcessor(), timestamp);
}
std::shared_ptr<NativeImage> ImageReceiver::LastNativeImage()
{
if (GetBufferProcessor() == nullptr) {
return nullptr;
}
int64_t timestamp = 0;
auto surfaceBuffer = ReadLastImage(timestamp);
if (surfaceBuffer == nullptr) {
return nullptr;
}
return std::make_shared<NativeImage>(surfaceBuffer, GetBufferProcessor(), timestamp);
}
}
}