Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

libheif: support for shared auxl alpha images #1148

Merged
merged 1 commit into from
Jun 11, 2024
Merged

libheif: support for shared auxl alpha images #1148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
209 changes: 99 additions & 110 deletions libheif/context.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -573,8 +573,6 @@ Error HeifContext::interpret_heif_file()


// --- read through properties for each image and extract image resolutions
// Note: this has to be executed before assigning the auxiliary images below because we will only
// merge the alpha image with the main image when their resolutions are the same.

for (auto& pair : m_all_images) {
auto& image = pair.second;
Expand Down Expand Up @@ -656,34 +654,29 @@ Error HeifContext::interpret_heif_file()
// --- this is a thumbnail image, attach to the main image

std::vector<heif_item_id> refs = ref.to_item_ID;
if (refs.size() != 1) {
return Error(heif_error_Invalid_input,
heif_suberror_Unspecified,
"Too many thumbnail references");
}

image->set_is_thumbnail_of(refs[0]);
for (heif_item_id ref: refs) {
image->set_is_thumbnail();

auto master_iter = m_all_images.find(refs[0]);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Thumbnail references a non-existing image");
}
auto master_iter = m_all_images.find(ref);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Thumbnail references a non-existing image");
}

if (master_iter->second->is_thumbnail()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Thumbnail references another thumbnail");
}
if (master_iter->second->is_thumbnail()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Thumbnail references another thumbnail");
}

if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive thumbnail image detected");
if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive thumbnail image detected");
}
master_iter->second->add_thumbnail(image);
}
master_iter->second->add_thumbnail(image);

remove_top_level_image(image);
}
else if (type == fourcc("auxl")) {
Expand Down Expand Up @@ -714,39 +707,30 @@ Error HeifContext::interpret_heif_file()
}

std::vector<heif_item_id> refs = ref.to_item_ID;
if (refs.size() != 1) {
return Error(heif_error_Invalid_input,
heif_suberror_Unspecified,
"Too many auxiliary image references");
}


// alpha channel

if (auxC_property->get_aux_type() == "urn:mpeg:avc:2015:auxid:1" || // HEIF (avc)
auxC_property->get_aux_type() == "urn:mpeg:hevc:2015:auxid:1" || // HEIF (h265)
auxC_property->get_aux_type() == "urn:mpeg:mpegB:cicp:systems:auxiliary:alpha") { // MIAF

auto master_iter = m_all_images.find(refs[0]);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing alpha image referenced");
}

auto master_img = master_iter->second;

if (image.get() == master_img.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive alpha image detected");
}
for (heif_item_id ref: refs) {
auto master_iter = m_all_images.find(ref);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing alpha image referenced");
}

auto master_img = master_iter->second;

if (image->get_width() == master_img->get_width() &&
image->get_height() == master_img->get_height()) {
if (image.get() == master_img.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive alpha image detected");
}

image->set_is_alpha_channel_of(refs[0], true);
image->set_is_alpha_channel();
master_img->set_alpha_channel(image);
}
}
Expand All @@ -756,54 +740,58 @@ Error HeifContext::interpret_heif_file()

if (auxC_property->get_aux_type() == "urn:mpeg:hevc:2015:auxid:2" || // HEIF
auxC_property->get_aux_type() == "urn:mpeg:mpegB:cicp:systems:auxiliary:depth") { // AVIF
image->set_is_depth_channel_of(refs[0]);

auto master_iter = m_all_images.find(refs[0]);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing depth image referenced");
}
if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive depth image detected");
}
master_iter->second->set_depth_channel(image);
image->set_is_depth_channel();

for (heif_item_id ref: refs) {
auto master_iter = m_all_images.find(ref);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing depth image referenced");
}
if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive depth image detected");
}
master_iter->second->set_depth_channel(image);

auto subtypes = auxC_property->get_subtypes();
auto subtypes = auxC_property->get_subtypes();

std::vector<std::shared_ptr<SEIMessage>> sei_messages;
err = decode_hevc_aux_sei_messages(subtypes, sei_messages);
std::vector<std::shared_ptr<SEIMessage>> sei_messages;
err = decode_hevc_aux_sei_messages(subtypes, sei_messages);

for (auto& msg : sei_messages) {
auto depth_msg = std::dynamic_pointer_cast<SEIMessage_depth_representation_info>(msg);
if (depth_msg) {
image->set_depth_representation_info(*depth_msg);
for (auto& msg : sei_messages) {
auto depth_msg = std::dynamic_pointer_cast<SEIMessage_depth_representation_info>(msg);
if (depth_msg) {
image->set_depth_representation_info(*depth_msg);
}
}
}
}


// --- generic aux image

image->set_is_aux_image_of(refs[0], auxC_property->get_aux_type());
image->set_is_aux_image(auxC_property->get_aux_type());

auto master_iter = m_all_images.find(refs[0]);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing aux image referenced");
}
if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive aux image detected");
}
for (heif_item_id ref: refs) {
auto master_iter = m_all_images.find(ref);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing aux image referenced");
}
if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive aux image detected");
}

master_iter->second->add_aux_image(image);
master_iter->second->add_aux_image(image);

remove_top_level_image(image);
remove_top_level_image(image);
}
}
else {
// 'image' is a normal image, keep it as a top-level image
Expand Down Expand Up @@ -915,21 +903,17 @@ Error HeifContext::interpret_heif_file()
for (const auto& ref : references) {
if (ref.header.get_short_type() == fourcc("cdsc")) {
std::vector<uint32_t> refs = ref.to_item_ID;
if (refs.size() != 1) {
return Error(heif_error_Invalid_input,
heif_suberror_Unspecified,
"Metadata not correctly assigned to image");
}

uint32_t exif_image_id = refs[0];
auto img_iter = m_all_images.find(exif_image_id);
if (img_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Metadata assigned to non-existing image");
for(uint32_t ref: refs) {
uint32_t exif_image_id = ref;
auto img_iter = m_all_images.find(exif_image_id);
if (img_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Metadata assigned to non-existing image");
}
img_iter->second->add_metadata(metadata);
}

img_iter->second->add_metadata(metadata);
}
else if (ref.header.get_short_type() == fourcc("prem")) {
uint32_t color_image_id = ref.from_item_ID;
Expand Down Expand Up @@ -964,20 +948,17 @@ Error HeifContext::interpret_heif_file()
for (const auto& ref : references) {
if (ref.header.get_short_type() == fourcc("cdsc")) {
std::vector<uint32_t> refs = ref.to_item_ID;
if (refs.size() != 1) {
return Error(heif_error_Invalid_input,
heif_suberror_Unspecified,
"Region item not correctly assigned to image");
}
uint32_t image_id = refs[0];
auto img_iter = m_all_images.find(image_id);
if (img_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Region item assigned to non-existing image");
for (uint32_t ref: refs) {
uint32_t image_id = ref;
auto img_iter = m_all_images.find(image_id);
if (img_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Region item assigned to non-existing image");
}
img_iter->second->add_region_item_id(id);
m_region_items.push_back(region_item);
}
img_iter->second->add_region_item_id(id);
m_region_items.push_back(region_item);
}

/* When the geometry 'mask' of a region is represented by a mask stored in
Expand Down Expand Up @@ -1599,6 +1580,14 @@ Error HeifContext::decode_image_planar(heif_item_id ID,
heif_suberror_Unsupported_color_conversion);
}

if ((alpha_image->get_width() != img->get_width()) || (alpha_image->get_height() != img->get_height())) {
std::shared_ptr<HeifPixelImage> scaled_alpha;
err = alpha->scale_nearest_neighbor(scaled_alpha, img->get_width(), img->get_height());
if (err) {
return err;
}
alpha = std::move(scaled_alpha);
}
img->transfer_plane_from_image_as(alpha, channel, heif_channel_Alpha);

if (imginfo->is_premultiplied_alpha()) {
Expand Down
21 changes: 5 additions & 16 deletions libheif/context.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,10 +134,9 @@ class HeifContext : public ErrorBuffer

// -- thumbnails

void set_is_thumbnail_of(heif_item_id id)
void set_is_thumbnail()
{
m_is_thumbnail = true;
m_thumbnail_ref_id = id;
}

void add_thumbnail(const std::shared_ptr<Image>& img) { m_thumbnails.push_back(img); }
Expand All @@ -149,11 +148,9 @@ class HeifContext : public ErrorBuffer

// --- alpha channel

void set_is_alpha_channel_of(heif_item_id id, bool consumed)
void set_is_alpha_channel()
{
m_is_alpha_channel = true;
m_alpha_channel_ref_id = id;
m_implicitly_consumed_alpha = consumed;
}

void set_alpha_channel(std::shared_ptr<Image> img) { m_alpha_channel = std::move(img); }
Expand All @@ -169,10 +166,9 @@ class HeifContext : public ErrorBuffer

// --- depth channel

void set_is_depth_channel_of(heif_item_id id)
void set_is_depth_channel()
{
m_is_depth_channel = true;
m_depth_channel_ref_id = id;
}

void set_depth_channel(std::shared_ptr<Image> img) { m_depth_channel = std::move(img); }
Expand Down Expand Up @@ -201,10 +197,9 @@ class HeifContext : public ErrorBuffer

// --- generic aux image

void set_is_aux_image_of(heif_item_id id, const std::string& aux_type)
void set_is_aux_image(const std::string& aux_type)
{
m_is_aux_image = true;
m_aux_image_ref_id = id;
m_aux_image_type = aux_type;
}

Expand All @@ -222,8 +217,7 @@ class HeifContext : public ErrorBuffer
else {
std::vector<std::shared_ptr<Image>> auxImgs;
for (const auto& aux : m_aux_images) {
if ((aux_image_filter & LIBHEIF_AUX_IMAGE_FILTER_OMIT_ALPHA) &&
aux->is_alpha_channel() && aux->m_implicitly_consumed_alpha) {
if ((aux_image_filter & LIBHEIF_AUX_IMAGE_FILTER_OMIT_ALPHA) && aux->is_alpha_channel()) {
continue;
}

Expand Down Expand Up @@ -290,24 +284,19 @@ class HeifContext : public ErrorBuffer
bool m_is_primary = false;

bool m_is_thumbnail = false;
heif_item_id m_thumbnail_ref_id = 0;

std::vector<std::shared_ptr<Image>> m_thumbnails;

bool m_is_alpha_channel = false;
bool m_premultiplied_alpha = false;
bool m_implicitly_consumed_alpha = false; // alpha data was integrated into main color image
heif_item_id m_alpha_channel_ref_id = 0;
std::shared_ptr<Image> m_alpha_channel;

bool m_is_depth_channel = false;
heif_item_id m_depth_channel_ref_id = 0;
std::shared_ptr<Image> m_depth_channel;

bool m_has_depth_representation_info = false;
struct heif_depth_representation_info m_depth_representation_info;

heif_item_id m_aux_image_ref_id = 0;
bool m_is_aux_image = false;
std::string m_aux_image_type;
std::vector<std::shared_ptr<Image>> m_aux_images;
Expand Down