From b1865db51209c2d184925e81bcab68b2bb12977c Mon Sep 17 00:00:00 2001 From: Joshua Goins Date: Sat, 19 Aug 2023 14:04:19 -0400 Subject: [PATCH 1/2] Add support for xyz.amorgan.blurhash in ImageContent This adds an optional key to ImageContent, for displaying a blurhash on the client before an image is loaded. --- Quotient/events/eventcontent.cpp | 5 ++++- Quotient/events/eventcontent.h | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Quotient/events/eventcontent.cpp b/Quotient/events/eventcontent.cpp index d5e93725d..3f9c314c8 100644 --- a/Quotient/events/eventcontent.cpp +++ b/Quotient/events/eventcontent.cpp @@ -82,9 +82,10 @@ ImageInfo::ImageInfo(const QFileInfo& fi, QSize imageSize) ImageInfo::ImageInfo(FileSourceInfo sourceInfo, qint64 fileSize, const QMimeType& type, QSize imageSize, - const QString& originalFilename) + const QString& originalFilename, const QString &imageBlurhash) : FileInfo(std::move(sourceInfo), fileSize, type, originalFilename) , imageSize(imageSize) + , blurhash(imageBlurhash) {} ImageInfo::ImageInfo(FileSourceInfo sourceInfo, const QJsonObject& infoJson, @@ -100,6 +101,8 @@ QJsonObject Quotient::EventContent::toInfoJson(const ImageInfo& info) infoJson.insert(QStringLiteral("w"), info.imageSize.width()); if (info.imageSize.height() != -1) infoJson.insert(QStringLiteral("h"), info.imageSize.height()); + if (!info.blurhash.isEmpty()) + infoJson.insert(QStringLiteral("xyz.amorgan.blurhash"), info.blurhash); return infoJson; } diff --git a/Quotient/events/eventcontent.h b/Quotient/events/eventcontent.h index 0273c4cf6..52bf0124d 100644 --- a/Quotient/events/eventcontent.h +++ b/Quotient/events/eventcontent.h @@ -127,11 +127,13 @@ struct QUOTIENT_API ImageInfo : public FileInfo { explicit ImageInfo(const QFileInfo& fi, QSize imageSize = {}); explicit ImageInfo(FileSourceInfo sourceInfo, qint64 fileSize = -1, const QMimeType& type = {}, QSize imageSize = {}, - const QString& originalFilename = {}); + const QString& originalFilename = {}, + const QString& imageBlurhash = {}); ImageInfo(FileSourceInfo sourceInfo, const QJsonObject& infoJson, const QString& originalFilename = {}); QSize imageSize; + QString blurhash; }; QUOTIENT_API QJsonObject toInfoJson(const ImageInfo& info); From 8bc55f0834cd9a807564f2398b2cf9b9489251ff Mon Sep 17 00:00:00 2001 From: Joshua Goins Date: Thu, 19 Oct 2023 15:29:36 -0400 Subject: [PATCH 2/2] Add blurhash implementation to Quotient So every client that wants to implement blurhashes doesn't have to copy this file over and over. --- CMakeLists.txt | 2 + Quotient/blurhash.cpp | 274 +++++++++++++++++++++++++++++++++++++ Quotient/blurhash.h | 107 +++++++++++++++ autotests/CMakeLists.txt | 1 + autotests/testblurhash.cpp | 153 +++++++++++++++++++++ 5 files changed, 537 insertions(+) create mode 100644 Quotient/blurhash.cpp create mode 100644 Quotient/blurhash.h create mode 100644 autotests/testblurhash.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d402ccf04..4458b8479 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,7 @@ target_sources(${QUOTIENT_LIB_NAME} PUBLIC FILE_SET HEADERS BASE_DIRS . Quotient/eventitem.h Quotient/accountregistry.h Quotient/mxcreply.h + Quotient/blurhash.h Quotient/events/event.h Quotient/events/roomevent.h Quotient/events/stateevent.h @@ -212,6 +213,7 @@ target_sources(${QUOTIENT_LIB_NAME} PUBLIC FILE_SET HEADERS BASE_DIRS . Quotient/eventitem.cpp Quotient/accountregistry.cpp Quotient/mxcreply.cpp + Quotient/blurhash.cpp Quotient/events/event.cpp Quotient/events/roomevent.cpp Quotient/events/stateevent.cpp diff --git a/Quotient/blurhash.cpp b/Quotient/blurhash.cpp new file mode 100644 index 000000000..39d38ffec --- /dev/null +++ b/Quotient/blurhash.cpp @@ -0,0 +1,274 @@ +// SPDX-FileCopyrightText: 2024 Joshua Goins +// SPDX-License-Identifier: MIT + +#include "blurhash.h" + +#include + +// From https://github.com/woltapp/blurhash/blob/master/Algorithm.md#base-83 +const static QString b83Characters{QStringLiteral("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~")}; + +const static auto toLinearSRGB = QColorSpace(QColorSpace::SRgb).transformationToColorSpace(QColorSpace::SRgbLinear); +const static auto fromLinearSRGB = QColorSpace(QColorSpace::SRgbLinear).transformationToColorSpace(QColorSpace::SRgb); + +using namespace Quotient; + +QImage BlurHash::decode(const QString &blurhash, const QSize &size) +{ + // 10 is the minimum length of a blurhash string + if (blurhash.length() < 10) + return {}; + + // First character is the number of components + const auto components83 = decode83(blurhash.first(1)); + if (!components83.has_value()) + return {}; + + const auto components = unpackComponents(*components83); + const auto minimumSize = 1 + 1 + 4 + (components.x * components.y - 1) * 2; + if (components.x < 1 || components.y < 1 || blurhash.size() != minimumSize) + return {}; + + // Second character is the maximum AC component value + const auto maxAC83 = decode83(blurhash.mid(1, 1)); + if (!maxAC83.has_value()) + return {}; + + const auto maxAC = decodeMaxAC(*maxAC83); + + // Third character onward is the average color of the image + const auto averageColor83 = decode83(blurhash.mid(2, 4)); + if (!averageColor83.has_value()) + return {}; + + const auto averageColor = toLinearSRGB.map(decodeAverageColor(*averageColor83)); + + QList values = {averageColor}; + + // Iterate through the rest of the string for the color values + // Each AC component is two characters each + for (qsizetype c = 6; c < blurhash.size(); c += 2) { + const auto acComponent83 = decode83(blurhash.mid(c, 2)); + if (!acComponent83.has_value()) + return {}; + + values.append(decodeAC(*acComponent83, maxAC)); + } + + QImage image(size, QImage::Format_RGB888); + image.setColorSpace(QColorSpace::SRgb); + + const auto basisX = calculateWeights(size.width(), components.x); + const auto basisY = calculateWeights(size.height(), components.y); + + for (int y = 0; y < size.height(); y++) { + for (int x = 0; x < size.width(); x++) { + float linearSumR = 0.0f; + float linearSumG = 0.0f; + float linearSumB = 0.0f; + + for (int nx = 0; nx < components.x; nx++) { + for (int ny = 0; ny < components.y; ny++) { + const float basis = basisX[x * components.x + nx] * basisY[y * components.y + ny]; + + linearSumR += values[nx + ny * components.x].redF() * basis; + linearSumG += values[nx + ny * components.x].greenF() * basis; + linearSumB += values[nx + ny * components.x].blueF() * basis; + } + } + + auto linearColor = QColor::fromRgbF(linearSumR, linearSumG, linearSumB); + image.setPixelColor(x, y, fromLinearSRGB.map(linearColor)); + } + } + + return image; +} + +QString BlurHash::encode(const QImage &image, const int componentsX, const int componentsY) +{ + Q_ASSERT(componentsX >= 1 && componentsX <= 9); + Q_ASSERT(componentsY >= 1 && componentsY <= 9); + + if (image.isNull()) + return {}; + + const auto basisX = calculateWeights(image.width(), componentsX); + const auto basisY = calculateWeights(image.height(), componentsY); + + QList factors; + factors.resize(componentsX * componentsY); + + const float normalizationFactor = 1.0f / static_cast(image.width()); + + for (int y = 0; y < image.height(); y++) { + for (int x = 0; x < image.width(); x++) { + const QColor srgbColor = image.pixelColor(x, y); + const QColor linearColor = toLinearSRGB.map(srgbColor); + + float linearR = linearColor.redF(); + float linearG = linearColor.greenF(); + float linearB = linearColor.blueF(); + + linearR *= normalizationFactor; + linearG *= normalizationFactor; + linearB *= normalizationFactor; + + for (int ny = 0; ny < componentsY; ny++) { + for (int nx = 0; nx < componentsX; nx++) { + const float basis = basisX[x * componentsX + nx] * basisY[y * componentsY + ny]; + + float factorR = factors[ny * componentsX + nx].redF(); + float factorG = factors[ny * componentsX + nx].greenF(); + float factorB = factors[ny * componentsX + nx].blueF(); + + factors[ny * componentsX + nx] = QColor::fromRgbF(factorR + linearR * basis, factorG + linearG * basis, factorB + linearB * basis); + } + } + } + } + + // Scale by normalization. Half the scaling is done in the previous loop to prevent going + // too far outside the float range. + for (qsizetype i = 0; i < factors.size(); i++) { + float normalisation = (i == 0) ? 1 : 2; + float scale = normalisation / static_cast(image.height()); + + float factorR = factors[i].redF() * scale; + float factorG = factors[i].greenF() * scale; + float factorB = factors[i].blueF() * scale; + + factors[i] = QColor::fromRgbF(factorR, factorG, factorB); + } + + const auto averageColor = factors.takeFirst(); + + QString encodedString; + encodedString.append(encode83(packComponents(Components(componentsX, componentsY))).rightJustified(1, QLatin1Char('0'))); + + float maximumValue; + if (!factors.empty()) { + float actualMaximumValue = 0; + for (auto ac : factors) { + actualMaximumValue = std::max({ + std::abs(ac.redF()), + std::abs(ac.greenF()), + std::abs(ac.blueF()), + actualMaximumValue, + }); + } + + int quantisedMaximumValue = encodeMaxAC(actualMaximumValue); + maximumValue = (static_cast(quantisedMaximumValue) + 1) / 166; + encodedString.append(encode83(quantisedMaximumValue).leftJustified(1, QLatin1Char('0'))); + } else { + maximumValue = 1; + encodedString.append(encode83(0).leftJustified(1, QLatin1Char('0'))); + } + + encodedString.append(encode83(encodeAverageColor(fromLinearSRGB.map(averageColor))).leftJustified(4, QLatin1Char('0'))); + + for (auto ac : factors) + encodedString.append(encode83(encodeAC(ac, maximumValue)).leftJustified(2, QLatin1Char('0'))); + + return encodedString; +} + +std::optional BlurHash::decode83(const QString &encodedString) +{ + int temp = 0; + for (const QChar c : encodedString) { + const auto index = b83Characters.indexOf(c); + if (index == -1) + return std::nullopt; + + temp = temp * 83 + static_cast(index); + } + + return temp; +} + +QString BlurHash::encode83(int value) +{ + QString buffer; + + do { + buffer += b83Characters[value % 83]; + } while ((value = value / 83)); + + std::ranges::reverse(buffer); + + return buffer; +} + +BlurHash::Components BlurHash::unpackComponents(const int packedComponents) +{ + return {packedComponents % 9 + 1, packedComponents / 9 + 1}; +} + +int BlurHash::packComponents(const Components &components) +{ + return (components.x - 1) + (components.y - 1) * 9; +} + +float BlurHash::decodeMaxAC(const int value) +{ + return static_cast(value + 1) / 166.f; +} + +int BlurHash::encodeMaxAC(const float value) +{ + return std::clamp(static_cast(value * 166 - 0.5f), 0, 82); +} + +QColor BlurHash::decodeAverageColor(const int encodedValue) +{ + const int intR = encodedValue >> 16; + const int intG = (encodedValue >> 8) & 255; + const int intB = encodedValue & 255; + + return QColor::fromRgb(intR, intG, intB); +} + +int BlurHash::encodeAverageColor(const QColor &averageColor) +{ + return (averageColor.red() << 16) + (averageColor.green() << 8) + averageColor.blue(); +} + +float BlurHash::signPow(const float value, const float exp) +{ + return std::copysign(std::pow(std::abs(value), exp), value); +} + +QColor BlurHash::decodeAC(const int value, const float maxAC) +{ + const auto quantR = value / (19 * 19); + const auto quantG = (value / 19) % 19; + const auto quantB = value % 19; + + return QColor::fromRgbF(signPow((static_cast(quantR) - 9) / 9, 2) * maxAC, + signPow((static_cast(quantG) - 9) / 9, 2) * maxAC, + signPow((static_cast(quantB) - 9) / 9, 2) * maxAC); +} + +int BlurHash::encodeAC(const QColor value, const float maxAC) +{ + const auto quantR = static_cast(std::max(0., std::min(18., std::floor(signPow(value.redF() / maxAC, 0.5) * 9 + 9.5)))); + const auto quantG = static_cast(std::max(0., std::min(18., std::floor(signPow(value.greenF() / maxAC, 0.5) * 9 + 9.5)))); + const auto quantB = static_cast(std::max(0., std::min(18., std::floor(signPow(value.blueF() / maxAC, 0.5) * 9 + 9.5)))); + + return quantR * 19 * 19 + quantG * 19 + quantB; +} + +QList BlurHash::calculateWeights(const qsizetype dimension, const qsizetype components) +{ + QList bases(dimension * components, 0.0f); + + const auto scale = static_cast(std::numbers::pi) / static_cast(dimension); + for (qsizetype x = 0; x < dimension; x++) { + for (qsizetype nx = 0; nx < components; nx++) { + bases[x * components + nx] = std::cos(scale * static_cast(nx * x)); + } + } + return bases; +} diff --git a/Quotient/blurhash.h b/Quotient/blurhash.h new file mode 100644 index 000000000..f22543ff1 --- /dev/null +++ b/Quotient/blurhash.h @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: 2024 Joshua Goins +// SPDX-License-Identifier: MIT + +#pragma once + +#include "quotient_export.h" + +#include + +class TestBlurHash; + +namespace Quotient { +/** + * @brief Encodes and decodes image to and from the BlurHash format. See https://blurha.sh/. + * + * @note This class has been adapted from https://github.com/redstrate/QtBlurHash. + */ +class QUOTIENT_API BlurHash +{ +public: + /** Decodes the @p blurhash string creating an image of @p size. + * @note Returns a null image if decoding failed. + */ + static QUOTIENT_API QImage decode(const QString &blurhash, const QSize &size); + + /** Encodes the @p image and returns a blurhash string. + * @param image A non-null image. + * @param componentsX the number of components X-wise. Must be between 1 and 9. + * @param componentsY the number of components Y-wise. Must be between 1 and 9. + * @note Returns an empty string if it failed to encode the image. + */ + static QUOTIENT_API QString encode(const QImage &image, int componentsX = 4, int componentsY = 4); + +protected: + struct Components { + int x, y; + + bool operator==(const Components &other) const + { + return x == other.x && y == other.y; + } + }; + + /** + * @brief Decodes a base 83 string to it's integer value. Returns std::nullopt if there's an invalid character in the blurhash. + */ + static QUOTIENT_API std::optional decode83(const QString &encodedString); + + /** + * @brief Encodes an integer to it's base 83 representation. + */ + static QUOTIENT_API QString encode83(int value); + + /** + * @brief Unpacks an integer to it's @c Components value. + */ + static QUOTIENT_API Components unpackComponents(int packedComponents); + + /** + * @brief Packs @c Components to it's integer representation. + */ + static QUOTIENT_API int packComponents(const Components &components); + + /** + * @brief Decodes a encoded max AC component value. + */ + static QUOTIENT_API float decodeMaxAC(int value); + + /** + * @brief Encodes the maximum AC component value to an integer repsentation. + */ + static QUOTIENT_API int encodeMaxAC(float value); + + /** + * @brief Decodes the average color from the encoded RGB value. + * @note This returns the color as SRGB. + */ + static QUOTIENT_API QColor decodeAverageColor(int encodedValue); + + /** + * @brief Encodes the average color into it's integer representation. + */ + static QUOTIENT_API int encodeAverageColor(const QColor &averageColor); + + /** + * @brief Calls pow() with @p exp on @p value, while keeping the sign. + */ + static QUOTIENT_API float signPow(float value, float exp); + + /** + * @brief Decodes a encoded AC component value. + */ + static QUOTIENT_API QColor decodeAC(int value, float maxAC); + + /** + * @brief Encodes the AC component into it's integer representation. + */ + static QUOTIENT_API int encodeAC(QColor value, float maxAC); + + /** + * @brief Calculates the weighted sum for @p dimension across @p components. + */ + static QUOTIENT_API QList calculateWeights(qsizetype dimension, qsizetype components); + + friend class ::TestBlurHash; +}; +} // namespace Quotient \ No newline at end of file diff --git a/autotests/CMakeLists.txt b/autotests/CMakeLists.txt index 24f6b7623..d3f6759b9 100644 --- a/autotests/CMakeLists.txt +++ b/autotests/CMakeLists.txt @@ -27,3 +27,4 @@ quotient_add_test(NAME testcryptoutils) quotient_add_test(NAME testkeyverification) quotient_add_test(NAME testcrosssigning) quotient_add_test(NAME testkeyimport) +quotient_add_test(NAME testblurhash) diff --git a/autotests/testblurhash.cpp b/autotests/testblurhash.cpp new file mode 100644 index 000000000..fd31f657e --- /dev/null +++ b/autotests/testblurhash.cpp @@ -0,0 +1,153 @@ +// SPDX-FileCopyrightText: 2024 Joshua Goins +// SPDX-License-Identifier: MIT + +#include + +#include + +class TestBlurHash : public QObject +{ + Q_OBJECT + +private Q_SLOTS: + void encode83_data(); + void encode83(); + + void decode83_data(); + void decode83(); + + void unpackComponents(); + void packComponents(); + + void decodeMaxAC(); + void encodeMaxAC(); + + void decodeAverageColor_data(); + void decodeAverageColor(); + + void decodeAC(); + void encodeAC(); + + void decodeImage(); + void encodeImage(); +}; + +void TestBlurHash::encode83_data() +{ + QTest::addColumn("value"); + QTest::addColumn("expected"); + + QTest::addRow("encoding 1") << 0 << "0"; + QTest::addRow("encoding 2") << 21 << "L"; + QTest::addRow("encoding 3") << 30 << "U"; + QTest::addRow("encoding 4") << 34 << "Y"; + QTest::addRow("encoding 5") << 1 << "1"; +} + +void TestBlurHash::encode83() +{ + QFETCH(int, value); + QFETCH(QString, expected); + + QCOMPARE(Quotient::BlurHash::encode83(value), expected); +} + +void TestBlurHash::decode83_data() +{ + QTest::addColumn("value"); + QTest::addColumn>("expected"); + + // invalid base83 characters + QTest::addRow("decoding 1") << "試し" << std::optional(std::nullopt); + QTest::addRow("decoding 2") << "(" << std::optional(std::nullopt); + + QTest::addRow("decoding 3") << "0" << std::optional(0); + QTest::addRow("decoding 4") << "L" << std::optional(21); + QTest::addRow("decoding 5") << "U" << std::optional(30); + QTest::addRow("decoding 6") << "Y" << std::optional(34); + QTest::addRow("decoding 7") << "1" << std::optional(1); +} + +void TestBlurHash::decode83() +{ + QFETCH(QString, value); + QFETCH(std::optional, expected); + + QCOMPARE(Quotient::BlurHash::decode83(value), expected); +} + +void TestBlurHash::unpackComponents() +{ + QCOMPARE(Quotient::BlurHash::unpackComponents(50), Quotient::BlurHash::Components(6, 6)); +} + +void TestBlurHash::packComponents() +{ + QCOMPARE(Quotient::BlurHash::packComponents(Quotient::BlurHash::Components(6, 6)), 50); +} + +void TestBlurHash::decodeMaxAC() +{ + QCOMPARE(Quotient::BlurHash::decodeMaxAC(50), 0.307229f); +} + +void TestBlurHash::encodeMaxAC() +{ + QCOMPARE(Quotient::BlurHash::encodeMaxAC(0.307229f), 50); +} + +void TestBlurHash::decodeAverageColor_data() +{ + QTest::addColumn("value"); + QTest::addColumn("expected"); + + QTest::addRow("decoding 1") << 12688010 << QColor(0xffc19a8a); + QTest::addRow("decoding 2") << 9934485 << QColor(0xff979695); + QTest::addRow("decoding 3") << 8617624 << QColor(0xff837e98); + QTest::addRow("decoding 4") << 14604757 << QColor(0xffded9d5); + QTest::addRow("decoding 5") << 13742755 << QColor(0xffd1b2a3); +} + +void TestBlurHash::decodeAverageColor() +{ + QFETCH(int, value); + QFETCH(QColor, expected); + + QCOMPARE(Quotient::BlurHash::decodeAverageColor(value), expected); +} + +void TestBlurHash::decodeAC() +{ + constexpr auto maxAC = 0.289157f; + QCOMPARE(Quotient::BlurHash::decodeAC(0, maxAC), QColor::fromRgbF(-0.289063f, -0.289063f, -0.289063f)); +} + +void TestBlurHash::encodeAC() +{ + constexpr auto maxAC = 0.289157f; + QCOMPARE(Quotient::BlurHash::encodeAC(QColor::fromRgbF(-0.289063f, -0.289063f, -0.289063f), maxAC), 0); +} + +void TestBlurHash::decodeImage() +{ + const auto image = Quotient::BlurHash::decode(QStringLiteral("eBB4=;054UK$=402%s%|r^O%06#?*7RijMxGpYMzniVNT@rFN3#=Kt"), QSize(50, 50)); + QVERIFY(!image.isNull()); + + QCOMPARE(image.width(), 50); + QCOMPARE(image.height(), 50); + QCOMPARE(image.pixelColor(0, 0), QColor(0xff005f00)); + QCOMPARE(image.pixelColor(30, 30), QColor(0xff99b76d)); +} + +void TestBlurHash::encodeImage() +{ + auto image = QImage(QSize(360, 200), QImage::Format_RGB888); + image.fill(Qt::black); + + const auto encodedString = Quotient::BlurHash::encode(image, 4, 3); + QCOMPARE(encodedString.size(), 28); + QCOMPARE(encodedString, QStringLiteral("L00000fQfQfQfQfQfQfQfQfQfQfQ")); +} + +QTEST_GUILESS_MAIN(TestBlurHash) +#include "testblurhash.moc" \ No newline at end of file