From fa944282d9d0d762c47908b97ad255dec25bb44d Mon Sep 17 00:00:00 2001 From: tiagohm Date: Sun, 31 Mar 2024 22:30:33 -0300 Subject: [PATCH] [api]: Implement XISF write --- .../nebulosa/io/AbstractSeekableSink.kt | 8 +-- .../nebulosa/io/AbstractSeekableSource.kt | 2 +- .../main/kotlin/nebulosa/xisf/XisfFormat.kt | 8 ++- .../nebulosa/xisf/XisfHeaderInputStream.kt | 16 ++++-- .../nebulosa/xisf/XisfMonolithicFileHeader.kt | 2 +- .../src/test/kotlin/XisfFormatTest.kt | 56 +++++++++++++++---- .../test/kotlin/XisfHeaderInputStreamTest.kt | 28 +++++----- .../src/main/kotlin/nebulosa/xml/XmlHelper.kt | 19 +++++++ 8 files changed, 100 insertions(+), 39 deletions(-) diff --git a/nebulosa-io/src/main/kotlin/nebulosa/io/AbstractSeekableSink.kt b/nebulosa-io/src/main/kotlin/nebulosa/io/AbstractSeekableSink.kt index 8b929d13c..aade67cdf 100644 --- a/nebulosa-io/src/main/kotlin/nebulosa/io/AbstractSeekableSink.kt +++ b/nebulosa-io/src/main/kotlin/nebulosa/io/AbstractSeekableSink.kt @@ -37,10 +37,10 @@ abstract class AbstractSeekableSink : SeekableSink { var remaining = byteCount - while (remaining > 0L) { - timeout.throwIfReached() + source.readUnsafe(cursor).use { + while (remaining > 0L) { + timeout.throwIfReached() - source.readUnsafe(cursor).use { it.seek(0L) val length = computeTransferedSize(it, remaining) @@ -62,5 +62,5 @@ abstract class AbstractSeekableSink : SeekableSink { override fun flush() = Unit - override fun close() = cursor.close() + override fun close() = Unit } diff --git a/nebulosa-io/src/main/kotlin/nebulosa/io/AbstractSeekableSource.kt b/nebulosa-io/src/main/kotlin/nebulosa/io/AbstractSeekableSource.kt index 65c980303..2c8338b95 100644 --- a/nebulosa-io/src/main/kotlin/nebulosa/io/AbstractSeekableSource.kt +++ b/nebulosa-io/src/main/kotlin/nebulosa/io/AbstractSeekableSource.kt @@ -55,5 +55,5 @@ abstract class AbstractSeekableSource : SeekableSource { override fun timeout() = timeout - override fun close() = cursor.close() + override fun close() = Unit } diff --git a/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfFormat.kt b/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfFormat.kt index 58146170e..7e704efb8 100644 --- a/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfFormat.kt +++ b/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfFormat.kt @@ -8,6 +8,7 @@ import nebulosa.image.format.ImageHdu import nebulosa.io.* import nebulosa.xisf.XisfMonolithicFileHeader.ColorSpace import nebulosa.xisf.XisfMonolithicFileHeader.ImageType +import nebulosa.xml.escapeXml import okio.Buffer import okio.Sink import okio.Timeout @@ -75,7 +76,7 @@ data object XisfFormat : ImageFormat { if (key != null) { if (key.valueType == ValueType.STRING || key.valueType == ValueType.ANY) { val value = header.getStringOrNull(key) ?: continue - buffer.writeUtf8(STRING_PROPERTY_TAG.format(name, value)) + buffer.writeUtf8(STRING_PROPERTY_TAG.format(name, value.escapeXml())) } else if (key.valueType == ValueType.LOGICAL) { val value = header.getBooleanOrNull(key) ?: continue buffer.writeUtf8(NON_STRING_PROPERTY_TAG.format(name, XisfPropertyType.BOOLEAN.typeName, if (value) 1 else 0)) @@ -90,7 +91,8 @@ data object XisfFormat : ImageFormat { } for (keyword in header) { - buffer.writeUtf8(FITS_KEYWORD_TAG.format(keyword.key, keyword.value, keyword.comment)) + val value = if (keyword.isStringType) "'${keyword.value.escapeXml()}'" else keyword.value + buffer.writeUtf8(FITS_KEYWORD_TAG.format(keyword.key, value, keyword.comment.escapeXml())) } } } @@ -98,7 +100,7 @@ data object XisfFormat : ImageFormat { buffer.writeUtf8(IMAGE_END_TAG) buffer.writeUtf8(XISF_END_TAG) - val headerSize = buffer.size - 8 + val headerSize = buffer.size - 16 buffer.readAndWriteUnsafe().use { it.seek(8L) diff --git a/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfHeaderInputStream.kt b/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfHeaderInputStream.kt index 681e9cdd2..2c703da0d 100644 --- a/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfHeaderInputStream.kt +++ b/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfHeaderInputStream.kt @@ -60,12 +60,18 @@ class XisfHeaderInputStream(source: InputStream) : Closeable { ) } - private fun parseKeywords(): Pair, Image?> { + private fun parseKeywords(): Pair, Image?> { val name = reader.localName - val keywords = ArrayList() + val cards = ArrayList() var thumbnail: Image? = null + fun addHeaderCard(card: HeaderCard) { + if (cards.find { it.key == card.key } == null) { + cards.add(card) + } + } + while (reader.hasNext()) { val type = reader.next() @@ -73,14 +79,14 @@ class XisfHeaderInputStream(source: InputStream) : Closeable { break } else if (type == XMLStreamConstants.START_ELEMENT) { when (reader.localName) { - "FITSKeyword" -> keywords.add(parseFITSKeyword()) + "FITSKeyword" -> addHeaderCard(parseFITSKeyword()) "Thumbnail" -> thumbnail = parseImage() - "Property" -> keywords.add(parseProperty() ?: continue) + "Property" -> addHeaderCard(parseProperty() ?: continue) } } } - return keywords to thumbnail + return cards to thumbnail } private fun parseFITSKeyword(): HeaderCard { diff --git a/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfMonolithicFileHeader.kt b/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfMonolithicFileHeader.kt index 68b4b2def..24c99282f 100644 --- a/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfMonolithicFileHeader.kt +++ b/nebulosa-xisf/src/main/kotlin/nebulosa/xisf/XisfMonolithicFileHeader.kt @@ -98,7 +98,7 @@ sealed interface XisfMonolithicFileHeader { @JvmField val compressionFormat: CompressionFormat? = null, @JvmField val imageType: ImageType = ImageType.LIGHT, @JvmField val bounds: ClosedFloatingPointRange = DEFAULT_BOUNDS, - @JvmField val keywords: List = emptyList(), + @JvmField val keywords: Collection = emptyList(), @JvmField val thumbnail: Image? = null, ) : XisfMonolithicFileHeader diff --git a/nebulosa-xisf/src/test/kotlin/XisfFormatTest.kt b/nebulosa-xisf/src/test/kotlin/XisfFormatTest.kt index 0ca6da519..4f7ebbc11 100644 --- a/nebulosa-xisf/src/test/kotlin/XisfFormatTest.kt +++ b/nebulosa-xisf/src/test/kotlin/XisfFormatTest.kt @@ -1,5 +1,6 @@ import io.kotest.engine.spec.tempfile import io.kotest.matchers.collections.shouldHaveSize +import io.kotest.matchers.floats.shouldBeExactly import io.kotest.matchers.ints.shouldBeExactly import io.kotest.matchers.shouldBe import io.kotest.matchers.types.shouldBeSameInstanceAs @@ -26,7 +27,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 1 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldBeSameInstanceAs data.red data.blue shouldBeSameInstanceAs data.green @@ -45,7 +46,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 1 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldBeSameInstanceAs data.red data.blue shouldBeSameInstanceAs data.green @@ -64,7 +65,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 1 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldBeSameInstanceAs data.red data.blue shouldBeSameInstanceAs data.green @@ -83,7 +84,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 1 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldBeSameInstanceAs data.red data.blue shouldBeSameInstanceAs data.green @@ -102,7 +103,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 1 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldBeSameInstanceAs data.red data.blue shouldBeSameInstanceAs data.green @@ -121,7 +122,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 3 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldNotBeSameInstanceAs data.red data.blue shouldNotBeSameInstanceAs data.green @@ -140,7 +141,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 3 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldNotBeSameInstanceAs data.red data.blue shouldNotBeSameInstanceAs data.green @@ -159,7 +160,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 3 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldNotBeSameInstanceAs data.red data.blue shouldNotBeSameInstanceAs data.green @@ -178,7 +179,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 3 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldNotBeSameInstanceAs data.red data.blue shouldNotBeSameInstanceAs data.green @@ -197,7 +198,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 3 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldNotBeSameInstanceAs data.red data.blue shouldNotBeSameInstanceAs data.green @@ -216,7 +217,7 @@ class XisfFormatTest : FitsStringSpec() { width shouldBeExactly 512 height shouldBeExactly 512 numberOfChannels shouldBeExactly 1 - header shouldHaveSize 29 + header shouldHaveSize 17 data.red.size shouldBeExactly 512 * 512 data.green shouldBeSameInstanceAs data.red data.blue shouldBeSameInstanceAs data.green @@ -238,7 +239,40 @@ class XisfFormatTest : FitsStringSpec() { val hdus1 = XisfFormat.read(source1) hdus1 shouldHaveSize 1 + hdus1[0].data.numberOfChannels shouldBeExactly 1 hdus1[0].header.size shouldBeExactly hdus0[0].header.size + hdus1[0].data.red.size shouldBeExactly hdus0[0].data.red.size + hdus1[0].data.red shouldNotBeSameInstanceAs hdus0[0].data.red + hdus1[0].data.red.forEachIndexed { i, value -> value shouldBeExactly hdus0[0].data.red[i] } + + val image = hdus1[0].makeImage() + image.save("xisf-mono-write").second shouldBe "0dca7efedef5b3525f8037f401518b0b" + } + "color:write" { + val source0 = closeAfterEach(M82_COLOR_32_XISF.seekableSource()) + val hdus0 = XisfFormat.read(source0) + + val outputPath = tempfile() + val sink = closeAfterEach(outputPath.seekableSink()) + + XisfFormat.write(sink, hdus0) + + val source1 = closeAfterEach(outputPath.seekableSource()) + val hdus1 = XisfFormat.read(source1) + + hdus1 shouldHaveSize 1 + hdus1[0].data.numberOfChannels shouldBeExactly 3 + hdus1[0].header.size shouldBeExactly hdus0[0].header.size + hdus1[0].data.red.size shouldBeExactly hdus0[0].data.red.size + hdus1[0].data.red shouldNotBeSameInstanceAs hdus0[0].data.red + hdus1[0].data.red.forEachIndexed { i, value -> value shouldBeExactly hdus0[0].data.red[i] } + hdus1[0].data.green shouldNotBeSameInstanceAs hdus0[0].data.green + hdus1[0].data.green.forEachIndexed { i, value -> value shouldBeExactly hdus0[0].data.green[i] } + hdus1[0].data.blue shouldNotBeSameInstanceAs hdus0[0].data.blue + hdus1[0].data.blue.forEachIndexed { i, value -> value shouldBeExactly hdus0[0].data.blue[i] } + + val image = hdus1[0].makeImage() + image.save("xisf-color-write").second shouldBe "89beed384ee9e97ce033ba447a377937" } } diff --git a/nebulosa-xisf/src/test/kotlin/XisfHeaderInputStreamTest.kt b/nebulosa-xisf/src/test/kotlin/XisfHeaderInputStreamTest.kt index 201390956..83f9e6764 100644 --- a/nebulosa-xisf/src/test/kotlin/XisfHeaderInputStreamTest.kt +++ b/nebulosa-xisf/src/test/kotlin/XisfHeaderInputStreamTest.kt @@ -24,7 +24,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { sampleFormat shouldBe XisfMonolithicFileHeader.SampleFormat.UINT8 colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.GRAY byteOrder shouldBe ByteOrder.LITTLE - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:8:gray:zlib" { @@ -40,7 +40,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.GRAY byteOrder shouldBe ByteOrder.LITTLE compressionFormat.shouldNotBeNull().type shouldBe XisfMonolithicFileHeader.CompressionType.ZLIB - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:8:gray:lz4" { @@ -56,7 +56,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.GRAY byteOrder shouldBe ByteOrder.LITTLE compressionFormat.shouldNotBeNull().type shouldBe XisfMonolithicFileHeader.CompressionType.LZ4 - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:8:gray:lz4-hc" { @@ -72,7 +72,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.GRAY byteOrder shouldBe ByteOrder.LITTLE compressionFormat.shouldNotBeNull().type shouldBe XisfMonolithicFileHeader.CompressionType.LZ4_HC - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:8:gray:zstd" { @@ -88,7 +88,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.GRAY byteOrder shouldBe ByteOrder.LITTLE compressionFormat.shouldNotBeNull().type shouldBe XisfMonolithicFileHeader.CompressionType.ZSTD - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:16:gray" { @@ -103,7 +103,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { sampleFormat shouldBe XisfMonolithicFileHeader.SampleFormat.UINT16 colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.GRAY byteOrder shouldBe ByteOrder.LITTLE - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:32:gray" { @@ -118,7 +118,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { sampleFormat shouldBe XisfMonolithicFileHeader.SampleFormat.UINT32 colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.GRAY byteOrder shouldBe ByteOrder.LITTLE - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:F32:gray" { @@ -133,7 +133,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { sampleFormat shouldBe XisfMonolithicFileHeader.SampleFormat.FLOAT32 colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.GRAY byteOrder shouldBe ByteOrder.LITTLE - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:F64:gray" { @@ -148,7 +148,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { sampleFormat shouldBe XisfMonolithicFileHeader.SampleFormat.FLOAT64 colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.GRAY byteOrder shouldBe ByteOrder.LITTLE - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:8:rgb" { @@ -163,7 +163,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { sampleFormat shouldBe XisfMonolithicFileHeader.SampleFormat.UINT8 colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.RGB byteOrder shouldBe ByteOrder.LITTLE - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:16:rgb" { @@ -178,7 +178,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { sampleFormat shouldBe XisfMonolithicFileHeader.SampleFormat.UINT16 colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.RGB byteOrder shouldBe ByteOrder.LITTLE - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:32:rgb" { @@ -193,7 +193,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { sampleFormat shouldBe XisfMonolithicFileHeader.SampleFormat.UINT32 colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.RGB byteOrder shouldBe ByteOrder.LITTLE - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:F32:rgb" { @@ -208,7 +208,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { sampleFormat shouldBe XisfMonolithicFileHeader.SampleFormat.FLOAT32 colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.RGB byteOrder shouldBe ByteOrder.LITTLE - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } "read:F64:rgb" { @@ -223,7 +223,7 @@ class XisfHeaderInputStreamTest : FitsStringSpec() { sampleFormat shouldBe XisfMonolithicFileHeader.SampleFormat.FLOAT64 colorSpace shouldBe XisfMonolithicFileHeader.ColorSpace.RGB byteOrder shouldBe ByteOrder.LITTLE - keywords shouldHaveSize 29 + keywords shouldHaveSize 17 } } } diff --git a/nebulosa-xml/src/main/kotlin/nebulosa/xml/XmlHelper.kt b/nebulosa-xml/src/main/kotlin/nebulosa/xml/XmlHelper.kt index a3e9eb174..db8c4e2ef 100644 --- a/nebulosa-xml/src/main/kotlin/nebulosa/xml/XmlHelper.kt +++ b/nebulosa-xml/src/main/kotlin/nebulosa/xml/XmlHelper.kt @@ -11,3 +11,22 @@ fun XMLStreamReader.attribute(name: String): String? { return null } + +private val XML_ESCAPE_CHARS = charArrayOf('"', '&', '<', '>') +private val XML_ESCAPE_CHAR_CODES = arrayOf(""", "&", "<", ">") + +fun String.escapeXml(): String { + if (none { it in XML_ESCAPE_CHARS }) return this + + return buildString(length) { + for (c in this) { + val index = XML_ESCAPE_CHARS.indexOf(c) + + if (index >= 0) { + append(XML_ESCAPE_CHAR_CODES[index]) + } else { + append(c) + } + } + } +}