diff --git a/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/CanonicalException.kt b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/CanonicalException.kt new file mode 100644 index 000000000..f855f3f5f --- /dev/null +++ b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/CanonicalException.kt @@ -0,0 +1,5 @@ +package it.krzeminski.snakeyaml.engine.kmp.usecases.inherited + +import it.krzeminski.snakeyaml.engine.kmp.exceptions.YamlEngineException + +class CanonicalException(message: String) : YamlEngineException(message) diff --git a/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/CanonicalParser.kt b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/CanonicalParser.kt new file mode 100644 index 000000000..87f84ee34 --- /dev/null +++ b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/CanonicalParser.kt @@ -0,0 +1,191 @@ +package it.krzeminski.snakeyaml.engine.kmp.usecases.inherited + +import it.krzeminski.snakeyaml.engine.kmp.common.Anchor +import it.krzeminski.snakeyaml.engine.kmp.common.FlowStyle +import it.krzeminski.snakeyaml.engine.kmp.common.ScalarStyle +import it.krzeminski.snakeyaml.engine.kmp.common.SpecVersion +import it.krzeminski.snakeyaml.engine.kmp.events.AliasEvent +import it.krzeminski.snakeyaml.engine.kmp.events.DocumentEndEvent +import it.krzeminski.snakeyaml.engine.kmp.events.DocumentStartEvent +import it.krzeminski.snakeyaml.engine.kmp.events.Event +import it.krzeminski.snakeyaml.engine.kmp.events.ImplicitTuple +import it.krzeminski.snakeyaml.engine.kmp.events.MappingEndEvent +import it.krzeminski.snakeyaml.engine.kmp.events.MappingStartEvent +import it.krzeminski.snakeyaml.engine.kmp.events.ScalarEvent +import it.krzeminski.snakeyaml.engine.kmp.events.SequenceEndEvent +import it.krzeminski.snakeyaml.engine.kmp.events.SequenceStartEvent +import it.krzeminski.snakeyaml.engine.kmp.events.StreamEndEvent +import it.krzeminski.snakeyaml.engine.kmp.events.StreamStartEvent +import it.krzeminski.snakeyaml.engine.kmp.nodes.Tag +import it.krzeminski.snakeyaml.engine.kmp.tokens.AliasToken +import it.krzeminski.snakeyaml.engine.kmp.tokens.AnchorToken +import it.krzeminski.snakeyaml.engine.kmp.tokens.ScalarToken +import it.krzeminski.snakeyaml.engine.kmp.tokens.TagToken +import it.krzeminski.snakeyaml.engine.kmp.tokens.Token + +class CanonicalParser(data: String, private val label: String) { + private val scanner = CanonicalScanner(data, label) + private val events = mutableListOf() + private var parsed = false + + // stream: STREAM-START document* STREAM-END + private fun parseStream() { + scanner.getToken(Token.ID.StreamStart) + events.add(StreamStartEvent(null, null)) + while (!scanner.checkToken(Token.ID.StreamEnd)) { + if (scanner.checkToken(Token.ID.Directive, Token.ID.DocumentStart)) { + parseDocument() + } else { + throw CanonicalException("document is expected, got ${scanner.tokens[0]} in $label") + } + } + scanner.getToken(Token.ID.StreamEnd) + events.add(StreamEndEvent(null, null)) + } + + // document: DIRECTIVE? DOCUMENT-START node + private fun parseDocument() { + if (scanner.checkToken(Token.ID.Directive)) { + scanner.getToken(Token.ID.Directive) + } + scanner.getToken(Token.ID.DocumentStart) + events.add(DocumentStartEvent(true, SpecVersion(1, 2), emptyMap(), null, null)) + parseNode(); + if (scanner.checkToken(Token.ID.DocumentEnd)) { + scanner.getToken(Token.ID.DocumentEnd) + } + events.add(DocumentEndEvent(true, null, null)) + } + + // node: ALIAS | ANCHOR? TAG? (SCALAR|sequence|mapping) + private fun parseNode() { + if (scanner.checkToken(Token.ID.Alias)) { + val token = scanner.next() as AliasToken + events.add(AliasEvent(token.value, null, null)) + } else { + var anchor: Anchor? = null + if (scanner.checkToken(Token.ID.Anchor)) { + val token = scanner.next() as AnchorToken + anchor = token.value + } + var tag: String? = null + if (scanner.checkToken(Token.ID.Tag)) { + var token = scanner.next() as TagToken + tag = token.value.handle + token.value.suffix + } + if (scanner.checkToken(Token.ID.Scalar)) { + val token = scanner.next() as ScalarToken + events.add( + ScalarEvent( + anchor, tag, ImplicitTuple(false, false), token.value, + ScalarStyle.PLAIN, null, null + ) + ) + } else if (scanner.checkToken(Token.ID.FlowSequenceStart)) { + events.add( + SequenceStartEvent( + anchor, Tag.SEQ.value, false, + FlowStyle.AUTO, null, null + ) + ) + parseSequence(); + } else if (scanner.checkToken(Token.ID.FlowMappingStart)) { + events.add( + MappingStartEvent( + anchor, Tag.MAP.value, false, + FlowStyle.AUTO, null, null + ) + ) + parseMapping(); + } else { + throw CanonicalException("SCALAR, '[', or '{' is expected, got ${scanner.tokens[0]}") + } + } + } + + // sequence: SEQUENCE-START (node (ENTRY node)*)? ENTRY? SEQUENCE-END + private fun parseSequence() { + scanner.getToken(Token.ID.FlowSequenceStart) + if (!scanner.checkToken(Token.ID.FlowSequenceEnd)) { + parseNode() + while (!scanner.checkToken(Token.ID.FlowSequenceEnd)) { + scanner.getToken(Token.ID.FlowEntry) + if (!scanner.checkToken(Token.ID.FlowSequenceEnd)) { + parseNode() + } + } + } + scanner.getToken(Token.ID.FlowSequenceEnd) + events.add(SequenceEndEvent(null, null)) + } + + // mapping: MAPPING-START (map_entry (ENTRY map_entry)*)? ENTRY? MAPPING-END + private fun parseMapping() { + scanner.getToken(Token.ID.FlowMappingStart) + if (!scanner.checkToken(Token.ID.FlowMappingEnd)) { + parseMapEntry() + while (!scanner.checkToken(Token.ID.FlowMappingEnd)) { + scanner.getToken(Token.ID.FlowEntry) + if (!scanner.checkToken(Token.ID.FlowMappingEnd)) { + parseMapEntry() + } + } + } + scanner.getToken(Token.ID.FlowMappingEnd) + events.add(MappingEndEvent(null, null)) + } + + // map_entry: KEY node VALUE node + private fun parseMapEntry() { + scanner.getToken(Token.ID.Key); + parseNode(); + scanner.getToken(Token.ID.Value); + parseNode(); + } + + fun parse() { + parseStream() + parsed = true + } + + fun next(): Event { + if (!parsed) { + parse() + } + return events.removeAt(0) + } + + /** + * Check the type of the next event. + */ + fun checkEvent(choice: Event.ID): Boolean { + if (!parsed) { + parse() + } + if (!events.isEmpty()) { + return events[0].eventId == choice + } + return false + } + + /** + * Get the next event. + */ + fun peekEvent(): Event { + if (!parsed) { + parse() + } + if (events.isEmpty()) { + throw NoSuchElementException("No more Events found.") + } else { + return events[0] + } + } + + fun hasNext(): Boolean { + if (!parsed) { + parse() + } + return !events.isEmpty() + } +} diff --git a/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/CanonicalScanner.kt b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/CanonicalScanner.kt index 9ff87a16f..d2bb6dc82 100644 --- a/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/CanonicalScanner.kt +++ b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/CanonicalScanner.kt @@ -18,7 +18,6 @@ import it.krzeminski.snakeyaml.engine.kmp.exceptions.Mark import it.krzeminski.snakeyaml.engine.kmp.nodes.Tag import it.krzeminski.snakeyaml.engine.kmp.scanner.Scanner import it.krzeminski.snakeyaml.engine.kmp.tokens.* -import org.snakeyaml.engine.usecases.inherited.CanonicalException class CanonicalScanner(data: String, private val label: String) : Scanner { private val data = data + "\u0000" diff --git a/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedCanonicalTest.kt b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedCanonicalTest.kt new file mode 100644 index 000000000..96f3b2c00 --- /dev/null +++ b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedCanonicalTest.kt @@ -0,0 +1,48 @@ +package it.krzeminski.snakeyaml.engine.kmp.usecases.inherited + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.collections.shouldNotBeEmpty +import io.kotest.matchers.comparables.shouldBeGreaterThan +import it.krzeminski.snakeyaml.engine.kmp.tokens.Token +import java.io.InputStream +import java.nio.file.Files + +class InheritedCanonicalTest: FunSpec({ + test("Canonical scan") { + val files = getStreamsByExtension(".canonical") + files.size shouldBeGreaterThan 0 + files.forEach { file -> + Files.newInputStream(file.toPath()).use { input -> + val tokens = canonicalScan(input, file.name) + tokens.shouldNotBeEmpty() + } + } + } + + test("Canonical parse") { + val files = getStreamsByExtension(".canonical") + files.size shouldBeGreaterThan 0 + files.forEach { file -> + Files.newInputStream(file.toPath()).use { input -> + val tokens = canonicalParse(input, file.name) + tokens.shouldNotBeEmpty() + } + } + } +}) + +private fun canonicalScan(input: InputStream, label: String): List { + val buffer = buildString { + var ch = input.read() + while (ch != -1) { + append(ch.toChar()) + ch = input.read() + } + } + val scanner = CanonicalScanner(buffer.toString().replace(System.lineSeparator(), "\n"), label) + return buildList { + while (scanner.hasNext()) { + add(scanner.next()) + } + } +} diff --git a/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedImportUtils.kt b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedImportUtils.kt new file mode 100644 index 000000000..cdc52e3bc --- /dev/null +++ b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedImportUtils.kt @@ -0,0 +1,82 @@ +package it.krzeminski.snakeyaml.engine.kmp.usecases.inherited + +import it.krzeminski.snakeyaml.engine.kmp.api.LoadSettings +import it.krzeminski.snakeyaml.engine.kmp.api.YamlUnicodeReader +import it.krzeminski.snakeyaml.engine.kmp.events.Event +import it.krzeminski.snakeyaml.engine.kmp.parser.ParserImpl +import it.krzeminski.snakeyaml.engine.kmp.scanner.StreamReader +import okio.source +import org.snakeyaml.engine.v2.utils.TestUtils +import java.io.File +import java.io.FilenameFilter +import java.io.InputStream + +private const val PATH = "/inherited_yaml_1_1" + +fun getResource(theName: String): String = + TestUtils.getResource("$PATH/$theName") + +@JvmOverloads +fun getStreamsByExtension( + extension: String, + onlyIfCanonicalPresent: Boolean = false, +): Array = + File("src/jvmTest/resources$PATH").also { + require(it.exists()) { "Folder not found: ${it.absolutePath}" } + require(it.isDirectory) + }.listFiles(InheritedFilenameFilter(extension, onlyIfCanonicalPresent)) + +fun getFileByName(name: String): File = + File("src/jvmTest/resources$PATH/$name").also { + require(it.exists()) { "Folder not found: ${it.absolutePath}" } + require(it.isFile) + } + +fun canonicalParse(input2: InputStream, label: String): List { + val settings = LoadSettings.builder().setLabel(label).build() + input2.use { + val reader = StreamReader(settings, YamlUnicodeReader(input2.source())) + val buffer = StringBuffer() + while (reader.peek() != 0) { + buffer.appendCodePoint(reader.peek()) + reader.forward() + } + val parser = CanonicalParser(buffer.toString().replace(System.lineSeparator(), "\n"), label) + val result = buildList { + while (parser.hasNext()) { + add(parser.next()) + } + } + return result + } +} + +fun parse(input: InputStream): List { + val settings = LoadSettings.builder().build() + input.use { + val reader = StreamReader(settings, YamlUnicodeReader(input.source())) + val parser = ParserImpl(settings, reader) + val result = buildList { + while (parser.hasNext()) { + add(parser.next()) + } + } + return result + } +} + +private class InheritedFilenameFilter( + private val extension: String, + private val onlyIfCanonicalPresent: Boolean, +) : FilenameFilter { + override fun accept(dir: File?, name: String): Boolean { + val position = name.lastIndexOf('.') + val canonicalFileName = name.substring(0, position) + ".canonical" + val canonicalFile = File(dir, canonicalFileName) + return if (onlyIfCanonicalPresent && !canonicalFile.exists()) { + false + } else { + name.endsWith(extension) + } + } +} diff --git a/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedMarkTest.kt b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedMarkTest.kt new file mode 100644 index 000000000..2dde38656 --- /dev/null +++ b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedMarkTest.kt @@ -0,0 +1,63 @@ +package it.krzeminski.snakeyaml.engine.kmp.usecases.inherited + +import io.kotest.assertions.withClue +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.comparables.shouldBeLessThan +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import it.krzeminski.snakeyaml.engine.kmp.exceptions.Mark +import kotlin.streams.toList + +class InheritedMarkTest: FunSpec({ + test("Marks") { + val content = getResource("test_mark.marks") + val inputs = content.split("---\n") + // Necessary because 'split' returns also what's before the first + // delimiter, as an empty string. + .dropWhile { it.isEmpty() } + inputs.forEach { input -> + println("Input: '$input'") + var index = 0 + var line = 0 + var column = 0 + while (input[index] != '*') { + if (input[index] != '\n') { + line++ + column = 0 + } else { + column++ + } + index++ + } + val inputCodepoints = input.codePoints().toList() + val mark = Mark( + name = "testMarks", + index = index, + line = line, + column = column, + codepoints = inputCodepoints, + pointer = index, + ) + val snippet = mark.createSnippet( + indentSize = 2, maxLength = 79, + ) + withClue("Must contain at least one '\\n'.") { + snippet shouldContain "\n" + } + withClue("Must only have only one '\n'.") { + snippet.indexOf("\n") shouldBe snippet.lastIndexOf("\n") + } + val lines = snippet.split("\n") + val data = lines[0] + val pointer = lines[1] + withClue("Mark must be restricted: $data") { + data.length shouldBeLessThan 82 + } + val dataPosition = data.indexOf("*") + val pointerPosition = pointer.indexOf("^") + withClue("Pointer should coincide with '*':\n $snippet") { + dataPosition shouldBe pointerPosition + } + } + } +}) diff --git a/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedReaderTest.kt b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedReaderTest.kt new file mode 100644 index 000000000..1cf22601b --- /dev/null +++ b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedReaderTest.kt @@ -0,0 +1,39 @@ +package it.krzeminski.snakeyaml.engine.kmp.usecases.inherited + +import io.kotest.assertions.fail +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.string.shouldContain +import it.krzeminski.snakeyaml.engine.kmp.api.LoadSettings +import it.krzeminski.snakeyaml.engine.kmp.api.YamlUnicodeReader +import it.krzeminski.snakeyaml.engine.kmp.exceptions.ReaderException +import it.krzeminski.snakeyaml.engine.kmp.exceptions.YamlEngineException +import it.krzeminski.snakeyaml.engine.kmp.scanner.StreamReader +import okio.source +import java.io.FileInputStream + +class InheritedReaderTest: FunSpec({ + test("Reader errors") { + val inputs = getStreamsByExtension(".stream-error") + inputs + // Skip these files - Okio seems to parse them correctly, so the test fails. + // Supporting UTF-16 will be much more difficult anyway as more code is transferred to KMP, because + // KMP basically only supports UTF-8. + .filter { it.name !in setOf("odd-utf16.stream-error", "invalid-utf8-byte.stream-error") } + .forEach { file -> + FileInputStream(file).use { input -> + val unicodeReader = YamlUnicodeReader(input.source()) + val stream = StreamReader(LoadSettings.builder().build(), unicodeReader) + try { + while (stream.peek() != 0) { + stream.forward() + } + fail("Invalid stream must not be accepted: ${file.absolutePath}; encoding=${unicodeReader.encoding}") + } catch (e: ReaderException) { + e.toString() shouldContain " special characters are not allowed" + } catch (e: YamlEngineException) { + e.toString() shouldContain " MalformedInputException" + } + } + } + } +}) diff --git a/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedTokensTest.kt b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedTokensTest.kt new file mode 100644 index 000000000..c752e6126 --- /dev/null +++ b/src/jvmTest/java/it/krzeminski/snakeyaml/engine/kmp/usecases/inherited/InheritedTokensTest.kt @@ -0,0 +1,112 @@ +package it.krzeminski.snakeyaml.engine.kmp.usecases.inherited + +import io.kotest.assertions.fail +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.comparables.shouldBeGreaterThan +import io.kotest.matchers.shouldBe +import it.krzeminski.snakeyaml.engine.kmp.api.LoadSettings +import it.krzeminski.snakeyaml.engine.kmp.api.YamlUnicodeReader +import it.krzeminski.snakeyaml.engine.kmp.scanner.ScannerImpl +import it.krzeminski.snakeyaml.engine.kmp.scanner.StreamReader +import it.krzeminski.snakeyaml.engine.kmp.tokens.StreamEndToken +import it.krzeminski.snakeyaml.engine.kmp.tokens.StreamStartToken +import it.krzeminski.snakeyaml.engine.kmp.tokens.Token +import okio.source +import java.io.FileInputStream + +class InheritedTokensTest: FunSpec({ + test("Tokens are correct") { + val replaces = mapOf( + Token.ID.Directive to "%", + Token.ID.DocumentStart to "---", + Token.ID.DocumentEnd to "...", + Token.ID.Alias to "*", + Token.ID.Anchor to "&", + Token.ID.Tag to "!", + Token.ID.Scalar to "_", + Token.ID.BlockSequenceStart to "[[", + Token.ID.BlockMappingStart to "{{", + Token.ID.BlockEnd to "]}", + Token.ID.FlowSequenceStart to "[", + Token.ID.FlowSequenceEnd to "]", + Token.ID.FlowMappingStart to "{", + Token.ID.FlowMappingEnd to "}", + Token.ID.BlockEntry to ",", + Token.ID.FlowEntry to ",", + Token.ID.Key to "?", + Token.ID.Value to ":" + ) + val tokensFiles = getStreamsByExtension(".tokens") + tokensFiles.size shouldBeGreaterThan 0 + tokensFiles.forEach { tokenFile -> + val position = tokenFile.name.lastIndexOf('.') + val dataName = tokenFile.name.substring(0, position) + ".data" + + val tokenFileData = getResource(tokenFile.name) + // This is needed because Java's split omits the last empty line - + // looks like Kotlin's implementation is correct. + .trimEnd() + val split = tokenFileData.split(Regex("\\s+")) + val tokens2 = buildList { + addAll(split) + } + + val tokens1 = mutableListOf() + val settings = LoadSettings.builder().build() + FileInputStream(getFileByName(dataName)).source().use { source -> + val reader = StreamReader( + loadSettings = settings, + stream = YamlUnicodeReader(source), + ) + val scanner = ScannerImpl(settings, reader) + try { + while (scanner.checkToken()) { + val token = scanner.next() + if (!(token is StreamStartToken || token is StreamEndToken)) { + val replacement = replaces[token.tokenId]!! + tokens1.add(replacement) + } + } + tokens1 shouldBe tokens2 + } catch (_: RuntimeException) { + println("File name: \n${tokenFile.name}") + val data = getResource(tokenFile.name) + println("Data: \n$data") + println("Tokens:") + tokens1.forEach { + println("token") + } + fail("Cannot scan: $tokenFile") + } + } + } + } + + test("Tokens are correct in data files") { + val files = getStreamsByExtension(".data", onlyIfCanonicalPresent = true) + files.size shouldBeGreaterThan 0 + files.forEach { file -> + val tokens = mutableListOf() + FileInputStream(file).source().use { source -> + val settings = LoadSettings.builder().build() + val reader = StreamReader(settings, YamlUnicodeReader(source)) + val scanner = ScannerImpl(settings, reader) + try { + while (scanner.checkToken()) { + val token = scanner.next() + tokens.add(token.javaClass.name) + } + } catch (e: RuntimeException) { + println("File name: \n${file.name}") + val data = getResource(file.name) + println("Data: \n$data") + println("Tokens:") + tokens.forEach { + println("token") + } + fail("Cannot scan: $file; ${e.message}") + } + } + } + } +}) diff --git a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/CanonicalException.java b/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/CanonicalException.java deleted file mode 100644 index 69833bcde..000000000 --- a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/CanonicalException.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2018, SnakeYAML - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.snakeyaml.engine.usecases.inherited; - -import it.krzeminski.snakeyaml.engine.kmp.exceptions.YamlEngineException; - -public class CanonicalException extends YamlEngineException { - - public CanonicalException(String message) { - super(message); - } -} diff --git a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/CanonicalParser.java b/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/CanonicalParser.java deleted file mode 100644 index e9d715e68..000000000 --- a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/CanonicalParser.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2018, SnakeYAML - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.snakeyaml.engine.usecases.inherited; - -import it.krzeminski.snakeyaml.engine.kmp.usecases.inherited.CanonicalScanner; -import org.jetbrains.annotations.NotNull; -import it.krzeminski.snakeyaml.engine.kmp.common.Anchor; -import it.krzeminski.snakeyaml.engine.kmp.common.FlowStyle; -import it.krzeminski.snakeyaml.engine.kmp.common.ScalarStyle; -import it.krzeminski.snakeyaml.engine.kmp.common.SpecVersion; -import it.krzeminski.snakeyaml.engine.kmp.events.*; -import it.krzeminski.snakeyaml.engine.kmp.nodes.Tag; -import it.krzeminski.snakeyaml.engine.kmp.parser.Parser; -import it.krzeminski.snakeyaml.engine.kmp.tokens.*; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.NoSuchElementException; - -public class CanonicalParser implements Parser { - - private final String label; - private final ArrayList events = new ArrayList<>(); - private final CanonicalScanner scanner; - private boolean parsed = false; - - public CanonicalParser(String data, String label) { - this.label = label; - scanner = new CanonicalScanner(data, label); - } - - // stream: STREAM-START document* STREAM-END - private void parseStream() { - scanner.getToken(Token.ID.StreamStart); - events.add(new StreamStartEvent(null, null)); - while (!scanner.checkToken(Token.ID.StreamEnd)) { - if (scanner.checkToken(Token.ID.Directive, Token.ID.DocumentStart)) { - parseDocument(); - } else { - throw new CanonicalException( - "document is expected, got " + scanner.tokens.get(0) + " in " + label); - } - } - scanner.getToken(Token.ID.StreamEnd); - events.add(new StreamEndEvent(null, null)); - } - - // document: DIRECTIVE? DOCUMENT-START node - private void parseDocument() { - if (scanner.checkToken(Token.ID.Directive)) { - scanner.getToken(Token.ID.Directive); - } - scanner.getToken(Token.ID.DocumentStart); - events.add(new DocumentStartEvent(true, new SpecVersion(1, 2), - Collections.emptyMap(), null, null)); - parseNode(); - if (scanner.checkToken(Token.ID.DocumentEnd)) { - scanner.getToken(Token.ID.DocumentEnd); - } - events.add(new DocumentEndEvent(true, null, null)); - } - - // node: ALIAS | ANCHOR? TAG? (SCALAR|sequence|mapping) - private void parseNode() { - if (scanner.checkToken(Token.ID.Alias)) { - AliasToken token = (AliasToken) scanner.next(); - events.add(new AliasEvent(token.getValue(), null, null)); - } else { - Anchor anchor = null; - if (scanner.checkToken(Token.ID.Anchor)) { - AnchorToken token = (AnchorToken) scanner.next(); - anchor = token.getValue(); - } - String tag = null; - if (scanner.checkToken(Token.ID.Tag)) { - TagToken token = (TagToken) scanner.next(); - tag = token.getValue().getHandle() + token.getValue().getSuffix(); - } - if (scanner.checkToken(Token.ID.Scalar)) { - ScalarToken token = (ScalarToken) scanner.next(); - events.add(new ScalarEvent(anchor, tag, new ImplicitTuple(false, false), token.getValue(), - ScalarStyle.PLAIN, null, null)); - } else if (scanner.checkToken(Token.ID.FlowSequenceStart)) { - events.add(new SequenceStartEvent(anchor, Tag.SEQ.getValue(), false, - FlowStyle.AUTO, null, null)); - parseSequence(); - } else if (scanner.checkToken(Token.ID.FlowMappingStart)) { - events.add(new MappingStartEvent(anchor, Tag.MAP.getValue(), false, - FlowStyle.AUTO, null, null)); - parseMapping(); - } else { - throw new CanonicalException( - "SCALAR, '[', or '{' is expected, got " + scanner.tokens.get(0)); - } - } - } - - // sequence: SEQUENCE-START (node (ENTRY node)*)? ENTRY? SEQUENCE-END - private void parseSequence() { - scanner.getToken(Token.ID.FlowSequenceStart); - if (!scanner.checkToken(Token.ID.FlowSequenceEnd)) { - parseNode(); - while (!scanner.checkToken(Token.ID.FlowSequenceEnd)) { - scanner.getToken(Token.ID.FlowEntry); - if (!scanner.checkToken(Token.ID.FlowSequenceEnd)) { - parseNode(); - } - } - } - scanner.getToken(Token.ID.FlowSequenceEnd); - events.add(new SequenceEndEvent(null, null)); - } - - // mapping: MAPPING-START (map_entry (ENTRY map_entry)*)? ENTRY? MAPPING-END - private void parseMapping() { - scanner.getToken(Token.ID.FlowMappingStart); - if (!scanner.checkToken(Token.ID.FlowMappingEnd)) { - parseMapEntry(); - while (!scanner.checkToken(Token.ID.FlowMappingEnd)) { - scanner.getToken(Token.ID.FlowEntry); - if (!scanner.checkToken(Token.ID.FlowMappingEnd)) { - parseMapEntry(); - } - } - } - scanner.getToken(Token.ID.FlowMappingEnd); - events.add(new MappingEndEvent(null, null)); - } - - // map_entry: KEY node VALUE node - private void parseMapEntry() { - scanner.getToken(Token.ID.Key); - parseNode(); - scanner.getToken(Token.ID.Value); - parseNode(); - } - - public void parse() { - parseStream(); - parsed = true; - } - - @NotNull - public Event next() { - if (!parsed) { - parse(); - } - return events.remove(0); - } - - /** - * Check the type of the next event. - */ - public boolean checkEvent(@NotNull Event.ID choice) { - if (!parsed) { - parse(); - } - if (!events.isEmpty()) { - return events.get(0).getEventId() == choice; - } - return false; - } - - /** - * Get the next event. - */ - @NotNull - public Event peekEvent() { - if (!parsed) { - parse(); - } - if (events.isEmpty()) { - throw new NoSuchElementException("No more Events found."); - } else { - return events.get(0); - } - } - - @Override - public boolean hasNext() { - if (!parsed) { - parse(); - } - return !events.isEmpty(); - } -} diff --git a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedCanonicalTest.java b/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedCanonicalTest.java deleted file mode 100644 index fccaf3cf8..000000000 --- a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedCanonicalTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2018, SnakeYAML - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.snakeyaml.engine.usecases.inherited; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; - -import it.krzeminski.snakeyaml.engine.kmp.usecases.inherited.CanonicalScanner; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; -import it.krzeminski.snakeyaml.engine.kmp.events.Event; -import it.krzeminski.snakeyaml.engine.kmp.tokens.Token; - -@org.junit.jupiter.api.Tag("fast") -public class InheritedCanonicalTest extends InheritedImportTest { - - @Test - @DisplayName("Canonical scan") - public void testCanonicalScanner() throws IOException { - File[] files = getStreamsByExtension(".canonical"); - assertTrue(files.length > 0, "No test files found."); - for (final File file : files) { - try (InputStream input = Files.newInputStream(file.toPath())) { - List tokens = canonicalScan(input, file.getName()); - assertFalse(tokens.isEmpty(), "expect tokens are not empty"); - } - } - } - - private List canonicalScan(InputStream input, String label) throws IOException { - int ch = input.read(); - StringBuilder buffer = new StringBuilder(); - while (ch != -1) { - buffer.append((char) ch); - ch = input.read(); - } - CanonicalScanner scanner = - new CanonicalScanner(buffer.toString().replace(System.lineSeparator(), "\n"), label); - List result = new ArrayList<>(); - while (scanner.hasNext()) { - result.add(scanner.next()); - } - return result; - } - - @Test - @DisplayName("Canonical parse") - public void testCanonicalParser() throws IOException { - final File[] files = getStreamsByExtension(".canonical"); - assertTrue(files.length > 0, "No test files found."); - for (final File file : files) { - try (InputStream input = Files.newInputStream(file.toPath())) { - List tokens = canonicalParse(input, file.getName()); - assertFalse(tokens.isEmpty(), "expect tokens are not empty"); - } - } - } -} diff --git a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedImportTest.java b/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedImportTest.java deleted file mode 100644 index 30f2d8daf..000000000 --- a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedImportTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2018, SnakeYAML - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.snakeyaml.engine.usecases.inherited; - -import okio.Okio; -import it.krzeminski.snakeyaml.engine.kmp.api.LoadSettings; -import it.krzeminski.snakeyaml.engine.kmp.api.YamlUnicodeReader; -import it.krzeminski.snakeyaml.engine.kmp.events.Event; -import it.krzeminski.snakeyaml.engine.kmp.parser.Parser; -import it.krzeminski.snakeyaml.engine.kmp.parser.ParserImpl; -import it.krzeminski.snakeyaml.engine.kmp.scanner.StreamReader; -import org.snakeyaml.engine.v2.utils.TestUtils; - -import java.io.File; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertTrue; - - -public abstract class InheritedImportTest { - - public static final String PATH = "/inherited_yaml_1_1"; - - - protected String getResource(String theName) { - //noinspection InjectedReferences - return TestUtils.getResource(PATH + "/" + theName); - } - - protected File[] getStreamsByExtension(String extension) { - return getStreamsByExtension(extension, false); - } - - protected File[] getStreamsByExtension(String extension, boolean onlyIfCanonicalPresent) { - File file = new File("src/jvmTest/resources" + PATH); - assertTrue(file.exists(), "Folder not found: " + file.getAbsolutePath()); - assertTrue(file.isDirectory()); - return file.listFiles(new InheritedFilenameFilter(extension, onlyIfCanonicalPresent)); - } - - protected File getFileByName(String name) { - File file = new File("src/jvmTest/resources" + PATH + "/" + name); - assertTrue(file.exists(), "Folder not found: " + file.getAbsolutePath()); - assertTrue(file.isFile()); - return file; - } - - protected List canonicalParse(InputStream input2, String label) throws IOException { - LoadSettings setting = LoadSettings.builder().setLabel(label).build(); - StreamReader reader = new StreamReader(setting, new YamlUnicodeReader(Okio.source(input2))); - StringBuilder buffer = new StringBuilder(); - while (reader.peek() != '\0') { - buffer.appendCodePoint(reader.peek()); - reader.forward(); - } - CanonicalParser parser = - new CanonicalParser(buffer.toString().replace(System.lineSeparator(), "\n"), label); - List result = new ArrayList<>(); - while (parser.hasNext()) { - result.add(parser.next()); - } - input2.close(); - return result; - } - - protected List parse(InputStream input) throws IOException { - LoadSettings settings = LoadSettings.builder().build(); - StreamReader reader = new StreamReader(settings, new YamlUnicodeReader(Okio.source(input))); - Parser parser = new ParserImpl(settings, reader); - List result = new ArrayList<>(); - while (parser.hasNext()) { - result.add(parser.next()); - } - input.close(); - return result; - } - - private static class InheritedFilenameFilter implements FilenameFilter { - - private final String extension; - private final boolean onlyIfCanonicalPresent; - - public InheritedFilenameFilter(String extension, boolean onlyIfCanonicalPresent) { - this.extension = extension; - this.onlyIfCanonicalPresent = onlyIfCanonicalPresent; - } - - public boolean accept(File dir, String name) { - int position = name.lastIndexOf('.'); - String canonicalFileName = name.substring(0, position) + ".canonical"; - File canonicalFile = new File(dir, canonicalFileName); - if (onlyIfCanonicalPresent && !canonicalFile.exists()) { - return false; - } else { - return name.endsWith(extension); - } - } - } -} diff --git a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedMarkTest.java b/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedMarkTest.java deleted file mode 100644 index fff259381..000000000 --- a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedMarkTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2018, SnakeYAML - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.snakeyaml.engine.usecases.inherited; - -import it.krzeminski.snakeyaml.engine.kmp.exceptions.Mark; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import java.util.List; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@org.junit.jupiter.api.Tag("fast") -public class InheritedMarkTest extends InheritedImportTest { - - @Test - @DisplayName("Marks") - public void testMarks() { - String content = getResource("test_mark.marks"); - String[] inputs = content.split("---\n"); - for (int i = 1; i < inputs.length; i++) { - String input = inputs[i]; - int index = 0; - int line = 0; - int column = 0; - while (input.charAt(index) != '*') { - if (input.charAt(index) != '\n') { - line += 1; - column = 0; - } else { - column += 1; - } - index += 1; - } - List inputCodepoints = input.codePoints().boxed().collect(Collectors.toList()); - Mark mark = new Mark("testMarks", index, line, column, inputCodepoints, index); - String snippet = mark.createSnippet(2, 79); - assertTrue(snippet.contains("\n"), "Must contain at least one '\n'."); - assertEquals(snippet.indexOf("\n"), snippet.lastIndexOf("\n"), "Must only have only one '\n'."); - String[] lines = snippet.split("\n"); - String data = lines[0]; - String pointer = lines[1]; - assertTrue(data.length() < 82, "Mark must be restricted: " + data); - int dataPosition = data.indexOf("*"); - int pointerPosition = pointer.indexOf("^"); - assertEquals(dataPosition, pointerPosition, "Pointer should coincide with '*':\n " + snippet); - } - } -} diff --git a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedReaderTest.java b/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedReaderTest.java deleted file mode 100644 index 41c75869e..000000000 --- a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedReaderTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2018, SnakeYAML - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.snakeyaml.engine.usecases.inherited; - -import okio.Okio; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; -import it.krzeminski.snakeyaml.engine.kmp.api.LoadSettings; -import it.krzeminski.snakeyaml.engine.kmp.api.YamlUnicodeReader; -import it.krzeminski.snakeyaml.engine.kmp.exceptions.ReaderException; -import it.krzeminski.snakeyaml.engine.kmp.exceptions.YamlEngineException; -import it.krzeminski.snakeyaml.engine.kmp.scanner.StreamReader; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -@org.junit.jupiter.api.Tag("fast") -public class InheritedReaderTest extends InheritedImportTest { - - @Test - @DisplayName("Reader errors") - public void testReaderUnicodeErrors() throws IOException { - File[] inputs = getStreamsByExtension(".stream-error"); - for (File file : inputs) { - if ( - // Skip these files - Okio seems to parse them correctly, so the test fails. - // Supporting UTF-16 will be much more difficult anyway as more code is transferred to KMP, because - // KMP basically only supports UTF-8. - file.getName().equals("odd-utf16.stream-error") - || file.getName().equals("invalid-utf8-byte.stream-error") - ) { - continue; - } - InputStream input = new FileInputStream(file); - YamlUnicodeReader unicodeReader = new YamlUnicodeReader(Okio.source(input)); - StreamReader stream = new StreamReader(LoadSettings.builder().build(), unicodeReader); - try { - while (stream.peek() != '\u0000') { - stream.forward(); - } - fail("Invalid stream must not be accepted: " + file.getAbsolutePath() + "; encoding=" - + unicodeReader.getEncoding()); - } catch (ReaderException e) { - assertTrue(e.toString().contains(" special characters are not allowed"), e.toString()); - } catch (YamlEngineException e) { - assertTrue(e.toString().contains("MalformedInputException"), e.toString()); - } finally { - input.close(); - } - } - } -} diff --git a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedTokensTest.java b/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedTokensTest.java deleted file mode 100644 index f29b24b04..000000000 --- a/src/jvmTest/java/org/snakeyaml/engine/usecases/inherited/InheritedTokensTest.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2018, SnakeYAML - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except - * in compliance with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.snakeyaml.engine.usecases.inherited; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import okio.Okio; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; -import it.krzeminski.snakeyaml.engine.kmp.api.LoadSettings; -import it.krzeminski.snakeyaml.engine.kmp.api.YamlUnicodeReader; -import it.krzeminski.snakeyaml.engine.kmp.scanner.Scanner; -import it.krzeminski.snakeyaml.engine.kmp.scanner.ScannerImpl; -import it.krzeminski.snakeyaml.engine.kmp.scanner.StreamReader; -import it.krzeminski.snakeyaml.engine.kmp.tokens.StreamEndToken; -import it.krzeminski.snakeyaml.engine.kmp.tokens.StreamStartToken; -import it.krzeminski.snakeyaml.engine.kmp.tokens.Token; - -@org.junit.jupiter.api.Tag("fast") -public class InheritedTokensTest extends InheritedImportTest { - - @Test - @DisplayName("Tokens are correct") - public void testTokens() throws FileNotFoundException { - Map replaces = new HashMap(); - replaces.put(Token.ID.Directive, "%"); - replaces.put(Token.ID.DocumentStart, "---"); - replaces.put(Token.ID.DocumentEnd, "..."); - replaces.put(Token.ID.Alias, "*"); - replaces.put(Token.ID.Anchor, "&"); - replaces.put(Token.ID.Tag, "!"); - replaces.put(Token.ID.Scalar, "_"); - replaces.put(Token.ID.BlockSequenceStart, "[["); - replaces.put(Token.ID.BlockMappingStart, "{{"); - replaces.put(Token.ID.BlockEnd, "]}"); - replaces.put(Token.ID.FlowSequenceStart, "["); - replaces.put(Token.ID.FlowSequenceEnd, "]"); - replaces.put(Token.ID.FlowMappingStart, "{"); - replaces.put(Token.ID.FlowMappingEnd, "}"); - replaces.put(Token.ID.BlockEntry, ","); - replaces.put(Token.ID.FlowEntry, ","); - replaces.put(Token.ID.Key, "?"); - replaces.put(Token.ID.Value, ":"); - // - File[] tokensFiles = getStreamsByExtension(".tokens"); - assertTrue(tokensFiles.length > 0, "No test files found."); - for (int i = 0; i < tokensFiles.length; i++) { - String name = tokensFiles[i].getName(); - int position = name.lastIndexOf('.'); - String dataName = name.substring(0, position) + ".data"; - // - String tokenFileData = getResource(name); - String[] split = tokenFileData.split("\\s+"); - List tokens2 = new ArrayList<>(); - Collections.addAll(tokens2, split); - // - List tokens1 = new ArrayList<>(); - LoadSettings settings = LoadSettings.builder().build(); - StreamReader reader = new StreamReader(settings, - new YamlUnicodeReader(Okio.source(new FileInputStream(getFileByName(dataName))))); - Scanner scanner = new ScannerImpl(settings, reader); - try { - while (scanner.checkToken()) { - Token token = scanner.next(); - if (!(token instanceof StreamStartToken || token instanceof StreamEndToken)) { - String replacement = replaces.get(token.getTokenId()); - tokens1.add(replacement); - } - } - assertEquals(tokens1.size(), tokens2.size(), tokenFileData); - assertEquals(tokens1, tokens2); - } catch (RuntimeException e) { - System.out.println("File name: \n" + tokensFiles[i].getName()); - String data = getResource(tokensFiles[i].getName()); - System.out.println("Data: \n" + data); - System.out.println("Tokens:"); - for (String token : tokens1) { - System.out.println(token); - } - fail("Cannot scan: " + tokensFiles[i]); - } - } - } - - @Test - @DisplayName("Tokens are correct in data files") - public void testScanner() throws IOException { - File[] files = getStreamsByExtension(".data", true); - assertTrue(files.length > 0, "No test files found."); - for (File file : files) { - List tokens = new ArrayList<>(); - InputStream input = new FileInputStream(file); - LoadSettings settings = LoadSettings.builder().build(); - StreamReader reader = new StreamReader(settings, new YamlUnicodeReader(Okio.source(input))); - Scanner scanner = new ScannerImpl(settings, reader); - try { - while (scanner.checkToken()) { - Token token = scanner.next(); - tokens.add(token.getClass().getName()); - } - } catch (RuntimeException e) { - System.out.println("File name: \n" + file.getName()); - String data = getResource(file.getName()); - System.out.println("Data: \n" + data); - System.out.println("Tokens:"); - for (String token : tokens) { - System.out.println(token); - } - fail("Cannot scan: " + file + "; " + e.getMessage()); - } finally { - input.close(); - } - } - } -}