Skip to content

Commit

Permalink
v1
Browse files Browse the repository at this point in the history
  • Loading branch information
@Zouxxyy
Zouxxyy committed Jan 10, 2025
1 parent 13ed895 commit 02b20c0
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 11 deletions.
32 changes: 25 additions & 7 deletions paimon-common/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,33 @@
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.apache.paimon.utils.Preconditions.checkState;

/**
* Parser for creating instances of {@link org.apache.paimon.types.DataType} from a serialized
* string created with {@link org.apache.paimon.types.DataType#serializeJson}.
*/
public final class DataTypeJsonParser {

public static DataField parseDataField(JsonNode json) {
int id = json.get("id").asInt();
return parseDataField(json, null);
}

private static DataField parseDataField(JsonNode json, AtomicInteger fieldId) {
int id;
JsonNode idNode = json.get("id");
if (idNode != null) {
checkState(fieldId == null || fieldId.get() == -1, "Partial field id is not allowed.");
id = idNode.asInt();
} else {
id = fieldId.incrementAndGet();
}
String name = json.get("name").asText();
DataType type = parseDataType(json.get("type"));
DataType type = parseDataType(json.get("type"), fieldId);
JsonNode descriptionNode = json.get("description");
String description = null;
if (descriptionNode != null) {
Expand All @@ -48,26 +62,30 @@ public static DataField parseDataField(JsonNode json) {
}

public static DataType parseDataType(JsonNode json) {
return parseDataType(json, new AtomicInteger(-1));
}

public static DataType parseDataType(JsonNode json, AtomicInteger fieldId) {
if (json.isTextual()) {
return parseAtomicTypeSQLString(json.asText());
} else if (json.isObject()) {
String typeString = json.get("type").asText();
if (typeString.startsWith("ARRAY")) {
DataType element = parseDataType(json.get("element"));
DataType element = parseDataType(json.get("element"), fieldId);
return new ArrayType(!typeString.contains("NOT NULL"), element);
} else if (typeString.startsWith("MULTISET")) {
DataType element = parseDataType(json.get("element"));
DataType element = parseDataType(json.get("element"), fieldId);
return new MultisetType(!typeString.contains("NOT NULL"), element);
} else if (typeString.startsWith("MAP")) {
DataType key = parseDataType(json.get("key"));
DataType value = parseDataType(json.get("value"));
DataType key = parseDataType(json.get("key"), fieldId);
DataType value = parseDataType(json.get("value"), fieldId);
return new MapType(!typeString.contains("NOT NULL"), key, value);
} else if (typeString.startsWith("ROW")) {
JsonNode fieldArray = json.get("fields");
Iterator<JsonNode> iterator = fieldArray.elements();
List<DataField> fields = new ArrayList<>(fieldArray.size());
while (iterator.hasNext()) {
fields.add(parseDataField(iterator.next()));
fields.add(parseDataField(iterator.next(), fieldId));
}
return new RowType(!typeString.contains("NOT NULL"), fields);
}
Expand Down
6 changes: 5 additions & 1 deletion paimon-common/src/main/java/org/apache/paimon/types/RowType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,11 @@ public static int currentHighestFieldId(List<DataField> fields) {
}

public static Builder builder() {
return builder(true, new AtomicInteger(-1));
return builder(new AtomicInteger(-1));
}

public static Builder builder(AtomicInteger fieldId) {
return builder(true, fieldId);
}

public static Builder builder(boolean isNullable, AtomicInteger fieldId) {
Expand Down
45 changes: 42 additions & 3 deletions paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypeJsonParser;
import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.DateType;
import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.DoubleType;
Expand All @@ -43,13 +44,15 @@
import org.apache.paimon.types.VarCharType;
import org.apache.paimon.utils.JsonSerdeUtil;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;

import javax.annotation.Nullable;

import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Stream;

import static org.assertj.core.api.Assertions.assertThat;
Expand Down Expand Up @@ -199,6 +202,44 @@ void testErrorMessage(TestSpec testSpec) {
}
}

@Test
void testParingRowTypeWithoutFieldId() {
String jsonString1 =
"{\"type\":\"ROW\",\"fields\":[{\"name\":\"field1\",\"type\":\"INT\"},{\"name\":\"field2\",\"type\":\"STRING\"}]}";
RowType rowType1 =
RowType.builder()
.fields(
new DataType[] {DataTypes.INT(), DataTypes.STRING()},
new String[] {"field1", "field2"})
.build();
assertThat(parse(jsonString1)).isEqualTo(rowType1);

String jsonString2 =
"{\"type\":\"ROW\",\"fields\":[{\"name\":\"field1\",\"type\":\"INT\"},{\"name\":\"field2\",\"type\":{\"type\":\"ROW\",\"fields\":[{\"name\":\"s1\",\"type\":\"INT\"},{\"name\":\"s2\",\"type\":\"STRING\"}]}}]}";
RowType rowType2 =
RowType.builder()
.fields(
new DataType[] {
DataTypes.INT(),
RowType.builder(new AtomicInteger(1))
.fields(
new DataType[] {
DataTypes.INT(), DataTypes.STRING()
},
new String[] {"s1", "s2"})
.build()
},
new String[] {"field1", "field2"})
.build();
assertThat(parse(jsonString2)).isEqualTo(rowType2);

String jsonString3 =
"{\"type\":\"ROW\",\"fields\":[{\"name\":\"field1\",\"type\":\"INT\"},{\"id\":1, \"name\":\"field2\",\"type\":\"STRING\"}]}";
assertThatThrownBy(() -> parse(jsonString3))
.isInstanceOf(IllegalStateException.class)
.hasMessageContaining("Partial field id is not allowed.");
}

private static String toJson(DataType type) {
return JsonSerdeUtil.toFlatJson(type);
}
Expand All @@ -207,9 +248,7 @@ private static DataType parse(String json) {
if (!json.startsWith("\"") && !json.startsWith("{")) {
json = "\"" + json + "\"";
}
String dataFieldJson =
String.format("{\"id\": 0, \"name\": \"dummy\", \"type\": %s}", json);
return JsonSerdeUtil.fromJson(dataFieldJson, DataField.class).type();
return JsonSerdeUtil.fromJson(json, DataType.class);
}

// --------------------------------------------------------------------------------------------
Expand Down

0 comments on commit 02b20c0

Please sign in to comment.