forked from apache/paimon
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[parquet] support read parquet nested columns.
- Loading branch information
wenchao.wu
committed
Jul 1, 2024
1 parent
c1ebb2e
commit f0eef13
Showing
17 changed files
with
2,332 additions
and
62 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
65 changes: 65 additions & 0 deletions
65
paimon-common/src/main/java/org/apache/paimon/utils/BooleanArrayList.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.paimon.utils; | ||
|
|
||
| import java.util.Arrays; | ||
|
|
||
| /** Minimal implementation of an array-backed list of booleans. */ | ||
| public class BooleanArrayList { | ||
| private int size; | ||
|
|
||
| private boolean[] array; | ||
|
|
||
| public BooleanArrayList(int capacity) { | ||
| this.size = 0; | ||
| this.array = new boolean[capacity]; | ||
| } | ||
|
|
||
| public int size() { | ||
| return size; | ||
| } | ||
|
|
||
| public boolean add(boolean element) { | ||
| grow(size + 1); | ||
| array[size++] = element; | ||
| return true; | ||
| } | ||
|
|
||
| public void clear() { | ||
| size = 0; | ||
| } | ||
|
|
||
| public boolean isEmpty() { | ||
| return (size == 0); | ||
| } | ||
|
|
||
| public boolean[] toArray() { | ||
| return Arrays.copyOf(array, size); | ||
| } | ||
|
|
||
| private void grow(int length) { | ||
| if (length > array.length) { | ||
| final int newLength = | ||
| (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); | ||
| final boolean[] t = new boolean[newLength]; | ||
| System.arraycopy(array, 0, t, 0, size); | ||
| array = t; | ||
| } | ||
| } | ||
| } |
89 changes: 89 additions & 0 deletions
89
paimon-common/src/main/java/org/apache/paimon/utils/IntArrayList.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.paimon.utils; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.NoSuchElementException; | ||
|
|
||
| /** Minimal implementation of an array-backed list of ints. */ | ||
| public class IntArrayList { | ||
|
|
||
| private int size; | ||
|
|
||
| private int[] array; | ||
|
|
||
| public IntArrayList(final int capacity) { | ||
| this.size = 0; | ||
| this.array = new int[capacity]; | ||
| } | ||
|
|
||
| public int size() { | ||
| return size; | ||
| } | ||
|
|
||
| public boolean add(final int number) { | ||
| grow(size + 1); | ||
| array[size++] = number; | ||
| return true; | ||
| } | ||
|
|
||
| public int removeLast() { | ||
| if (size == 0) { | ||
| throw new NoSuchElementException(); | ||
| } | ||
| --size; | ||
| return array[size]; | ||
| } | ||
|
|
||
| public void clear() { | ||
| size = 0; | ||
| } | ||
|
|
||
| public boolean isEmpty() { | ||
| return size == 0; | ||
| } | ||
|
|
||
| private void grow(final int length) { | ||
| if (length > array.length) { | ||
| final int newLength = | ||
| (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); | ||
| final int[] t = new int[newLength]; | ||
| System.arraycopy(array, 0, t, 0, size); | ||
| array = t; | ||
| } | ||
| } | ||
|
|
||
| public int[] toArray() { | ||
| return Arrays.copyOf(array, size); | ||
| } | ||
|
|
||
| public static final IntArrayList EMPTY = | ||
| new IntArrayList(0) { | ||
|
|
||
| @Override | ||
| public boolean add(int number) { | ||
| throw new UnsupportedOperationException(); | ||
| } | ||
|
|
||
| @Override | ||
| public int removeLast() { | ||
| throw new UnsupportedOperationException(); | ||
| } | ||
| }; | ||
| } |
79 changes: 79 additions & 0 deletions
79
paimon-common/src/main/java/org/apache/paimon/utils/LongArrayList.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,79 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.paimon.utils; | ||
|
|
||
| import java.util.Arrays; | ||
|
|
||
| /** Minimal implementation of an array-backed list of longs. */ | ||
| public class LongArrayList { | ||
|
|
||
| private int size; | ||
|
|
||
| private long[] array; | ||
|
|
||
| public LongArrayList(int capacity) { | ||
| this.size = 0; | ||
| this.array = new long[capacity]; | ||
| } | ||
|
|
||
| public int size() { | ||
| return size; | ||
| } | ||
|
|
||
| public boolean add(long number) { | ||
| grow(size + 1); | ||
| array[size++] = number; | ||
| return true; | ||
| } | ||
|
|
||
| public long removeLong(int index) { | ||
| if (index >= size) { | ||
| throw new IndexOutOfBoundsException( | ||
| "Index (" + index + ") is greater than or equal to list size (" + size + ")"); | ||
| } | ||
| final long old = array[index]; | ||
| size--; | ||
| if (index != size) { | ||
| System.arraycopy(array, index + 1, array, index, size - index); | ||
| } | ||
| return old; | ||
| } | ||
|
|
||
| public void clear() { | ||
| size = 0; | ||
| } | ||
|
|
||
| public boolean isEmpty() { | ||
| return (size == 0); | ||
| } | ||
|
|
||
| public long[] toArray() { | ||
| return Arrays.copyOf(array, size); | ||
| } | ||
|
|
||
| private void grow(int length) { | ||
| if (length > array.length) { | ||
| final int newLength = | ||
| (int) Math.max(Math.min(2L * array.length, Integer.MAX_VALUE - 8), length); | ||
| final long[] t = new long[newLength]; | ||
| System.arraycopy(array, 0, t, 0, size); | ||
| array = t; | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
54 changes: 54 additions & 0 deletions
54
...on-format/src/main/java/org/apache/paimon/format/parquet/position/CollectionPosition.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| /* | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package org.apache.paimon.format.parquet.position; | ||
|
|
||
| import javax.annotation.Nullable; | ||
|
|
||
| /** To represent collection's position in repeated type. */ | ||
| public class CollectionPosition { | ||
| @Nullable private final boolean[] isNull; | ||
| private final long[] offsets; | ||
|
|
||
| private final long[] length; | ||
|
|
||
| private final int valueCount; | ||
|
|
||
| public CollectionPosition(boolean[] isNull, long[] offsets, long[] length, int valueCount) { | ||
| this.isNull = isNull; | ||
| this.offsets = offsets; | ||
| this.length = length; | ||
| this.valueCount = valueCount; | ||
| } | ||
|
|
||
| public boolean[] getIsNull() { | ||
| return isNull; | ||
| } | ||
|
|
||
| public long[] getOffsets() { | ||
| return offsets; | ||
| } | ||
|
|
||
| public long[] getLength() { | ||
| return length; | ||
| } | ||
|
|
||
| public int getValueCount() { | ||
| return valueCount; | ||
| } | ||
| } |
Oops, something went wrong.