Skip to content

Commit

Permalink
[orc] ORC Support ZStandard compression using zstd-jni (#3056)
Browse files Browse the repository at this point in the history
  • Loading branch information
zyl891229 authored Mar 20, 2024
1 parent 47bc30f commit 532cd8f
Show file tree
Hide file tree
Showing 10 changed files with 4,232 additions and 5 deletions.
33 changes: 32 additions & 1 deletion paimon-format/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,13 @@ under the License.

<properties>
<parquet.version>1.13.1</parquet.version>
<orc.version>1.8.3</orc.version>
<orc.version>1.9.2</orc.version>
<joda-time.version>2.5</joda-time.version>
<commons.pool.version>1.6</commons.pool.version>
<commons.lang3.version>3.12.0</commons.lang3.version>
<zstd-jni.version>1.5.5-11</zstd-jni.version>
<storage-api.version>2.8.1</storage-api.version>
<protobuf-java.version>3.17.3</protobuf-java.version>
</properties>

<dependencies>
Expand All @@ -53,6 +56,12 @@ under the License.
<version>${snappy.version}</version>
</dependency>

<dependency>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
<version>${zstd-jni.version}</version>
</dependency>

<!-- Hadoop -->

<dependency>
Expand Down Expand Up @@ -116,6 +125,24 @@ under the License.
<version>${commons.lang3.version}</version>
</dependency>

<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
<version>${storage-api.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>${protobuf-java.version}</version>
</dependency>

<!-- Orc End -->

<!-- Avro Start -->
Expand Down Expand Up @@ -150,6 +177,10 @@ under the License.
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
<exclusion>
<artifactId>zstd-jni</artifactId>
<groupId>com.github.luben</groupId>
</exclusion>
</exclusions>
</dependency>

Expand Down
32 changes: 32 additions & 0 deletions paimon-format/src/main/java/org/apache/orc/CompressionKind.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.orc;

/**
* An enumeration that lists the generic compression algorithms that can be applied to ORC files.
*/
public enum CompressionKind {
NONE,
ZLIB,
SNAPPY,
LZO,
LZ4,
ZSTD,
BROTLI
}
Loading

0 comments on commit 532cd8f

Please sign in to comment.