diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index de84ea7d7137..121b5800de08 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -272,6 +272,8 @@ import static org.apache.calcite.sql.fun.SqlLibraryOperators.UNIX_MICROS; import static org.apache.calcite.sql.fun.SqlLibraryOperators.UNIX_MILLIS; import static org.apache.calcite.sql.fun.SqlLibraryOperators.UNIX_SECONDS; +import static org.apache.calcite.sql.fun.SqlLibraryOperators.URL_DECODE; +import static org.apache.calcite.sql.fun.SqlLibraryOperators.URL_ENCODE; import static org.apache.calcite.sql.fun.SqlLibraryOperators.XML_TRANSFORM; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.ABS; import static org.apache.calcite.sql.fun.SqlStdOperatorTable.ACOS; @@ -894,6 +896,10 @@ Builder populate2() { // Compression Operators defineMethod(COMPRESS, BuiltInMethod.COMPRESS.method, NullPolicy.ARG0); + // Url Operators + defineMethod(URL_ENCODE, BuiltInMethod.URL_ENCODE.method, NullPolicy.ARG0); + defineMethod(URL_DECODE, BuiltInMethod.URL_DECODE.method, NullPolicy.ARG0); + // Xml Operators defineMethod(EXTRACT_VALUE, BuiltInMethod.EXTRACT_VALUE.method, NullPolicy.ARG0); defineMethod(XML_TRANSFORM, BuiltInMethod.XML_TRANSFORM.method, NullPolicy.ARG0); diff --git a/core/src/main/java/org/apache/calcite/runtime/UrlFunctions.java b/core/src/main/java/org/apache/calcite/runtime/UrlFunctions.java new file mode 100644 index 000000000000..1cfb12666ef2 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/runtime/UrlFunctions.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.runtime; + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import static org.apache.calcite.util.Static.RESOURCE; + +/** + * A collection of functions used in Url processing. + */ +public class UrlFunctions { + + private UrlFunctions() { + } + + private static final Charset UTF_8 = StandardCharsets.UTF_8; + + /** The "URL_DECODE(string)" function for Hive and Spark, + * which returns original value when decoded error. */ + public static String urlDecode(String value) { + try { + return URLDecoder.decode(value, UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw RESOURCE.charsetEncoding(value, UTF_8.name()).ex(); + } catch (RuntimeException e) { + return value; + } + } + + /** The "URL_ENCODE(string)" function for Hive and Spark. */ + public static String urlEncode(String url) { + String value; + try { + value = URLEncoder.encode(url, UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw RESOURCE.charsetEncoding(url, UTF_8.name()).ex(); + } + return value; + } +} diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index f850ab1cc06d..8f4b0277c5ee 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -522,6 +522,22 @@ static RelDataType deriveTypeSplit(SqlOperatorBinding operatorBinding, ReturnTypes.VARBINARY_NULLABLE, OperandTypes.STRING, SqlFunctionCategory.STRING); + /** The "URL_DECODE(string)" function. */ + @LibraryOperator(libraries = {SPARK}) + public static final SqlFunction URL_DECODE = + SqlBasicFunction.create("URL_DECODE", + ReturnTypes.VARCHAR_NULLABLE, + OperandTypes.STRING, + SqlFunctionCategory.STRING); + + /** The "URL_ENCODE(string)" function. */ + @LibraryOperator(libraries = {SPARK}) + public static final SqlFunction URL_ENCODE = + SqlBasicFunction.create("URL_ENCODE", + ReturnTypes.VARCHAR_NULLABLE, + OperandTypes.STRING, + SqlFunctionCategory.STRING); + @LibraryOperator(libraries = {MYSQL}) public static final SqlFunction EXTRACT_VALUE = SqlBasicFunction.create("EXTRACTVALUE", diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index 4652ca6bab87..5106d96ee61f 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -97,6 +97,7 @@ import org.apache.calcite.runtime.SpatialTypeFunctions; import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.runtime.SqlFunctions.FlatProductInputType; +import org.apache.calcite.runtime.UrlFunctions; import org.apache.calcite.runtime.Utilities; import org.apache.calcite.runtime.XmlFunctions; import org.apache.calcite.schema.FilterableTable; @@ -385,6 +386,8 @@ public enum BuiltInMethod { SHA512(SqlFunctions.class, "sha512", String.class), THROW_UNLESS(SqlFunctions.class, "throwUnless", boolean.class, String.class), COMPRESS(CompressionFunctions.class, "compress", String.class), + URL_DECODE(UrlFunctions.class, "urlDecode", String.class), + URL_ENCODE(UrlFunctions.class, "urlEncode", String.class), EXTRACT_VALUE(XmlFunctions.class, "extractValue", String.class, String.class), XML_TRANSFORM(XmlFunctions.class, "xmlTransform", String.class, String.class), EXTRACT_XML(XmlFunctions.class, "extractXml", String.class, String.class, String.class), diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 857445d27f4e..68217e39dbc3 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2854,6 +2854,8 @@ BigQuery's type system uses confusingly different names for types and functions: | b | UNIX_MILLIS(timestamp) | Returns the number of milliseconds since 1970-01-01 00:00:00 | b | UNIX_SECONDS(timestamp) | Returns the number of seconds since 1970-01-01 00:00:00 | b | UNIX_DATE(date) | Returns the number of days since 1970-01-01 +| s | URL_DECODE(string) | Decodes a *string* in 'application/x-www-form-urlencoded' format using a specific encoding scheme, returns original *string* when decoded error +| s | URL_ENCODE(string) | Translates a *string* into 'application/x-www-form-urlencoded' format using a specific encoding scheme | o | XMLTRANSFORM(xml, xslt) | Applies XSLT transform *xslt* to XML string *xml* and returns the result Note: diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index e4b774ac59cd..ba293b39eea6 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -5491,6 +5491,60 @@ private static void checkIf(SqlOperatorFixture f) { "07000000789c4bad48cc2dc84905000bc002ed", "VARBINARY NOT NULL"); } + @Test void testUrlDecode() { + final SqlOperatorFixture f0 = fixture().setFor(SqlLibraryOperators.URL_DECODE); + f0.checkFails("^URL_DECODE('https://calcite.apache.org')^", + "No match found for function signature URL_DECODE\\(\\)", + false); + final SqlOperatorFixture f = f0.withLibrary(SqlLibrary.SPARK); + f.checkString("URL_DECODE('https%3A%2F%2Fcalcite.apache.org')", + "https://calcite.apache.org", + "VARCHAR NOT NULL"); + f.checkString("URL_DECODE('http%3A%2F%2Ftest%3Fa%3Db%26c%3Dd')", + "http://test?a=b&c=d", + "VARCHAR NOT NULL"); + f.checkString("URL_DECODE('http%3A%2F%2F%E4%BD%A0%E5%A5%BD')", + "http://\u4F60\u597D", + "VARCHAR NOT NULL"); + f.checkString("URL_DECODE('test')", + "test", + "VARCHAR NOT NULL"); + f.checkString("URL_DECODE('')", + "", + "VARCHAR NOT NULL"); + f.checkString("URL_DECODE('https%%3A%2F%2Fcalcite.apache.org')", + "https%%3A%2F%2Fcalcite.apache.org", + "VARCHAR NOT NULL"); + f.checkString("URL_DECODE('https%3A%2F%2Fcalcite.apache.org%')", + "https%3A%2F%2Fcalcite.apache.org%", + "VARCHAR NOT NULL"); + f.checkNull("URL_DECODE(cast(null as varchar))"); + } + + @Test void testUrlEncode() { + final SqlOperatorFixture f0 = fixture().setFor(SqlLibraryOperators.URL_ENCODE); + f0.checkFails("^URL_ENCODE('https://calcite.apache.org')^", + "No match found for function signature URL_ENCODE\\(\\)", + false); + final SqlOperatorFixture f = f0.withLibrary(SqlLibrary.SPARK); + f.checkString("URL_ENCODE('https://calcite.apache.org')", + "https%3A%2F%2Fcalcite.apache.org", + "VARCHAR NOT NULL"); + f.checkString("URL_ENCODE('http://test?a=b&c=d')", + "http%3A%2F%2Ftest%3Fa%3Db%26c%3Dd", + "VARCHAR NOT NULL"); + f.checkString("URL_ENCODE(_UTF8'http://\u4F60\u597D')", + "http%3A%2F%2F%E4%BD%A0%E5%A5%BD", + "VARCHAR NOT NULL"); + f.checkString("URL_ENCODE('test')", + "test", + "VARCHAR NOT NULL"); + f.checkString("URL_ENCODE('')", + "", + "VARCHAR NOT NULL"); + f.checkNull("URL_ENCODE(cast(null as varchar))"); + } + @Test void testExtractValue() { SqlOperatorFixture f = fixture().withLibrary(SqlLibrary.MYSQL); f.checkNull("ExtractValue(NULL, '//b')");