From 787aa9eeaf7bbcb20f016de1ea1e9df09aaa5dcf Mon Sep 17 00:00:00 2001 From: XuQianJin-Stars Date: Sat, 28 Dec 2019 22:39:11 +0800 Subject: [PATCH] [CALCITE-2884] Implement JSON_INSERT, JSON_REPLACE, JSON_SET functions Close apache/calcite#3020 --- .../adapter/enumerable/RexImpTable.java | 6 + .../calcite/runtime/CalciteResource.java | 21 ++- .../apache/calcite/runtime/JsonFunctions.java | 136 ++++++++++++++++++ .../sql/fun/SqlJsonModifyFunction.java | 97 +++++++++++++ .../calcite/sql/fun/SqlLibraryOperators.java | 9 ++ .../calcite/sql/fun/SqlStdOperatorTable.java | 9 ++ .../apache/calcite/util/BuiltInMethod.java | 3 + .../runtime/CalciteResource.properties | 15 +- .../rel/rel2sql/RelToSqlConverterTest.java | 36 +++++ .../org/apache/calcite/test/JdbcTest.java | 30 ++++ .../apache/calcite/test/SqlFunctionsTest.java | 4 +- .../calcite/test/SqlJsonFunctionsTest.java | 57 ++++++++ .../apache/calcite/test/SqlValidatorTest.java | 24 ++++ site/_docs/reference.md | 61 +++++++- .../apache/calcite/test/SqlOperatorTest.java | 54 +++++++ 15 files changed, 542 insertions(+), 20 deletions(-) create mode 100644 core/src/main/java/org/apache/calcite/sql/fun/SqlJsonModifyFunction.java diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java index 4f7e62d65d9f..0ecb370b356f 100644 --- a/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java +++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/RexImpTable.java @@ -134,10 +134,13 @@ import static org.apache.calcite.sql.fun.SqlLibraryOperators.FROM_BASE64; import static org.apache.calcite.sql.fun.SqlLibraryOperators.ILIKE; import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_DEPTH; +import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_INSERT; import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_KEYS; import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_LENGTH; import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_PRETTY; import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_REMOVE; +import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_REPLACE; +import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_SET; import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_STORAGE_SIZE; import static org.apache.calcite.sql.fun.SqlLibraryOperators.JSON_TYPE; import static org.apache.calcite.sql.fun.SqlLibraryOperators.LEFT; @@ -644,11 +647,14 @@ Builder populate2() { defineMethod(JSON_QUERY, BuiltInMethod.JSON_QUERY.method, NullPolicy.ARG0); defineMethod(JSON_TYPE, BuiltInMethod.JSON_TYPE.method, NullPolicy.ARG0); defineMethod(JSON_DEPTH, BuiltInMethod.JSON_DEPTH.method, NullPolicy.ARG0); + defineMethod(JSON_INSERT, BuiltInMethod.JSON_INSERT.method, NullPolicy.ARG0); defineMethod(JSON_KEYS, BuiltInMethod.JSON_KEYS.method, NullPolicy.ARG0); defineMethod(JSON_PRETTY, BuiltInMethod.JSON_PRETTY.method, NullPolicy.ARG0); defineMethod(JSON_LENGTH, BuiltInMethod.JSON_LENGTH.method, NullPolicy.ARG0); defineMethod(JSON_REMOVE, BuiltInMethod.JSON_REMOVE.method, NullPolicy.ARG0); defineMethod(JSON_STORAGE_SIZE, BuiltInMethod.JSON_STORAGE_SIZE.method, NullPolicy.ARG0); + defineMethod(JSON_REPLACE, BuiltInMethod.JSON_REPLACE.method, NullPolicy.ARG0); + defineMethod(JSON_SET, BuiltInMethod.JSON_SET.method, NullPolicy.ARG0); defineMethod(JSON_OBJECT, BuiltInMethod.JSON_OBJECT.method, NullPolicy.NONE); defineMethod(JSON_ARRAY, BuiltInMethod.JSON_ARRAY.method, NullPolicy.NONE); aggMap.put(JSON_OBJECTAGG.with(SqlJsonConstructorNullClause.ABSENT_ON_NULL), diff --git a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java index 3f7df4cfd95d..913d04fd3ea6 100644 --- a/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java +++ b/core/src/main/java/org/apache/calcite/runtime/CalciteResource.java @@ -950,30 +950,39 @@ ExInstWithCause failedToAccessField( @BaseMessage("While executing SQL [{0}] on JDBC sub-schema") ExInst exceptionWhilePerformingQueryOnJdbcSubSchema(String sql); - @BaseMessage("Not a valid input for JSON_TYPE: ''{0}''") + @BaseMessage("Invalid input for JSON_TYPE: ''{0}''") ExInst invalidInputForJsonType(String value); - @BaseMessage("Not a valid input for JSON_DEPTH: ''{0}''") + @BaseMessage("Invalid input for JSON_DEPTH: ''{0}''") ExInst invalidInputForJsonDepth(String value); @BaseMessage("Cannot serialize object to JSON: ''{0}''") ExInst exceptionWhileSerializingToJson(String value); - @BaseMessage("Not a valid input for JSON_LENGTH: ''{0}''") + @BaseMessage("Invalid input for JSON_LENGTH: ''{0}''") ExInst invalidInputForJsonLength(String value); - @BaseMessage("Not a valid input for JSON_KEYS: ''{0}''") + @BaseMessage("Invalid input for JSON_KEYS: ''{0}''") ExInst invalidInputForJsonKeys(String value); @BaseMessage("Invalid input for JSON_REMOVE: document: ''{0}'', jsonpath expressions: ''{1}''") ExInst invalidInputForJsonRemove(String value, String pathSpecs); - @BaseMessage("Not a valid input for JSON_STORAGE_SIZE: ''{0}''") + @BaseMessage("Invalid input for JSON_STORAGE_SIZE: ''{0}''") ExInst invalidInputForJsonStorageSize(String value); - @BaseMessage("Not a valid input for REGEXP_REPLACE: ''{0}''") + @BaseMessage("Invalid input for REGEXP_REPLACE: ''{0}''") ExInst invalidInputForRegexpReplace(String value); + @BaseMessage("Invalid input for JSON_INSERT: jsonDoc: ''{0}'', kvs: ''{1}''") + ExInst invalidInputForJsonInsert(String jsonDoc, String kvs); + + @BaseMessage("Invalid input for JSON_REPLACE: jsonDoc: ''{0}'', kvs: ''{1}''") + ExInst invalidInputForJsonReplace(String jsonDoc, String kvs); + + @BaseMessage("Invalid input for JSON_SET: jsonDoc: ''{0}'', kvs: ''{1}''") + ExInst invalidInputForJsonSet(String jsonDoc, String kvs); + @BaseMessage("Illegal xslt specified : ''{0}''") ExInst illegalXslt(String xslt); diff --git a/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java b/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java index 9108d6b59026..2ff87521f3c0 100644 --- a/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java +++ b/core/src/main/java/org/apache/calcite/runtime/JsonFunctions.java @@ -77,6 +77,8 @@ public class JsonFunctions { new DefaultPrettyPrinter().withObjectIndenter( DefaultIndenter.SYSTEM_LINEFEED_INSTANCE.withLinefeed("\n")); + private static final String JSON_ROOT_PATH = "$"; + private JsonFunctions() { } @@ -658,6 +660,130 @@ public static Integer jsonStorageSize(JsonValueContext input) { } } + private static String jsonModify(JsonValueContext jsonDoc, JsonModifyMode type, Object... kvs) { + int step = type == JsonModifyMode.REMOVE ? 1 : 2; + assert kvs.length % step == 0; + String result = null; + DocumentContext ctx = JsonPath.parse(jsonDoc.obj(), + Configuration + .builder() + .options(Option.SUPPRESS_EXCEPTIONS) + .jsonProvider(JSON_PATH_JSON_PROVIDER) + .mappingProvider(JSON_PATH_MAPPING_PROVIDER) + .build()); + + for (int i = 0; i < kvs.length; i += step) { + String k = (String) kvs[i]; + Object v = kvs[i + step - 1]; + if (!(JsonPath.isPathDefinite(k) && k.contains(JSON_ROOT_PATH))) { + throw RESOURCE.validationError(k).ex(); + } + switch (type) { + case REPLACE: + if (k.equals(JSON_ROOT_PATH)) { + result = jsonize(v); + } else { + if (ctx.read(k) != null) { + ctx.set(k, v); + } + } + break; + case INSERT: + if (!k.equals(JSON_ROOT_PATH) && ctx.read(k) == null) { + insertToJson(ctx, k, v); + } + break; + case SET: + if (k.equals(JSON_ROOT_PATH)) { + result = jsonize(v); + } else { + if (ctx.read(k) != null) { + ctx.set(k, v); + } else { + insertToJson(ctx, k, v); + } + } + break; + case REMOVE: + if (ctx.read(k) != null) { + ctx.delete(k); + } + break; + } + } + + return result == null || result.isEmpty() ? ctx.jsonString() : result; + } + + private static void insertToJson(DocumentContext ctx, String path, Object value) { + final String parentPath; + final String key; + + //The following paragraph of logic is mainly used to obtain the parent node path of path and + //the key value that should be inserted when the preant Path is map. + //eg: path is $.a ,parentPath is $ + //eg: path is $.a[1] ,parentPath is $.a + Integer dotIndex = path.lastIndexOf("."); + Integer leftBracketIndex = path.lastIndexOf("["); + if (dotIndex.equals(-1) && leftBracketIndex.equals(-1)) { + parentPath = path; + key = path; + } else if (!dotIndex.equals(-1) && leftBracketIndex.equals(-1)) { + parentPath = path.substring(0, dotIndex); + key = path.substring(dotIndex + 1); + } else if (dotIndex.equals(-1) && !leftBracketIndex.equals(-1)) { + parentPath = path.substring(0, leftBracketIndex); + key = path.substring(leftBracketIndex + 1); + } else { + int position = dotIndex > leftBracketIndex ? dotIndex : leftBracketIndex; + parentPath = path.substring(0, position); + key = path.substring(position); + } + + Object obj = ctx.read(parentPath); + if (obj instanceof Map) { + ctx.put(parentPath, key, value.toString()); + } else if (obj instanceof Collection) { + ctx.add(parentPath, value); + } + } + + public static String jsonReplace(String jsonDoc, Object... kvs) { + return jsonReplace(jsonValueExpression(jsonDoc), kvs); + } + + public static String jsonReplace(JsonValueContext input, Object... kvs) { + try { + return jsonModify(input, JsonModifyMode.REPLACE, kvs); + } catch (Exception ex) { + throw RESOURCE.invalidInputForJsonReplace(input.toString(), jsonize(kvs)).ex(); + } + } + + public static String jsonInsert(String jsonDoc, Object... kvs) { + return jsonInsert(jsonValueExpression(jsonDoc), kvs); + } + + public static String jsonInsert(JsonValueContext input, Object... kvs) { + try { + return jsonModify(input, JsonModifyMode.INSERT, kvs); + } catch (Exception ex) { + throw RESOURCE.invalidInputForJsonInsert(input.toString(), jsonize(kvs)).ex(); + } + } + + public static String jsonSet(String jsonDoc, Object... kvs) { + return jsonSet(jsonValueExpression(jsonDoc), kvs); + } + + public static String jsonSet(JsonValueContext input, Object... kvs) { + try { + return jsonModify(input, JsonModifyMode.SET, kvs); + } catch (Exception ex) { + throw RESOURCE.invalidInputForJsonSet(input.toString(), jsonize(kvs)).ex(); + } + } + public static boolean isJsonValue(String input) { try { dejsonize(input); @@ -818,4 +944,14 @@ public enum PathMode { UNKNOWN, NONE } + + /** + * Used in the JsonModify function. + */ + public enum JsonModifyMode { + REPLACE, + INSERT, + SET, + REMOVE + } } diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonModifyFunction.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonModifyFunction.java new file mode 100644 index 000000000000..ba8896bf7c72 --- /dev/null +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlJsonModifyFunction.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql.fun; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlCall; +import org.apache.calcite.sql.SqlCallBinding; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperandCountRange; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlOperandCountRanges; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; +import org.apache.calcite.sql.type.SqlTypeTransforms; +import org.apache.calcite.sql.type.SqlTypeUtil; +import org.apache.calcite.sql.validate.SqlValidator; + +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.util.Locale; + +import static org.apache.calcite.util.Static.RESOURCE; + +/** + * Definition of the JSON_INSERT, JSON_REPLACE + * and JSON_SET JSON Modify functions. + */ +public class SqlJsonModifyFunction extends SqlFunction { + public SqlJsonModifyFunction(String name) { + super(name, + SqlKind.OTHER_FUNCTION, + ReturnTypes.cascade(ReturnTypes.VARCHAR_2000, + SqlTypeTransforms.FORCE_NULLABLE), + null, + OperandTypes.ANY, + SqlFunctionCategory.SYSTEM); + } + + @Override public SqlOperandCountRange getOperandCountRange() { + return SqlOperandCountRanges.from(3); + } + + @Override protected void checkOperandCount(SqlValidator validator, + @Nullable SqlOperandTypeChecker argType, SqlCall call) { + assert (call.operandCount() >= 3) && (call.operandCount() % 2 == 1); + } + + @Override public boolean checkOperandTypes(SqlCallBinding callBinding, + boolean throwOnFailure) { + final int count = callBinding.getOperandCount(); + for (int i = 1; i < count; i += 2) { + RelDataType nameType = callBinding.getOperandType(i); + if (!SqlTypeUtil.isCharacter(nameType)) { + if (throwOnFailure) { + throw callBinding.newError(RESOURCE.expectedCharacter()); + } + return false; + } + if (nameType.isNullable()) { + if (throwOnFailure) { + throw callBinding.newError( + RESOURCE.argumentMustNotBeNull( + callBinding.operand(i).toString())); + } + return false; + } + } + return true; + } + + @Override public String getSignatureTemplate(int operandsCount) { + assert operandsCount % 2 == 1; + StringBuilder sb = new StringBuilder(); + sb.append("{0}("); + for (int i = 1; i < operandsCount; i++) { + sb.append(String.format(Locale.ROOT, "{%d} ", i + 1)); + } + sb.append("{1})"); + return sb.toString(); + } +} diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java index 43fe6e98ee3a..2891d7df8b1a 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlLibraryOperators.java @@ -305,6 +305,15 @@ private SqlLibraryOperators() { @LibraryOperator(libraries = {MYSQL}) public static final SqlFunction JSON_STORAGE_SIZE = new SqlJsonStorageSizeFunction(); + @LibraryOperator(libraries = {MYSQL}) + public static final SqlFunction JSON_INSERT = new SqlJsonModifyFunction("JSON_INSERT"); + + @LibraryOperator(libraries = {MYSQL}) + public static final SqlFunction JSON_REPLACE = new SqlJsonModifyFunction("JSON_REPLACE"); + + @LibraryOperator(libraries = {MYSQL}) + public static final SqlFunction JSON_SET = new SqlJsonModifyFunction("JSON_SET"); + @LibraryOperator(libraries = {MYSQL, ORACLE}) public static final SqlFunction REGEXP_REPLACE = new SqlRegexpReplaceFunction(); diff --git a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java index 6a1f6403ba00..51f3dcba18cd 100644 --- a/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java +++ b/core/src/main/java/org/apache/calcite/sql/fun/SqlStdOperatorTable.java @@ -1415,6 +1415,15 @@ public class SqlStdOperatorTable extends ReflectiveSqlOperatorTable { @Deprecated // to be removed before 2.0 public static final SqlFunction JSON_STORAGE_SIZE = SqlLibraryOperators.JSON_STORAGE_SIZE; + @Deprecated // to be removed before 2.0 + public static final SqlFunction JSON_INSERT = SqlLibraryOperators.JSON_INSERT; + + @Deprecated // to be removed before 2.0 + public static final SqlFunction JSON_REPLACE = SqlLibraryOperators.JSON_REPLACE; + + @Deprecated // to be removed before 2.0 + public static final SqlFunction JSON_SET = SqlLibraryOperators.JSON_SET; + public static final SqlJsonArrayAggAggFunction JSON_ARRAYAGG = new SqlJsonArrayAggAggFunction(SqlKind.JSON_ARRAYAGG, SqlJsonConstructorNullClause.ABSENT_ON_NULL); diff --git a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java index ba4c6d0f0cd9..e213ac01a02f 100644 --- a/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java +++ b/core/src/main/java/org/apache/calcite/util/BuiltInMethod.java @@ -379,10 +379,13 @@ public enum BuiltInMethod { JSON_TYPE(JsonFunctions.class, "jsonType", String.class), JSON_DEPTH(JsonFunctions.class, "jsonDepth", String.class), JSON_KEYS(JsonFunctions.class, "jsonKeys", String.class), + JSON_INSERT(JsonFunctions.class, "jsonInsert", String.class, Object.class), JSON_PRETTY(JsonFunctions.class, "jsonPretty", String.class), JSON_LENGTH(JsonFunctions.class, "jsonLength", String.class), + JSON_REPLACE(JsonFunctions.class, "jsonReplace", String.class, Object.class), JSON_REMOVE(JsonFunctions.class, "jsonRemove", String.class), JSON_STORAGE_SIZE(JsonFunctions.class, "jsonStorageSize", String.class), + JSON_SET(JsonFunctions.class, "jsonSet", String.class, Object.class), JSON_OBJECTAGG_ADD(JsonFunctions.class, "jsonObjectAggAdd", Map.class, String.class, Object.class, SqlJsonConstructorNullClause.class), JSON_ARRAY(JsonFunctions.class, "jsonArray", diff --git a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties index 86aad977e9d3..d340a3a2dacd 100644 --- a/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties +++ b/core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties @@ -310,14 +310,17 @@ NullKeyOfJsonObjectNotAllowed=Null key of JSON object is not allowed QueryExecutionTimeoutReached=Timeout of ''{0}'' ms for query execution is reached. Query execution started at ''{1}'' AmbiguousSortOrderInJsonArrayAggFunc=Including both WITHIN GROUP(...) and inside ORDER BY in a single JSON_ARRAYAGG call is not allowed ExceptionWhilePerformingQueryOnJdbcSubSchema = While executing SQL [{0}] on JDBC sub-schema -InvalidInputForJsonType=Not a valid input for JSON_TYPE: ''{0}'' -InvalidInputForJsonDepth=Not a valid input for JSON_DEPTH: ''{0}'' +InvalidInputForJsonType=Invalid input for JSON_TYPE: ''{0}'' +InvalidInputForJsonDepth=Invalid input for JSON_DEPTH: ''{0}'' ExceptionWhileSerializingToJson=Cannot serialize object to JSON: ''{0}'' -InvalidInputForJsonLength=Not a valid input for JSON_LENGTH: ''{0}'' -InvalidInputForJsonKeys=Not a valid input for JSON_KEYS: ''{0}'' +InvalidInputForJsonLength=Invalid input for JSON_LENGTH: ''{0}'' +InvalidInputForJsonKeys=Invalid input for JSON_KEYS: ''{0}'' InvalidInputForJsonRemove=Invalid input for JSON_REMOVE: document: ''{0}'', jsonpath expressions: ''{1}'' -InvalidInputForJsonStorageSize=Not a valid input for JSON_STORAGE_SIZE: ''{0}'' -InvalidInputForRegexpReplace=Not a valid input for REGEXP_REPLACE: ''{0}'' +InvalidInputForJsonStorageSize=Invalid input for JSON_STORAGE_SIZE: ''{0}'' +InvalidInputForRegexpReplace=Invalid input for REGEXP_REPLACE: ''{0}'' +InvalidInputForJsonInsert=Invalid input for JSON_INSERT: jsonDoc: ''{0}'', kvs: ''{1}'' +InvalidInputForJsonReplace=Invalid input for JSON_REPLACE: jsonDoc: ''{0}'', kvs: ''{1}'' +InvalidInputForJsonSet=Invalid input for JSON_SET: jsonDoc: ''{0}'', kvs: ''{1}'' IllegalXslt=Illegal xslt specified : ''{0}'' InvalidInputForXmlTransform=Invalid input for XMLTRANSFORM xml: ''{0}'' InvalidInputForExtractValue=Invalid input for EXTRACTVALUE: xml: ''{0}'', xpath expression: ''{1}'' diff --git a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java index 940d4b34fac5..8aa886ceee76 100644 --- a/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java +++ b/core/src/test/java/org/apache/calcite/rel/rel2sql/RelToSqlConverterTest.java @@ -6107,6 +6107,42 @@ private void checkLiteral2(String expression, String expected) { sql(query).ok(expected); } + @Test public void testJsonInsert() { + String query0 = "select json_insert(\"product_name\", '$', 10) from \"product\""; + String query1 = "select json_insert(cast(null as varchar), '$', 10, '$', null, '$'," + + " '\n\t\n') from \"product\""; + final String expected0 = "SELECT JSON_INSERT(\"product_name\", '$', 10)\n" + + "FROM \"foodmart\".\"product\""; + final String expected1 = "SELECT JSON_INSERT(NULL, '$', 10, '$', NULL, '$', " + + "u&'\\000a\\0009\\000a')\nFROM \"foodmart\".\"product\""; + sql(query0).ok(expected0); + sql(query1).ok(expected1); + } + + @Test public void testJsonReplace() { + String query = "select json_replace(\"product_name\", '$', 10) from \"product\""; + String query1 = "select json_replace(cast(null as varchar), '$', 10, '$', null, '$'," + + " '\n\t\n') from \"product\""; + final String expected = "SELECT JSON_REPLACE(\"product_name\", '$', 10)\n" + + "FROM \"foodmart\".\"product\""; + final String expected1 = "SELECT JSON_REPLACE(NULL, '$', 10, '$', NULL, '$', " + + "u&'\\000a\\0009\\000a')\nFROM \"foodmart\".\"product\""; + sql(query).ok(expected); + sql(query1).ok(expected1); + } + + @Test public void testJsonSet() { + String query = "select json_set(\"product_name\", '$', 10) from \"product\""; + String query1 = "select json_set(cast(null as varchar), '$', 10, '$', null, '$'," + + " '\n\t\n') from \"product\""; + final String expected = "SELECT JSON_SET(\"product_name\", '$', 10)\n" + + "FROM \"foodmart\".\"product\""; + final String expected1 = "SELECT JSON_SET(NULL, '$', 10, '$', NULL, '$', " + + "u&'\\000a\\0009\\000a')\nFROM \"foodmart\".\"product\""; + sql(query).ok(expected); + sql(query1).ok(expected1); + } + @Test void testUnionAllWithNoOperandsUsingOracleDialect() { String query = "select A.\"department_id\" " + "from \"foodmart\".\"employee\" A " diff --git a/core/src/test/java/org/apache/calcite/test/JdbcTest.java b/core/src/test/java/org/apache/calcite/test/JdbcTest.java index c7033255c839..61f97f90fae5 100644 --- a/core/src/test/java/org/apache/calcite/test/JdbcTest.java +++ b/core/src/test/java/org/apache/calcite/test/JdbcTest.java @@ -7878,6 +7878,36 @@ private void checkGetTimestamp(Connection con) throws SQLException { .returns("A=29; B=35; C=37; D=36\n"); } + @Test public void testJsonInsert() { + CalciteAssert.that() + .with(CalciteConnectionProperty.FUN, "mysql") + .query("SELECT JSON_INSERT(v, '$.a', 10, '$.c', '[1]') AS c1\n" + + ",JSON_INSERT(v, '$', 10, '$.c', '[1]') AS c2\n" + + "FROM (VALUES ('{\"a\": 1,\"b\":[2]}')) AS t(v)\n" + + "limit 10") + .returns("C1={\"a\":1,\"b\":[2],\"c\":\"[1]\"}; C2={\"a\":1,\"b\":[2],\"c\":\"[1]\"}\n"); + } + + @Test public void testJsonReplace() { + CalciteAssert.that() + .with(CalciteConnectionProperty.FUN, "mysql") + .query("SELECT JSON_REPLACE(v, '$.a', 10, '$.c', '[1]') AS c1\n" + + ",JSON_REPLACE(v, '$', 10, '$.c', '[1]') AS c2\n" + + "FROM (VALUES ('{\"a\": 1,\"b\":[2]}')) AS t(v)\n" + + "limit 10") + .returns("C1={\"a\":10,\"b\":[2]}; C2=10\n"); + } + + @Test public void testJsonSet() { + CalciteAssert.that() + .with(CalciteConnectionProperty.FUN, "mysql") + .query("SELECT JSON_SET(v, '$.a', 10, '$.c', '[1]') AS c1\n" + + ",JSON_SET(v, '$', 10, '$.c', '[1]') AS c2\n" + + "FROM (VALUES ('{\"a\": 1,\"b\":[2]}')) AS t(v)\n" + + "limit 10") + .returns("C1={\"a\":10,\"b\":[2],\"c\":\"[1]\"}; C2=10\n"); + } + /** * Test case for * [CALCITE-2609] diff --git a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java index ce60ca8d564a..683333b59628 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlFunctionsTest.java @@ -157,7 +157,7 @@ class SqlFunctionsTest { fail("'regexp_replace' on an invalid pos is not possible"); } catch (CalciteException e) { assertThat(e.getMessage(), - is("Not a valid input for REGEXP_REPLACE: '0'")); + is("Invalid input for REGEXP_REPLACE: '0'")); } try { @@ -165,7 +165,7 @@ class SqlFunctionsTest { fail("'regexp_replace' on an invalid matchType is not possible"); } catch (CalciteException e) { assertThat(e.getMessage(), - is("Not a valid input for REGEXP_REPLACE: 'WWW'")); + is("Invalid input for REGEXP_REPLACE: 'WWW'")); } } diff --git a/core/src/test/java/org/apache/calcite/test/SqlJsonFunctionsTest.java b/core/src/test/java/org/apache/calcite/test/SqlJsonFunctionsTest.java index 0d1a3b3b6338..605b99320f7e 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlJsonFunctionsTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlJsonFunctionsTest.java @@ -595,6 +595,39 @@ class SqlJsonFunctionsTest { assertIsJsonScalar("{]", is(false)); } + @Test public void testJsonInsert() { + assertJsonInsert( + JsonFunctions.jsonValueExpression("{\"a\": 1, \"b\": [2]}"), + new Object[]{"$.a", 10, "$.c", "[true]"}, + is("{\"a\":1,\"b\":[2],\"c\":\"[true]\"}")); + assertJsonInsert( + JsonFunctions.jsonValueExpression("{\"a\": 1, \"b\": [2]}"), + new Object[]{"$", 10, "$.c", "[true]"}, + is("{\"a\":1,\"b\":[2],\"c\":\"[true]\"}")); + } + + @Test public void testJsonReplace() { + assertJsonReplace( + JsonFunctions.jsonValueExpression("{\"a\": 1, \"b\": [2]}"), + new Object[]{"$.a", 10, "$.c", "[true]"}, + is("{\"a\":10,\"b\":[2]}")); + assertJsonReplace( + JsonFunctions.jsonValueExpression("{\"a\": 1, \"b\": [2]}"), + new Object[]{"$", 10, "$.c", "[true]"}, + is("10")); + } + + @Test public void testJsonSet() { + assertJsonSet( + JsonFunctions.jsonValueExpression("{\"a\": 1, \"b\": [2]}"), + new Object[]{"$.a", 10, "$.c", "[true]"}, + is("{\"a\":10,\"b\":[2],\"c\":\"[true]\"}")); + assertJsonSet( + JsonFunctions.jsonValueExpression("{\"a\": 1, \"b\": [2]}"), + new Object[]{"$", 10, "$.c", "[true]"}, + is("10")); + } + private void assertJsonValueExpression(String input, Matcher matcher) { assertThat( @@ -764,6 +797,30 @@ private void assertJsonStorageSizeFailed(String input, matcher); } + private void assertJsonInsert(JsonFunctions.JsonValueContext jsonDoc, + Object[] kvs, + Matcher matcher) { + assertThat(invocationDesc(BuiltInMethod.JSON_INSERT.getMethodName(), jsonDoc, kvs), + JsonFunctions.jsonInsert(jsonDoc, kvs), + matcher); + } + + private void assertJsonReplace(JsonFunctions.JsonValueContext jsonDoc, + Object[] kvs, + Matcher matcher) { + assertThat(invocationDesc(BuiltInMethod.JSON_REPLACE.getMethodName(), jsonDoc, kvs), + JsonFunctions.jsonReplace(jsonDoc, kvs), + matcher); + } + + private void assertJsonSet(JsonFunctions.JsonValueContext jsonDoc, + Object[] kvs, + Matcher matcher) { + assertThat(invocationDesc(BuiltInMethod.JSON_SET.getMethodName(), jsonDoc, kvs), + JsonFunctions.jsonSet(jsonDoc, kvs), + matcher); + } + private void assertDejsonize(String input, Matcher matcher) { assertThat(invocationDesc(BuiltInMethod.DEJSONIZE.getMethodName(), input), diff --git a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java index bad43e5ddda8..4dc72e96ae7d 100644 --- a/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java +++ b/core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java @@ -12530,6 +12530,30 @@ private void checkCustomColumnResolving(String table) { .fails("(?s).*Cannot apply.*"); } + @Test public void testJsonInsert() { + expr("json_insert('{ \"a\": 1, \"b\": [2]}', '$.a', 10, '$.c', '[true]')").ok(); + expr("json_insert('{ \"a\": 1, \"b\": [2]}', '$.a', 10, '$.c', '[true]')") + .columnType("VARCHAR(2000)"); + expr("select ^json_insert('{\"foo\":\"bar\"}')^") + .fails("(?s).*Invalid number of arguments.*"); + } + + @Test public void testJsonReplace() { + expr("json_replace('{ \"a\": 1, \"b\": [2]}', '$.a', 10, '$.c', '[true]')").ok(); + expr("json_replace('{ \"a\": 1, \"b\": [2]}', '$.a', 10, '$.c', '[true]')") + .columnType("VARCHAR(2000)"); + expr("select ^json_replace('{\"foo\":\"bar\"}')^") + .fails("(?s).*Invalid number of arguments.*"); + } + + @Test public void testJsonSet() { + expr("json_set('{ \"a\": 1, \"b\": [2]}', '$.a', 10, '$.c', '[true]')").ok(); + expr("json_set('{ \"a\": 1, \"b\": [2]}', '$.a', 10, '$.c', '[true]')") + .columnType("VARCHAR(2000)"); + expr("select ^json_set('{\"foo\":\"bar\"}')^") + .fails("(?s).*Invalid number of arguments.*"); + } + @Test void testRegexpReplace() { final SqlOperatorTable opTable = operatorTableFor(SqlLibrary.ORACLE); diff --git a/site/_docs/reference.md b/site/_docs/reference.md index 319b13ced807..bfdbf34dfdfb 100644 --- a/site/_docs/reference.md +++ b/site/_docs/reference.md @@ -2630,8 +2630,11 @@ semantics. | m | JSON_DEPTH(jsonValue) | Returns an integer value indicating the depth of *jsonValue* | m | JSON_PRETTY(jsonValue) | Returns a pretty-printing of *jsonValue* | m | JSON_LENGTH(jsonValue [, path ]) | Returns a integer indicating the length of *jsonValue* +| m | JSON_INSERT(jsonValue, path, val[, path, val]*) | Returns a JSON document insert a data of *jsonValue*, *path*, *val* | m | JSON_KEYS(jsonValue [, path ]) | Returns a string indicating the keys of a JSON *jsonValue* | m | JSON_REMOVE(jsonValue, path[, path]) | Removes data from *jsonValue* using a series of *path* expressions and returns the result +| m | JSON_REPLACE(jsonValue, path, val[, path, val]*) | Returns a JSON document replace a data of *jsonValue*, *path*, *val* +| m | JSON_SET(jsonValue, path, val[, path, val]*) | Returns a JSON document set a data of *jsonValue*, *path*, *val* | m | JSON_STORAGE_SIZE(jsonValue) | Returns the number of bytes used to store the binary representation of *jsonValue* | b o | LEAST(expr [, expr ]* ) | Returns the least of the expressions | b m p | LEFT(string, length) | Returns the leftmost *length* characters from the *string* @@ -2774,6 +2777,23 @@ Result |:------:|:-----:|:-------:|:-------:| | 1 | 2 | 1 | 1 | +##### JSON_INSERT example + +SQL + +```SQL +SELECT JSON_INSERT(v, '$.a', 10, '$.c', '[1]') AS c1, + JSON_INSERT(v, '$', 10, '$.c', '[1]') AS c2 +FROM (VALUES ('{"a": [10, true]}')) AS t(v) +LIMIT 10; +``` + +Result + +| c1 | c2 | +| ------------------------------ | ----------------------------- | +| {"a":1 , "b":[2] , "c":"[1]"} | {"a":1 , "b":[2] , "c":"[1]"} | + ##### JSON_KEYS example SQL @@ -2810,6 +2830,41 @@ LIMIT 10; |:----------:| | ["a", "d"] | +##### JSON_REPLACE example + +SQL + + ```SQL +SELECT +JSON_REPLACE(v, '$.a', 10, '$.c', '[1]') AS c1, +JSON_REPLACE(v, '$', 10, '$.c', '[1]') AS c2 +FROM (VALUES ('{\"a\": 1,\"b\":[2]}')) AS t(v) +limit 10; +``` + + Result + +| c1 | c2 | +| ------------------------------ | ------------------------------- | +| {"a":1 , "b":[2] , "c":"[1]"} | {"a":1 , "b":[2] , "c":"[1]"}") | + +##### JSON_SET example + +SQL + + ```SQL +SELECT +JSON_SET(v, '$.a', 10, '$.c', '[1]') AS c1, +JSON_SET(v, '$', 10, '$.c', '[1]') AS c2 +FROM (VALUES ('{\"a\": 1,\"b\":[2]}')) AS t(v) +limit 10; +``` + + Result + +| c1 | c2 | +| -------------------| -- | +| {"a":10, "b":[2]} | 10 | ##### JSON_STORAGE_SIZE example @@ -2868,12 +2923,6 @@ Result |:-----------:|:-----------:|:-----------:|:-----------:| | Aa_Bb_CcD_d | Aa_Bb_CcD_d | Aa_Bb_CcD_d | Aa_Bb_CcD_d | -Not implemented: - -* JSON_INSERT -* JSON_SET -* JSON_REPLACE - ## User-defined functions Calcite is extensible. You can define each kind of function using user code. diff --git a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java index 465110e9d6bd..429a10251309 100644 --- a/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java +++ b/testkit/src/main/java/org/apache/calcite/test/SqlOperatorTest.java @@ -4049,6 +4049,60 @@ private static void checkIf(SqlOperatorFixture f) { } + @Test public void testJsonInsert() { + final SqlOperatorFixture f = fixture().withLibrary(SqlLibrary.MYSQL); + f.checkString("json_insert('10', '$.a', 10, '$.c', '[true]')", + "10", "VARCHAR(2000)"); + f.checkString("json_insert('{ \"a\": 1, \"b\": [2]}', '$.a', 10, '$.c', '[true]')", + "{\"a\":1,\"b\":[2],\"c\":\"[true]\"}", "VARCHAR(2000)"); + f.checkString("json_insert('{ \"a\": 1, \"b\": [2]}', '$', 10, '$', '[true]')", + "{\"a\":1,\"b\":[2]}", "VARCHAR(2000)"); + f.checkString("json_insert('{ \"a\": 1, \"b\": [2]}', '$.b[1]', 10)", + "{\"a\":1,\"b\":[2,10]}", "VARCHAR(2000)"); + f.checkString("json_insert('{\"a\": 1, \"b\": [2, 3, [true]]}', '$.b[3]', 'false')", + "{\"a\":1,\"b\":[2,3,[true],\"false\"]}", "VARCHAR(2000)"); + f.checkFails("json_insert('{\"a\": 1, \"b\": [2, 3, [true]]}', 'a', 'false')", + "(?s).*Invalid input for.*", true); + // nulls + f.checkNull("json_insert(cast(null as varchar), '$', 10)"); + } + + @Test public void testJsonReplace() { + final SqlOperatorFixture f = fixture().withLibrary(SqlLibrary.MYSQL); + f.checkString("json_replace('10', '$.a', 10, '$.c', '[true]')", + "10", "VARCHAR(2000)"); + f.checkString("json_replace('{ \"a\": 1, \"b\": [2]}', '$.a', 10, '$.c', '[true]')", + "{\"a\":10,\"b\":[2]}", "VARCHAR(2000)"); + f.checkString("json_replace('{ \"a\": 1, \"b\": [2]}', '$', 10, '$.c', '[true]')", + "10", "VARCHAR(2000)"); + f.checkString("json_replace('{ \"a\": 1, \"b\": [2]}', '$.b', 10, '$.c', '[true]')", + "{\"a\":1,\"b\":10}", "VARCHAR(2000)"); + f.checkString("json_replace('{ \"a\": 1, \"b\": [2, 3]}', '$.b[1]', 10)", + "{\"a\":1,\"b\":[2,10]}", "VARCHAR(2000)"); + f.checkFails("json_replace('{\"a\": 1, \"b\": [2, 3, [true]]}', 'a', 'false')", + "(?s).*Invalid input for.*", true); + + // nulls + f.checkNull("json_replace(cast(null as varchar), '$', 10)"); + } + + @Test public void testJsonSet() { + final SqlOperatorFixture f = fixture().withLibrary(SqlLibrary.MYSQL); + f.checkString("json_set('10', '$.a', 10, '$.c', '[true]')", + "10", "VARCHAR(2000)"); + f.checkString("json_set('{ \"a\": 1, \"b\": [2]}', '$.a', 10, '$.c', '[true]')", + "{\"a\":10,\"b\":[2],\"c\":\"[true]\"}", "VARCHAR(2000)"); + f.checkString("json_set('{ \"a\": 1, \"b\": [2]}', '$', 10, '$.c', '[true]')", + "10", "VARCHAR(2000)"); + f.checkString("json_set('{ \"a\": 1, \"b\": [2, 3]}', '$.b[1]', 10, '$.c', '[true]')", + "{\"a\":1,\"b\":[2,10],\"c\":\"[true]\"}", "VARCHAR(2000)"); + f.checkFails("json_set('{\"a\": 1, \"b\": [2, 3, [true]]}', 'a', 'false')", + "(?s).*Invalid input for.*", true); + + // nulls + f.checkNull("json_set(cast(null as varchar), '$', 10)"); + } + @Test void testJsonValue() { final SqlOperatorFixture f = fixture(); if (false) {