From 4fa502d7e00cc290829fd5e78f1e6aff39e4e6f1 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 11 Dec 2024 11:30:58 +0100 Subject: [PATCH 1/6] _score not reserved attribute --- .../functions/description/categorize.asciidoc | 2 +- .../functions/kibana/definition/term.json | 2 +- .../esql/functions/kibana/docs/term.md | 8 +++--- .../xpack/esql/analysis/Verifier.java | 9 ------- .../xpack/esql/analysis/VerifierTests.java | 25 ------------------- 5 files changed, 6 insertions(+), 40 deletions(-) diff --git a/docs/reference/esql/functions/description/categorize.asciidoc b/docs/reference/esql/functions/description/categorize.asciidoc index 32af0051e91c8..c956066ad53f3 100644 --- a/docs/reference/esql/functions/description/categorize.asciidoc +++ b/docs/reference/esql/functions/description/categorize.asciidoc @@ -8,4 +8,4 @@ Groups text messages into categories of similarly formatted text values. * can't be used within other expressions * can't be used with multiple groupings -* can't be used or referenced within aggregations +* can't be used or referenced within aggregate functions diff --git a/docs/reference/esql/functions/kibana/definition/term.json b/docs/reference/esql/functions/kibana/definition/term.json index d8bb61fd596a1..b0f129afd239c 100644 --- a/docs/reference/esql/functions/kibana/definition/term.json +++ b/docs/reference/esql/functions/kibana/definition/term.json @@ -78,7 +78,7 @@ } ], "examples" : [ - "from books \n| where term(author, \"gabriel\") \n| keep book_no, title\n| limit 3;" + "FROM books \n| WHERE TERM(author, \"gabriel\") \n| KEEP book_no, title\n| LIMIT 3;" ], "preview" : true, "snapshot_only" : true diff --git a/docs/reference/esql/functions/kibana/docs/term.md b/docs/reference/esql/functions/kibana/docs/term.md index 83e61a949208d..ffecd26d737f7 100644 --- a/docs/reference/esql/functions/kibana/docs/term.md +++ b/docs/reference/esql/functions/kibana/docs/term.md @@ -6,8 +6,8 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ Performs a Term query on the specified field. Returns true if the provided term matches the row. ``` -from books -| where term(author, "gabriel") -| keep book_no, title -| limit 3; +FROM books +| WHERE TERM(author, "gabriel") +| KEEP book_no, title +| LIMIT 3; ``` diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index a0728c9a91088..c805adf5d5a57 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -18,7 +18,6 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; -import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.function.Function; @@ -208,7 +207,6 @@ else if (p instanceof Lookup lookup) { checkJoin(p, failures); }); checkRemoteEnrich(plan, failures); - checkMetadataScoreNameReserved(plan, failures); if (failures.isEmpty()) { checkLicense(plan, licenseState, failures); @@ -222,13 +220,6 @@ else if (p instanceof Lookup lookup) { return failures; } - private static void checkMetadataScoreNameReserved(LogicalPlan p, Set failures) { - // _score can only be set as metadata attribute - if (p.inputSet().stream().anyMatch(a -> MetadataAttribute.SCORE.equals(a.name()) && (a instanceof MetadataAttribute) == false)) { - failures.add(fail(p, "`" + MetadataAttribute.SCORE + "` is a reserved METADATA attribute")); - } - } - private void checkSort(LogicalPlan p, Set failures) { if (p instanceof OrderBy ob) { ob.order().forEach(o -> { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index d58d233168e2b..84dcdbadef9f0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -12,7 +12,6 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; -import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; @@ -22,7 +21,6 @@ import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.parser.QueryParam; import org.elasticsearch.xpack.esql.parser.QueryParams; -import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -1805,29 +1803,6 @@ public void testToDatePeriodToTimeDurationWithInvalidType() { ); } - public void testNonMetadataScore() { - assumeTrue("'METADATA _score' is disabled", EsqlCapabilities.Cap.METADATA_SCORE.isEnabled()); - assertEquals("1:12: `_score` is a reserved METADATA attribute", error("from foo | eval _score = 10")); - - assertEquals( - "1:48: `_score` is a reserved METADATA attribute", - error("from foo metadata _score | where qstr(\"bar\") | eval _score = _score + 1") - ); - } - - public void testScoreRenaming() { - assumeTrue("'METADATA _score' is disabled", EsqlCapabilities.Cap.METADATA_SCORE.isEnabled()); - assertEquals("1:33: `_score` is a reserved METADATA attribute", error("from foo METADATA _id, _score | rename _id as _score")); - - assertTrue(passes("from foo metadata _score | rename _score as foo").stream().anyMatch(a -> a.name().equals("foo"))); - } - - private List passes(String query) { - LogicalPlan logicalPlan = defaultAnalyzer.analyze(parser.createStatement(query)); - assertTrue(logicalPlan.resolved()); - return logicalPlan.output(); - } - public void testIntervalAsString() { // DateTrunc for (String interval : List.of("1 minu", "1 dy", "1.5 minutes", "0.5 days", "minutes 1", "day 5")) { From 57b971248d83d2907af944f3ceca93de7f69304c Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 11 Dec 2024 11:35:15 +0100 Subject: [PATCH 2/6] Update docs/changelog/118435.yaml --- docs/changelog/118435.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/118435.yaml diff --git a/docs/changelog/118435.yaml b/docs/changelog/118435.yaml new file mode 100644 index 0000000000000..200bdf6157aa5 --- /dev/null +++ b/docs/changelog/118435.yaml @@ -0,0 +1,5 @@ +pr: 118435 +summary: '`_score` should not be a reserved attribute in ES|QL' +area: ES|QL +type: enhancement +issues: [] From 1a28c8af37e3e9ae9c6228f7364146abc8f68536 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 11 Dec 2024 11:36:06 +0100 Subject: [PATCH 3/6] reverted unrelated changes --- .../esql/functions/description/categorize.asciidoc | 2 +- docs/reference/esql/functions/kibana/definition/term.json | 2 +- docs/reference/esql/functions/kibana/docs/term.md | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/reference/esql/functions/description/categorize.asciidoc b/docs/reference/esql/functions/description/categorize.asciidoc index c956066ad53f3..32af0051e91c8 100644 --- a/docs/reference/esql/functions/description/categorize.asciidoc +++ b/docs/reference/esql/functions/description/categorize.asciidoc @@ -8,4 +8,4 @@ Groups text messages into categories of similarly formatted text values. * can't be used within other expressions * can't be used with multiple groupings -* can't be used or referenced within aggregate functions +* can't be used or referenced within aggregations diff --git a/docs/reference/esql/functions/kibana/definition/term.json b/docs/reference/esql/functions/kibana/definition/term.json index b0f129afd239c..d8bb61fd596a1 100644 --- a/docs/reference/esql/functions/kibana/definition/term.json +++ b/docs/reference/esql/functions/kibana/definition/term.json @@ -78,7 +78,7 @@ } ], "examples" : [ - "FROM books \n| WHERE TERM(author, \"gabriel\") \n| KEEP book_no, title\n| LIMIT 3;" + "from books \n| where term(author, \"gabriel\") \n| keep book_no, title\n| limit 3;" ], "preview" : true, "snapshot_only" : true diff --git a/docs/reference/esql/functions/kibana/docs/term.md b/docs/reference/esql/functions/kibana/docs/term.md index ffecd26d737f7..83e61a949208d 100644 --- a/docs/reference/esql/functions/kibana/docs/term.md +++ b/docs/reference/esql/functions/kibana/docs/term.md @@ -6,8 +6,8 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ Performs a Term query on the specified field. Returns true if the provided term matches the row. ``` -FROM books -| WHERE TERM(author, "gabriel") -| KEEP book_no, title -| LIMIT 3; +from books +| where term(author, "gabriel") +| keep book_no, title +| limit 3; ``` From 0ff6fb39a6a24236f9592c9125046fc96340f33e Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 11 Dec 2024 12:35:57 +0100 Subject: [PATCH 4/6] add scoring manipulation/override csv tests --- .../src/main/resources/scoring.csv-spec | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec index d4c7b8c59fdbc..cb38204a71ab0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec @@ -283,3 +283,33 @@ book_no:keyword | c_score:double 7350 | 2.0 7140 | 3.0 ; + +QstrScoreManipulation +required_capability: metadata_score +required_capability: qstr_function + +from books metadata _score +| where qstr("title:rings") +| eval _score = _score + 1 +| keep book_no, title, _score +| limit 2; + +book_no:keyword | title:text | _score:double +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | 2.6404519081115723 +2714 | Return of the King Being the Third Part of The Lord of the Rings | 2.9239964485168457 +; + +QstrScoreOverride +required_capability: metadata_score +required_capability: qstr_function + +from books metadata _score +| where qstr("title:rings") +| eval _score = "foobar" +| keep book_no, title, _score +| limit 2; + +book_no:keyword | title:text | _score:keyword +4023 | A Tolkien Compass: Including J. R. R. Tolkien's Guide to the Names in The Lord of the Rings | foobar +2714 | Return of the King Being the Third Part of The Lord of the Rings | foobar +; From 31cbc9d1209c7d1f7de832fe0b531dc04e1d8ce5 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 11 Dec 2024 14:58:38 +0100 Subject: [PATCH 5/6] mute score manipulation csv tests for bwc tests (temporarily) --- muted-tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/muted-tests.yml b/muted-tests.yml index c0e3c217abce2..98c02314fd061 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -186,6 +186,12 @@ tests: - class: "org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT" method: "test {scoring.*}" issue: https://github.com/elastic/elasticsearch/issues/117641 +- class: "org.elasticsearch.xpack.esql.qa.mixed.MultilusterEsqlSpecIT" + method: "test {scoring.*}" + issue: https://github.com/elastic/elasticsearch/issues/118460 +- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT" + method: "test {scoring.*}" + issue: https://github.com/elastic/elasticsearch/issues/118460 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT method: test {scoring.QstrWithFieldAndScoringSortedEval} issue: https://github.com/elastic/elasticsearch/issues/117751 From a123ca55de949160e07531e22ee5456c68766703 Mon Sep 17 00:00:00 2001 From: Tommaso Teofili Date: Wed, 11 Dec 2024 15:20:14 +0100 Subject: [PATCH 6/6] Update docs/changelog/118435.yaml --- docs/changelog/118435.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog/118435.yaml b/docs/changelog/118435.yaml index 200bdf6157aa5..8bccbeb54698d 100644 --- a/docs/changelog/118435.yaml +++ b/docs/changelog/118435.yaml @@ -2,4 +2,5 @@ pr: 118435 summary: '`_score` should not be a reserved attribute in ES|QL' area: ES|QL type: enhancement -issues: [] +issues: + - 118460