From ee805080f6844ad6c030f4719d95b2ac9df65080 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Wed, 3 Jul 2024 16:19:12 +0200 Subject: [PATCH 1/4] Testing, still using Java 11 --- README.md | 5 + pom.xml | 7 +- .../config/parser/RequestConfigHandler.groovy | 25 +--- .../oak/index/config/parser/XmlConfig.groovy | 116 ------------------ .../oak/state/export/CndExporter.groovy | 61 --------- .../oak/state/export/JsonExporter.groovy | 36 ------ .../oak/state/export/NodeStateExporter.groovy | 23 ---- .../oak/state/export/XmlExporter.groovy | 85 ------------- .../oak/state/export/CndExporter.java | 61 +++++++++ .../oak/state/export/JsonExporter.java | 89 ++++++++++++++ .../oak/state/export/NodeStateExporter.java | 28 +++++ .../index/config/parser/XmlConfigTest.groovy | 104 ---------------- .../oak/state/export/ExporterTest.groovy | 43 ------- .../oak/state/export/ExporterTest.java | 43 +++++++ 14 files changed, 233 insertions(+), 493 deletions(-) delete mode 100644 src/main/groovy/com/chetanmeh/oak/index/config/parser/XmlConfig.groovy delete mode 100644 src/main/groovy/com/chetanmeh/oak/state/export/CndExporter.groovy delete mode 100644 src/main/groovy/com/chetanmeh/oak/state/export/JsonExporter.groovy delete mode 100644 src/main/groovy/com/chetanmeh/oak/state/export/NodeStateExporter.groovy delete mode 100644 src/main/groovy/com/chetanmeh/oak/state/export/XmlExporter.groovy create mode 100644 src/main/java/com/chetanmeh/oak/state/export/CndExporter.java create mode 100644 src/main/java/com/chetanmeh/oak/state/export/JsonExporter.java create mode 100644 src/main/java/com/chetanmeh/oak/state/export/NodeStateExporter.java delete mode 100644 src/test/groovy/com/chetanmeh/oak/index/config/parser/XmlConfigTest.groovy delete mode 100644 src/test/groovy/com/chetanmeh/oak/state/export/ExporterTest.groovy create mode 100644 src/test/java/com/chetanmeh/oak/state/export/ExporterTest.java diff --git a/README.md b/README.md index 97ca162..0646d9e 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,8 @@ Web app hosting few utilities for [Apache Jackrabbit Oak][1] using Google Appeng In action: [oakutils.appspot.com](http://oakutils.appspot.com) [1]: http://jackrabbit.apache.org/oak + +## Deploy + + jdk 11 + mvn appengine:deploy diff --git a/pom.xml b/pom.xml index d85250a..ef6054f 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 war - 1.0.1-SNAPSHOT + 1.0.3-SNAPSHOT com.chetanmeh.appengine oakutils-war @@ -25,6 +25,11 @@ + + org.apache.jackrabbit + oak-commons + ${oak.version} + org.apache.jackrabbit oak-core diff --git a/src/main/groovy/com/chetanmeh/oak/index/config/parser/RequestConfigHandler.groovy b/src/main/groovy/com/chetanmeh/oak/index/config/parser/RequestConfigHandler.groovy index b811279..2dc0e3f 100644 --- a/src/main/groovy/com/chetanmeh/oak/index/config/parser/RequestConfigHandler.groovy +++ b/src/main/groovy/com/chetanmeh/oak/index/config/parser/RequestConfigHandler.groovy @@ -28,34 +28,11 @@ class RequestConfigHandler { } static Indexes getIndexInfo(String fileName, InputStream is){ - if (fileName.endsWith("xml")){ - def text = Streams.asString(is, 'utf-8') - return new XmlConfig(text).parse() - } else if (fileName.endsWith("zip")){ - Indexes indexes = new Indexes() - readFromContentPackage(is, indexes) - indexes.afterPropertiesSet() - return indexes - } else if (fileName.endsWith("json")){ + if (fileName.endsWith("json")){ def text = Streams.asString(is, 'utf-8') return new JsonConfig(text).parse() } return null } - private static void readFromContentPackage(InputStream is, Indexes indexes) { - ZipInputStream zis = new ZipInputStream(is) - ZipEntry entry - byte[] buffer = new byte[2048]; - while ((entry = zis.getNextEntry())) { - if (entry.name.endsWith('/_oak_index/.content.xml')) { - ByteArrayOutputStream baos = new ByteArrayOutputStream() - int len = 0 - while ((len = zis.read(buffer)) > 0) { - baos.write(buffer, 0, len); - } - new XmlConfig(baos.toString('utf-8')).parseTo(indexes) - } - } - } } diff --git a/src/main/groovy/com/chetanmeh/oak/index/config/parser/XmlConfig.groovy b/src/main/groovy/com/chetanmeh/oak/index/config/parser/XmlConfig.groovy deleted file mode 100644 index d1df32a..0000000 --- a/src/main/groovy/com/chetanmeh/oak/index/config/parser/XmlConfig.groovy +++ /dev/null @@ -1,116 +0,0 @@ -package com.chetanmeh.oak.index.config.parser - -import groovy.util.slurpersupport.GPathResult - - -class XmlConfig { - final String xml - - public XmlConfig(String xml){ - this.xml = xml - } - - public Indexes parse(){ - Indexes indexes = new Indexes() - parseTo(indexes) - indexes.afterPropertiesSet() - return indexes - } - - public void parseTo(Indexes indexes){ - def content = new XmlSlurper(false, false).parseText(xml) - content.children().each {idx -> - switch (idx.@type){ - case 'property' : - indexes.propertyIndexes << parsePropertyIndexDefn(idx) - break - case 'lucene' : - indexes.luceneIndexes << parseLuceneIndexDefn(idx) - break - case 'disabled' : - indexes.disabledIndexes << idx.name() - break - } - } - } - - LuceneIndex parseLuceneIndexDefn(def idx) { - LuceneIndex li = new LuceneIndex() - if (hasAttr(idx, 'compatVersion')) { - li.compatVersion = parseJcrValue(idx.@compatVersion.text()) as int - } - li.path = "/oak:index/${idx.name()}" - li.evaluatePathRestrictions = parseJcrValue(idx.@evaluatePathRestrictions.text()) as boolean - li.includedPaths = parseJcrArray(idx.@includedPaths.text()) - li.excludedPaths = parseJcrArray(idx.@excludedPaths.text()) - - idx.indexRules.children().each {GPathResult irConf -> - IndexRule ir = new IndexRule() - ir.type = irConf.name() - irConf.properties.children().each{p -> - PropertyDefinition pd = new PropertyDefinition() - pd.name = p.@name.text() - pd.ordered = toBool(p, 'ordered', false) - pd.propertyIndex = toBool(p, 'propertyIndex', false) - pd.isRegexp = toBool(p, 'isRegexp', false) - pd.nullCheckEnabled = toBool(p, 'nullCheckEnabled', false) - pd.index = toBool(p, 'index', true) - pd.useInExcerpt = toBool(p, 'useInExcerpt', false) - pd.nodeScopeIndex = toBool(p, 'nodeScopeIndex', false) - pd.useInSuggest = toBool(p, 'useInSuggest', false) - pd.useInSpellcheck = toBool(p, 'useInSpellcheck', false) - pd.facets = toBool(p, 'facets', false) - - ir.properties << pd - } - - li.rules << ir - } - return li - } - - PropertyIndex parsePropertyIndexDefn(def idx) { - PropertyIndex pi = new PropertyIndex() - pi.path = "/oak:index/${idx.name()}" - pi.declaringNodeTypes = parseJcrArray(idx.@declaringNodeTypes.text()) - pi.propertyNames = parseJcrArray(idx.@propertyNames.text()) - pi.unique = toBool(idx, 'unique', false) - pi.includedPaths = parseJcrArray(idx.@includedPaths.text()) - pi.excludedPaths = parseJcrArray(idx.@excludedPaths.text()) - return pi - } - - private static boolean toBool(GPathResult xml, String attrName, boolean defaultValue){ - def prop = xml['@' + attrName] - if (prop && !prop.isEmpty()){ - return parseJcrValue(prop.text()).toBoolean() - } - return defaultValue - } - - private static boolean hasAttr(GPathResult xml, String attrName){ - def prop = xml['@' + attrName] - if (prop && !prop.isEmpty()){ - return true - } - return false - } - - static def parseJcrValue(String value){ - if (!value){ - return null - } - if (value.contains('}')) { - return value.substring(value.indexOf('}') + 1) - } - return value - } - - static def parseJcrArray(String value){ - if (value.endsWith("]")){ - String csv = value.substring(value.indexOf('[') + 1, value.indexOf(']')) - return csv.tokenize(',') - } - return [] - } -} diff --git a/src/main/groovy/com/chetanmeh/oak/state/export/CndExporter.groovy b/src/main/groovy/com/chetanmeh/oak/state/export/CndExporter.groovy deleted file mode 100644 index 3688f1a..0000000 --- a/src/main/groovy/com/chetanmeh/oak/state/export/CndExporter.groovy +++ /dev/null @@ -1,61 +0,0 @@ -package com.chetanmeh.oak.state.export - -import com.google.appengine.labs.repackaged.com.google.common.base.Strings -import org.apache.jackrabbit.oak.api.PropertyState -import org.apache.jackrabbit.oak.api.Type -import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry -import org.apache.jackrabbit.oak.spi.state.NodeState - - -/** - * Exports the NodeState in CND like format which provides a compact view - * - * 
- *     /oak:index/assetType
- *      - jcr:primaryType = "oak:QueryIndexDefinition"
- *      - compatVersion = 2
- *      - type = "lucene"
- *      - async = "async"
- *      + indexRules
- *          - jcr:primaryType = "nt:unstructured"
- *           + nt:base
- *               + properties
- *                   - jcr:primaryType = "nt:unstructured"
- *                   + assetType
- *                       - propertyIndex = true
- *                       - name = "assetType"
- *
- */ -class CndExporter { - - String toCNDFormat(NodeState state){ - def result = new StringBuilder() - copyNode(state, result, 0) - return result.toString() - } - - private static void copyNode(NodeState state, StringBuilder buffer, int depth) { - copyProperties(state, buffer, depth) - state.childNodeEntries.each { ChildNodeEntry cne -> - String primaryType = cne.nodeState.getName("jcr:primaryType") - String typeText = primaryType != 'nt:unstructured' ? "($primaryType)" : '' - buffer << Strings.repeat(" ", depth + 1) + " + ${cne.name} $typeText\n" - copyNode(cne.nodeState, buffer, depth + 1) - } - } - - private static void copyProperties(NodeState state, StringBuilder buffer, int depth) { - state.properties.each { PropertyState ps -> - String value = ps.getValue(ps.getType()).toString() - if (ps.name == 'jcr:primaryType' && value == 'nt:unstructured'){ - return - } - - if (ps.type == Type.STRING){ - value = "\"$value\"" - } - - buffer << Strings.repeat(" ", depth + 1) + " - ${ps.name} = $value\n" - } - } -} diff --git a/src/main/groovy/com/chetanmeh/oak/state/export/JsonExporter.groovy b/src/main/groovy/com/chetanmeh/oak/state/export/JsonExporter.groovy deleted file mode 100644 index b0bb861..0000000 --- a/src/main/groovy/com/chetanmeh/oak/state/export/JsonExporter.groovy +++ /dev/null @@ -1,36 +0,0 @@ -package com.chetanmeh.oak.state.export - -import groovy.json.JsonOutput -import org.apache.jackrabbit.oak.api.PropertyState -import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry -import org.apache.jackrabbit.oak.spi.state.NodeState - - -class JsonExporter { - - public String toJson(NodeState state){ - return JsonOutput.prettyPrint(JsonOutput.toJson(toMap(state))) - } - - public Map toMap(NodeState state){ - def result = [:] - return copyNode(state, result) - } - - private static Map copyNode(NodeState state, Map result){ - copyProperties(state, result) - state.childNodeEntries.each {ChildNodeEntry cne -> - def nodeMap = [:] - result.put(cne.name, nodeMap) - copyNode(cne.nodeState, nodeMap) - } - return result - } - - private static Map copyProperties(NodeState state, Map map) { - state.properties.each {PropertyState ps -> - map.put(ps.name, ps.getValue(ps.getType())) - } - return map - } -} diff --git a/src/main/groovy/com/chetanmeh/oak/state/export/NodeStateExporter.groovy b/src/main/groovy/com/chetanmeh/oak/state/export/NodeStateExporter.groovy deleted file mode 100644 index b30dcba..0000000 --- a/src/main/groovy/com/chetanmeh/oak/state/export/NodeStateExporter.groovy +++ /dev/null @@ -1,23 +0,0 @@ -package com.chetanmeh.oak.state.export - -import org.apache.jackrabbit.oak.spi.state.NodeState - - -class NodeStateExporter { - - static String toJson(NodeState state){ - return new JsonExporter().toJson(state) - } - - static Map toMap(NodeState state){ - return new JsonExporter().toMap(state) - } - - static String toCND(NodeState state){ - return new CndExporter().toCNDFormat(state) - } - - static String toXml(NodeState state){ - return new XmlExporter().toXml(state) - } -} diff --git a/src/main/groovy/com/chetanmeh/oak/state/export/XmlExporter.groovy b/src/main/groovy/com/chetanmeh/oak/state/export/XmlExporter.groovy deleted file mode 100644 index b61920a..0000000 --- a/src/main/groovy/com/chetanmeh/oak/state/export/XmlExporter.groovy +++ /dev/null @@ -1,85 +0,0 @@ -package com.chetanmeh.oak.state.export - -import groovy.xml.StreamingMarkupBuilder -import groovy.xml.XmlUtil -import org.apache.jackrabbit.oak.api.PropertyState -import org.apache.jackrabbit.oak.api.Type -import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry -import org.apache.jackrabbit.oak.spi.state.NodeState - -import javax.jcr.PropertyType - - -class XmlExporter { - static final def NAMESPACES = [ - oak : "http://jackrabbit.apache.org/oak/ns/1.0", - xmpMM : "http://ns.adobe.com/xap/1.0/mm/", - dc : "http://purl.org/dc/elements/1.1/", - slingevent : "http://sling.apache.org/jcr/event/1.0", - sling : "http://sling.apache.org/jcr/sling/1.0", - granite : "http://www.adobe.com/jcr/granite/1.0", - dam : "http://www.day.com/dam/1.0", - cq : "http://www.day.com/jcr/cq/1.0", - jcr : "http://www.jcp.org/jcr/1.0", - mix : "http://www.jcp.org/jcr/mix/1.0", - nt : "http://www.jcp.org/jcr/nt/1.0", - rep : "internal", - ] - - String toXml(NodeState state){ - def nsMap = collectNamespaces(state) - return XmlUtil.serialize(new StreamingMarkupBuilder().with {builder -> - builder.bind { binding -> - mkp.xmlDeclaration() - nsMap.each {ns, namespace -> - mkp.declareNamespace((ns) : namespace) - } - 'jcr:root' { - process(binding, state, 'myIndex') - } - } - }) - } - - private Map collectNamespaces(NodeState state){ - def nsMap = [:] - nsMap['jcr'] = NAMESPACES['jcr'] - collectNamespaces("", state, nsMap) - return nsMap - } - - private static void collectNamespaces(String name, NodeState state, def nsMap) { - if (name.contains(':')){ - String nsPrefix = name.substring(0, name.indexOf(':')) - String namespace = NAMESPACES[nsPrefix] - if (!namespace){ - namespace = "internal" - } - nsMap.put(nsPrefix, namespace) - } - state.childNodeEntries.each{collectNamespaces(it.name, it.nodeState, nsMap)} - } - - private def process = { binding, NodeState state, String name -> - binding."$name" (propertiesMap(state)) { - state.childNodeEntries.each { ChildNodeEntry cne -> - process (binding, cne.nodeState, cne.name) - } - } - } - - private static Map propertiesMap(NodeState state) { - def map = [:] - state.properties.each { PropertyState ps -> - String value - if (ps.type == Type.STRING || ps.name == 'jcr:primaryType'){ - value = ps.getValue(Type.STRING) - } else { - String typeName = PropertyType.nameFromValue(ps.type.tag()) - value = "{$typeName}${ps.getValue(ps.getType())}" - } - map.put(ps.name, value) - } - return map - } -} diff --git a/src/main/java/com/chetanmeh/oak/state/export/CndExporter.java b/src/main/java/com/chetanmeh/oak/state/export/CndExporter.java new file mode 100644 index 0000000..1453bea --- /dev/null +++ b/src/main/java/com/chetanmeh/oak/state/export/CndExporter.java @@ -0,0 +1,61 @@ +package com.chetanmeh.oak.state.export; + +import com.google.common.base.Strings; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +/** + * Exports the NodeState in CND like format which provides a compact view + * + * 
+ *     /oak:index/assetType
+ *      - jcr:primaryType = "oak:QueryIndexDefinition"
+ *      - compatVersion = 2
+ *      - type = "lucene"
+ *      - async = "async"
+ *      + indexRules
+ *          - jcr:primaryType = "nt:unstructured"
+ *           + nt:base
+ *               + properties
+ *                   - jcr:primaryType = "nt:unstructured"
+ *                   + assetType
+ *                       - propertyIndex = true
+ *                       - name = "assetType"
+ *
+ */ +public class CndExporter { + + public String toCNDFormat(NodeState state) { + StringBuilder result = new StringBuilder(); + copyNode(state, result, 0); + return result.toString(); + } + + private static void copyNode(NodeState state, StringBuilder buffer, int depth) { + copyProperties(state, buffer, depth); + for (ChildNodeEntry cne : state.getChildNodeEntries()) { + String primaryType = cne.getNodeState().getName("jcr:primaryType"); + String typeText = !primaryType.equals("nt:unstructured") ? "(" + primaryType + ")" : ""; + buffer.append(Strings.repeat(" ", depth + 1)).append(" + ").append(cne.getName()).append(" ").append(typeText).append("\n"); + copyNode(cne.getNodeState(), buffer, depth + 1); + } + } + + private static void copyProperties(NodeState state, StringBuilder buffer, int depth) { + for (PropertyState ps : state.getProperties()) { + String value = ps.getValue(ps.getType()).toString(); + if (ps.getName().equals("jcr:primaryType") && value.equals("nt:unstructured")) { + continue; + } + + if (ps.getType() == Type.STRING) { + value = "\"" + value + "\""; + } + + buffer.append(Strings.repeat(" ", depth + 1)).append(" - ").append(ps.getName()).append(" = ").append(value).append("\n"); + } + } +} + diff --git a/src/main/java/com/chetanmeh/oak/state/export/JsonExporter.java b/src/main/java/com/chetanmeh/oak/state/export/JsonExporter.java new file mode 100644 index 0000000..1eb31af --- /dev/null +++ b/src/main/java/com/chetanmeh/oak/state/export/JsonExporter.java @@ -0,0 +1,89 @@ +package com.chetanmeh.oak.state.export; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.commons.json.JsopBuilder; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +public class JsonExporter { + + public String toJson(NodeState state) { + JsopBuilder builder = new JsopBuilder(); + Map map = toMap(state); + write(map, builder); + return JsopBuilder.prettyPrint(builder.toString()); + } + + private static void write(Map map, JsopBuilder target) { + target.object(); + ArrayList keys = new ArrayList<>(map.keySet()); + Collections.sort(keys); + for (String key : keys) { + Object value = map.get(key); + if (value == null || !(value instanceof Map)) { + target.key(key); + writeObject(value, target); + } + } + for (String key : keys) { + Object value = map.get(key); + if (value instanceof Map) { + target.key(key); + writeObject(value, target); + } + } + target.endObject(); + } + + @SuppressWarnings("unchecked") + private static void writeObject(Object value, JsopBuilder target) { + if (value == null) { + target.value(null); + } else if (value instanceof Boolean) { + target.value((Boolean) value); + } else if (value instanceof Integer) { + target.value((Integer) value); + } else if (value instanceof Long) { + target.value((Long) value); + } else if (value instanceof Map) { + write((Map) value, target); + } else if (value instanceof List) { + List list = (List) value; + target.array(); + for (Object o : list) { + writeObject(o, target); + } + target.endArray(); + } else { + target.value(value.toString()); + } + } + + public Map toMap(NodeState state) { + Map result = new HashMap<>(); + return copyNode(state, result); + } + + private static Map copyNode(NodeState state, Map result) { + copyProperties(state, result); + for (ChildNodeEntry cne : state.getChildNodeEntries()) { + Map nodeMap = new HashMap<>(); + result.put(cne.getName(), nodeMap); + copyNode(cne.getNodeState(), nodeMap); + } + return result; + } + + private static Map copyProperties(NodeState state, Map map) { + for (PropertyState ps : state.getProperties()) { + map.put(ps.getName(), ps.getValue(ps.getType())); + } + return map; + } +} diff --git a/src/main/java/com/chetanmeh/oak/state/export/NodeStateExporter.java b/src/main/java/com/chetanmeh/oak/state/export/NodeStateExporter.java new file mode 100644 index 0000000..a0fe0b4 --- /dev/null +++ b/src/main/java/com/chetanmeh/oak/state/export/NodeStateExporter.java @@ -0,0 +1,28 @@ +package com.chetanmeh.oak.state.export; + +import java.util.Map; + +import org.apache.jackrabbit.oak.spi.state.NodeState; + +class NodeStateExporter { + + static String toJson(NodeState state){ + return new JsonExporter().toJson(state); + } + + static Map toMap(NodeState state){ + return new JsonExporter().toMap(state); + } + + static String toCND(NodeState state){ + return new CndExporter().toCNDFormat(state); + } + + static String toXml(NodeState state){ + // TODO + return ""; + +// return new XmlExporter().toXml(state); + } +} + diff --git a/src/test/groovy/com/chetanmeh/oak/index/config/parser/XmlConfigTest.groovy b/src/test/groovy/com/chetanmeh/oak/index/config/parser/XmlConfigTest.groovy deleted file mode 100644 index 9fed396..0000000 --- a/src/test/groovy/com/chetanmeh/oak/index/config/parser/XmlConfigTest.groovy +++ /dev/null @@ -1,104 +0,0 @@ -package com.chetanmeh.oak.index.config.parser - -import org.junit.Test - - -class XmlConfigTest { - - - @Test - public void propertyDefinition() throws Exception{ - def xml = createXml('''''') - - def indexes = new XmlConfig(xml).parse() - assert indexes.propertyIndexes.size() == 1 - PropertyIndex pi = indexes.propertyIndexes[0] - assert pi.path == '/oak:index/acPrincipalName' - assert pi.@propertyNames == ['rep:principalName'] - assert pi.declaringNodeTypesList as Set == ['rep:DenyACE','rep:GrantACE','rep:ACE'] as Set - assert pi.unique == false - } - - @Test - public void luceneDefinition() throws Exception{ - def xml = createXml(''' - - - - - - - - - - ''') - def indexes = new XmlConfig(xml).parse() - assert indexes.luceneIndexes.size() == 1 - LuceneIndex li = indexes.luceneIndexes[0] - assert li.path == '/oak:index/slingMapping' - assert li.compatVersion == 2 - assert li.includedPaths == ['/etc/map'] - assert li.excludedPaths == [] - assert li.rules.size() == 1 - IndexRule r = li.rules[0] - assert r.type == 'sling:Mapping' - assert r.properties.size() == 3 - PropertyDefinition pd = r.getPropDefn('sling:internalRedirect') - assert pd.propertyIndex - assert !pd.ordered - assert pd.index - - pd = r.getPropDefn('sling:redirect') - assert pd.ordered - - pd = r.getPropDefn("ontime") - assert !pd.index - } - - @Test - public void simpleValue() throws Exception{ - assert 'false' == XmlConfig.parseJcrValue('{Boolean}false') - assert '1' == XmlConfig.parseJcrValue('{Long}1') - } - - @Test - public void listValue() throws Exception{ - assert ['containeeInstanceId'] == XmlConfig.parseJcrArray('{Name}[containeeInstanceId]') - assert ['rep:DenyACE','rep:GrantACE','rep:ACE'] == XmlConfig.parseJcrArray('{Name}[rep:DenyACE,rep:GrantACE,rep:ACE]') - } - - def createXml(String fragment){ - "$xmlHeader $fragment $xmlFooter" - } - - static final def xmlHeader = ''' -''' - - static final def xmlFooter = '''''' - -} diff --git a/src/test/groovy/com/chetanmeh/oak/state/export/ExporterTest.groovy b/src/test/groovy/com/chetanmeh/oak/state/export/ExporterTest.groovy deleted file mode 100644 index 4634482..0000000 --- a/src/test/groovy/com/chetanmeh/oak/state/export/ExporterTest.groovy +++ /dev/null @@ -1,43 +0,0 @@ -package com.chetanmeh.oak.state.export - -import com.chetanmeh.oak.index.config.IndexDefinitionBuilder -import org.apache.jackrabbit.oak.spi.state.NodeState -import org.junit.Test - -class ExporterTest { - - @Test - public void jsonOutput() throws Exception{ - println NodeStateExporter.toJson(createState()) - } - - @Test - public void cnd() throws Exception{ - println NodeStateExporter.toCND(createState()) - } - - @Test - public void xml() throws Exception{ - println NodeStateExporter.toXml(createState()) - } - - private static NodeState createState(){ - IndexDefinitionBuilder builder = new IndexDefinitionBuilder() - builder.indexRule("nt:base") - .property("foo") - .ordered() - .enclosingRule() - .property("bar") - .analyzed() - .propertyIndex() - .enclosingRule() - .property("baz") - .propertyIndex() - - builder.includedPaths("/content", "/etc") - builder.aggregateRule('cq:Page').include('jcr:content').relativeNode() - builder.aggregateRule('dam:Asset', '*', '*/*') - - return builder.build() - } -} diff --git a/src/test/java/com/chetanmeh/oak/state/export/ExporterTest.java b/src/test/java/com/chetanmeh/oak/state/export/ExporterTest.java new file mode 100644 index 0000000..f84a884 --- /dev/null +++ b/src/test/java/com/chetanmeh/oak/state/export/ExporterTest.java @@ -0,0 +1,43 @@ +package com.chetanmeh.oak.state.export; + +import java.lang.*; +import java.io.*; +import java.net.*; +import java.util.*; + +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.junit.Test; + +import com.chetanmeh.oak.index.config.IndexDefinitionBuilder; + +import groovy.lang.*; +import groovy.util.*; + +public class ExporterTest { + @Test + public void jsonOutput() throws Exception { + System.out.println(NodeStateExporter.toJson(createState())); + } + + @Test + public void cnd() throws Exception { + System.out.println(NodeStateExporter.toCND(createState())); + } + + @Test + public void xml() throws Exception { + System.out.println(NodeStateExporter.toXml(createState())); + } + + private static NodeState createState() { + IndexDefinitionBuilder builder = new IndexDefinitionBuilder(); + builder.indexRule("nt:base").property("foo").ordered().enclosingRule().property("bar").analyzed() + .propertyIndex().enclosingRule().property("baz").propertyIndex(); + + builder.includedPaths("/content", "/etc"); + builder.aggregateRule("cq:Page").include("jcr:content").relativeNode(); + builder.aggregateRule("dam:Asset", "*", "*/*"); + + return builder.build(); + } +} From b0daeb8d70f1fabddc41c6b31bbc59e96a1e91dd Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Wed, 3 Jul 2024 17:34:53 +0200 Subject: [PATCH 2/4] Upgrade to Java 11, as needed --- README.md | 4 +++- src/main/webapp/WEB-INF/appengine-web.xml | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0646d9e..9013418 100644 --- a/README.md +++ b/README.md @@ -10,4 +10,6 @@ In action: [oakutils.appspot.com](http://oakutils.appspot.com) ## Deploy jdk 11 - mvn appengine:deploy + open https://cloud.google.com/sdk/docs/install-sdk + gcloud auth login + mvn package appengine:deploy -Dapp.deploy.projectId=oakutils -Dapp.deploy.version=20240703t163000 diff --git a/src/main/webapp/WEB-INF/appengine-web.xml b/src/main/webapp/WEB-INF/appengine-web.xml index 0c64cd0..ea55b0b 100644 --- a/src/main/webapp/WEB-INF/appengine-web.xml +++ b/src/main/webapp/WEB-INF/appengine-web.xml @@ -1,7 +1,7 @@ oakutils - java8 + java11 true true From 6b8edf61bfa0dd3c8a4d62bd6c436b4d4314c630 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Fri, 5 Jul 2024 11:45:28 +0200 Subject: [PATCH 3/4] Add warnings for not-that-great queries. --- README.md | 13 +- .../index/config/IndexDefinitionBuilder.java | 23 ++- .../generator/IndexConfigGenerator.java | 11 ++ src/main/webapp/codemirror/mode/sql.js | 2 +- .../IndexConfigGeneratorJavaTest.java | 144 ++++++++++++++++++ .../oak/state/export/ExporterTest.java | 8 - 6 files changed, 187 insertions(+), 14 deletions(-) create mode 100644 src/test/java/com/chetanmeh/oak/index/config/generator/IndexConfigGeneratorJavaTest.java diff --git a/README.md b/README.md index 9013418..bcaf1c7 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,12 @@ In action: [oakutils.appspot.com](http://oakutils.appspot.com) ## Deploy - jdk 11 - open https://cloud.google.com/sdk/docs/install-sdk - gcloud auth login - mvn package appengine:deploy -Dapp.deploy.projectId=oakutils -Dapp.deploy.version=20240703t163000 + jdk 11 + open https://cloud.google.com/sdk/docs/install-sdk + gcloud auth login + mvn package appengine:deploy -Dapp.deploy.projectId=oakutils -Dapp.deploy.version=20240703t163000 + +## Local Development + + Consider adding sufficient unit tests, and using + http://gaelyk.appspot.com/tutorial/run-deploy#run \ No newline at end of file diff --git a/src/main/java/com/chetanmeh/oak/index/config/IndexDefinitionBuilder.java b/src/main/java/com/chetanmeh/oak/index/config/IndexDefinitionBuilder.java index 86e8e58..9c720bb 100644 --- a/src/main/java/com/chetanmeh/oak/index/config/IndexDefinitionBuilder.java +++ b/src/main/java/com/chetanmeh/oak/index/config/IndexDefinitionBuilder.java @@ -23,6 +23,11 @@ import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; public class IndexDefinitionBuilder { + + private static final String WARN_MISSING_PATH_RESTRICTION = "warningPathRestrictionMissing"; + private static final String WARN_MISSING_INDEX_TAG = "warningTagMissing"; + private static final String WARN_COMMON_NODE_TYPE = "warningCommonNodeType"; + private final NodeBuilder builder = EMPTY_NODE.builder(); private final Map rules = Maps.newHashMap(); private final Map aggRules = Maps.newHashMap(); @@ -33,6 +38,9 @@ public IndexDefinitionBuilder(){ builder.setProperty(LuceneIndexConstants.COMPAT_MODE, 2); builder.setProperty("async", "async"); builder.setProperty("type", "lucene"); + builder.setProperty(WARN_MISSING_PATH_RESTRICTION, "Consider adding a path restriction to the query. The query currently does not have a path restriction, that means the index will need to cover all nodes, including for example the version store. This will slow down index generation, and can increase the index size."); + builder.setProperty(WARN_MISSING_INDEX_TAG, "Consider adding a tag to the query, via 'option(index tag abc)'. See also https://jackrabbit.apache.org/oak/docs/query/query-engine.html#query-option-index-tag . The query currently does not have a tag, which can result in the wrong index to be used. Also, it prevents to add 'selectionPolicy' = 'tag' to the index definition, meaning that other, unrelated queries might use this index by mistake."); + builder.setProperty(WARN_COMMON_NODE_TYPE, "Consider adding a more restrictive node type condition. Indexes on 'nt:base' or 'nt:unstructured' cover a lot of nodes, which increases the index size, and slows down query execution. Use a primary or mixin node type if possible."); builder.setProperty(LuceneIndexConstants.EVALUATE_PATH_RESTRICTION, true); builder.setProperty(JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", Type.NAME); indexRule = createChild(builder, LuceneIndexConstants.INDEX_RULES); @@ -45,11 +53,20 @@ public IndexDefinitionBuilder evaluatePathRestrictions(){ public IndexDefinitionBuilder includedPaths(String ... paths){ builder.setProperty(createProperty(PathFilter.PROP_INCLUDED_PATHS, Arrays.asList(paths), Type.STRINGS)); + builder.removeProperty(WARN_MISSING_PATH_RESTRICTION); return this; } public IndexDefinitionBuilder queryPaths(String ... paths){ builder.setProperty(createProperty(IndexConstants.QUERY_PATHS, Arrays.asList(paths), Type.STRINGS)); + builder.removeProperty(WARN_MISSING_PATH_RESTRICTION); + return this; + } + + public IndexDefinitionBuilder indexTag(String tag) { + builder.setProperty(createProperty(IndexConstants.INDEX_TAGS, Arrays.asList(tag), Type.STRINGS)); + builder.setProperty("selectionPolicy", "tag"); + builder.removeProperty(WARN_MISSING_INDEX_TAG); return this; } @@ -64,7 +81,10 @@ public NodeState build(){ //~--------------------------------------< IndexRule > - public IndexRule indexRule(String type){ + public IndexRule indexRule(String type) { + if (!"nt:unstructured".equals(type) && !"nt:base".equals(type)) { + builder.removeProperty(WARN_COMMON_NODE_TYPE); + } IndexRule rule = rules.get(type); if (rule == null){ rule = new IndexRule(createChild(indexRule, type), type); @@ -274,4 +294,5 @@ static String getSafePropName(String relativePropName) { propName = propName.replaceAll("\\W", ""); return propName; } + } diff --git a/src/main/java/com/chetanmeh/oak/index/config/generator/IndexConfigGenerator.java b/src/main/java/com/chetanmeh/oak/index/config/generator/IndexConfigGenerator.java index d629b76..8574a56 100644 --- a/src/main/java/com/chetanmeh/oak/index/config/generator/IndexConfigGenerator.java +++ b/src/main/java/com/chetanmeh/oak/index/config/generator/IndexConfigGenerator.java @@ -15,6 +15,7 @@ import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.core.ImmutableRoot; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; import org.apache.jackrabbit.oak.query.ExecutionContext; import org.apache.jackrabbit.oak.query.QueryEngineImpl; import org.apache.jackrabbit.oak.query.QueryEngineSettings; @@ -98,12 +99,22 @@ public NodeState getIndexConfig() { private void processFilter(Filter filter, List sortOrder) { addPathRestrictions(filter); + addIndexTag(filter); IndexRule rule = processNodeTypeConstraint(filter); processFulltextConstraints(filter, rule); processPropertyRestrictions(filter, rule); processSortConditions(sortOrder, rule); processPureNodeTypeConstraints(filter, rule); } + + private void addIndexTag(Filter filter) { + PropertyRestriction indexTag = filter.getPropertyRestriction(IndexConstants.INDEX_TAG_OPTION); + if (indexTag != null && indexTag.first != null) { + // index tag specified + String tag = indexTag.first.getValue(Type.STRING); + builder.indexTag(tag); + } + } private void addPathRestrictions(Filter filter) { if (!filter.getPath().isEmpty() && !"/".equals(filter.getPath())) { diff --git a/src/main/webapp/codemirror/mode/sql.js b/src/main/webapp/codemirror/mode/sql.js index e91c706..824a341 100644 --- a/src/main/webapp/codemirror/mode/sql.js +++ b/src/main/webapp/codemirror/mode/sql.js @@ -257,7 +257,7 @@ CodeMirror.defineMode("sql", function(config, parserConfig) { } // these keywords are used by all SQL dialects (however, a mode can still overwrite it) - var sqlKeywords = "alter and as asc between by count create delete desc distinct drop from group having in insert into is join like not on or order select set table union update values where limit "; + var sqlKeywords = "alter and as asc between by count create delete desc distinct drop from group having in insert into is join like not on or order select set table union update values where limit option index tag"; // turn a space-separated list into an array function set(str) { diff --git a/src/test/java/com/chetanmeh/oak/index/config/generator/IndexConfigGeneratorJavaTest.java b/src/test/java/com/chetanmeh/oak/index/config/generator/IndexConfigGeneratorJavaTest.java new file mode 100644 index 0000000..1d4351b --- /dev/null +++ b/src/test/java/com/chetanmeh/oak/index/config/generator/IndexConfigGeneratorJavaTest.java @@ -0,0 +1,144 @@ +package com.chetanmeh.oak.index.config.generator; + +import static org.junit.Assert.assertEquals; + +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.junit.Test; + +import com.chetanmeh.oak.state.export.JsonExporter; + +public class IndexConfigGeneratorJavaTest { + + @Test + public void testBadQuery() throws Exception { + NodeState ns = IndexConfigGeneratorHelper.getIndexConfig("/jcr:root//*[@a]"); + JsonExporter ex = new JsonExporter(); + String index = ex.toJson(ns); + assertEquals("{\n" + + " \"async\": \"async\",\n" + + " \"compatVersion\": 2,\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"jcr:primaryType\": \"oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"warningCommonNodeType\": \"Consider adding a more restrictive node type condition. Indexes on 'nt:base' or 'nt:unstructured' cover a lot of nodes, which increases the index size, and slows down query execution. Use a primary or mixin node type if possible.\",\n" + + " \"warningPathRestrictionMissing\": \"Consider adding a path restriction to the query. The query currently does not have a path restriction, that means the index will need to cover all nodes, including for example the version store. This will slow down index generation, and can increase the index size.\",\n" + + " \"warningTagMissing\": \"Consider adding a tag to the query, via 'option(index tag abc)'. See also https://jackrabbit.apache.org/oak/docs/query/query-engine.html#query-option-index-tag . The query currently does not have a tag, which can result in the wrong index to be used. Also, it prevents to add 'selectionPolicy' = 'tag' to the index definition, meaning that other, unrelated queries might use this index by mistake.\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"nt:base\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"a\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"name\": \"a\",\n" + + " \"notNullCheckEnabled\": true,\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", index); + } + + @Test + public void testQueryWithNodeType() throws Exception { + NodeState ns = IndexConfigGeneratorHelper.getIndexConfig("/jcr:root//element(*, dam:Asset)[@a]"); + JsonExporter ex = new JsonExporter(); + String index = ex.toJson(ns); + assertEquals("{\n" + + " \"async\": \"async\",\n" + + " \"compatVersion\": 2,\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"jcr:primaryType\": \"oak:QueryIndexDefinition\",\n" + + " \"type\": \"lucene\",\n" + + " \"warningPathRestrictionMissing\": \"Consider adding a path restriction to the query. The query currently does not have a path restriction, that means the index will need to cover all nodes, including for example the version store. This will slow down index generation, and can increase the index size.\",\n" + + " \"warningTagMissing\": \"Consider adding a tag to the query, via 'option(index tag abc)'. See also https://jackrabbit.apache.org/oak/docs/query/query-engine.html#query-option-index-tag . The query currently does not have a tag, which can result in the wrong index to be used. Also, it prevents to add 'selectionPolicy' = 'tag' to the index definition, meaning that other, unrelated queries might use this index by mistake.\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"a\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"name\": \"a\",\n" + + " \"notNullCheckEnabled\": true,\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", index); + } + + @Test + public void testQueryWithTag() throws Exception { + NodeState ns = IndexConfigGeneratorHelper.getIndexConfig( + "/jcr:root//element(*, dam:Asset)[@a]" + + "option(index tag abc)"); + JsonExporter ex = new JsonExporter(); + String index = ex.toJson(ns); + assertEquals("{\n" + + " \"async\": \"async\",\n" + + " \"compatVersion\": 2,\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"jcr:primaryType\": \"oak:QueryIndexDefinition\",\n" + + " \"selectionPolicy\": \"tag\",\n" + + " \"tags\": [\"abc\"],\n" + + " \"type\": \"lucene\",\n" + + " \"warningPathRestrictionMissing\": \"Consider adding a path restriction to the query. The query currently does not have a path restriction, that means the index will need to cover all nodes, including for example the version store. This will slow down index generation, and can increase the index size.\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"a\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"name\": \"a\",\n" + + " \"notNullCheckEnabled\": true,\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", index); + } + + @Test + public void testGoodQuery() throws Exception { + NodeState ns = IndexConfigGeneratorHelper.getIndexConfig( + "/jcr:root/content/dam//element(*, dam:Asset)[@a]" + + "option(index tag abc)"); + JsonExporter ex = new JsonExporter(); + String index = ex.toJson(ns); + assertEquals("{\n" + + " \"async\": \"async\",\n" + + " \"compatVersion\": 2,\n" + + " \"evaluatePathRestrictions\": true,\n" + + " \"includedPaths\": [\"/content/dam\"],\n" + + " \"jcr:primaryType\": \"oak:QueryIndexDefinition\",\n" + + " \"queryPaths\": [\"/content/dam\"],\n" + + " \"selectionPolicy\": \"tag\",\n" + + " \"tags\": [\"abc\"],\n" + + " \"type\": \"lucene\",\n" + + " \"indexRules\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"dam:Asset\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"properties\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"a\": {\n" + + " \"jcr:primaryType\": \"nt:unstructured\",\n" + + " \"name\": \"a\",\n" + + " \"notNullCheckEnabled\": true,\n" + + " \"propertyIndex\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}", index); + } + +} diff --git a/src/test/java/com/chetanmeh/oak/state/export/ExporterTest.java b/src/test/java/com/chetanmeh/oak/state/export/ExporterTest.java index f84a884..c8aac4d 100644 --- a/src/test/java/com/chetanmeh/oak/state/export/ExporterTest.java +++ b/src/test/java/com/chetanmeh/oak/state/export/ExporterTest.java @@ -1,18 +1,10 @@ package com.chetanmeh.oak.state.export; -import java.lang.*; -import java.io.*; -import java.net.*; -import java.util.*; - import org.apache.jackrabbit.oak.spi.state.NodeState; import org.junit.Test; import com.chetanmeh.oak.index.config.IndexDefinitionBuilder; -import groovy.lang.*; -import groovy.util.*; - public class ExporterTest { @Test public void jsonOutput() throws Exception { From ea2f9075b1dce8306c71e7f60359108d505f8578 Mon Sep 17 00:00:00 2001 From: Thomas Mueller Date: Tue, 9 Jul 2024 11:24:36 +0200 Subject: [PATCH 4/4] Link to query cheat sheet; improve examples --- src/main/webapp/WEB-INF/gsp/generateIndex.gsp | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/main/webapp/WEB-INF/gsp/generateIndex.gsp b/src/main/webapp/WEB-INF/gsp/generateIndex.gsp index 546bd9e..c2ae8c1 100644 --- a/src/main/webapp/WEB-INF/gsp/generateIndex.gsp +++ b/src/main/webapp/WEB-INF/gsp/generateIndex.gsp @@ -1,24 +1,17 @@ <% def QUERY_DEFAULT = ''' -#Paste your queries here - -SELECT - * +# Paste your queries here: +SELECT * FROM [dam:Asset] AS a -WHERE - a.[jcr:content/metadata/status] = 'published' -ORDER BY - a.[jcr:content/metadata/jcr:lastModified] DESC +WHERE a.[jcr:content/metadata/status] = 'published' +ORDER BY a.[jcr:content/metadata/jcr:lastModified] DESC -# There can be multiple queries added here and index generated would cover all -# of them -SELECT - * +# Multiple queries are supported +SELECT * FROM [dam:Asset] -WHERE - CONTAINS([mimetype], 'text/plain') +WHERE CONTAINS([mimetype], 'text/plain') -# You can also include xpath queries +# XPath queries are also supported /jcr:root/content/dam/element(*, dam:Asset)[@valid] ''' @@ -62,7 +55,13 @@ WHERE +
+
+ Query Cheat Sheet +
Back +
+
<% if (error == null){%>