Lens 1381

lens-api/src/main/java/org/apache/lens/api/ToXMLString.java

@@ -24,13 +24,21 @@
 import java.util.Map;
 
 import javax.xml.bind.*;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlType;
+import javax.xml.namespace.QName;
 
 import org.apache.lens.api.jaxb.LensJAXBContext;
 
 public abstract class ToXMLString {
   protected static final Map<Class<?>, JAXBContext> JAXB_CONTEXTS = new HashMap<>();
 
   public static String toString(Object o) {
+    if (!(o instanceof JAXBElement) && o.getClass().getAnnotation(XmlRootElement.class) == null
+      && o.getClass().getAnnotation(XmlType.class)!= null) {
+      o = new JAXBElement(new QName("uri:lens:cube:0.1", o.getClass().getAnnotation(XmlType.class).name()),
+        o.getClass(), null, o);
+    }
     try {
       StringWriter stringWriter = new StringWriter();
       Class cl = null;

lens-api/src/main/java/org/apache/lens/api/jaxb/LensJAXBContext.java

@@ -24,6 +24,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.Reader;
 
 import javax.xml.XMLConstants;
 import javax.xml.bind.*;
@@ -114,17 +115,26 @@ public Unmarshaller getUnmarshaller() {
     return UNMARSHALLER;
   }
 
+  public static <T> T unmarshall(File file) throws JAXBException, IOException {
+    return ((JAXBElement<T>) UNMARSHALLER.unmarshal(file)).getValue();
+  }
+  public static <T> T unmarshall(InputStream inputStream) throws JAXBException, IOException {
+    return ((JAXBElement<T>) UNMARSHALLER.unmarshal(inputStream)).getValue();
+  }
+  public static <T> T unmarshall(Reader reader) throws JAXBException, IOException {
+    return ((JAXBElement<T>) UNMARSHALLER.unmarshal(reader)).getValue();
+  }
   public static <T> T unmarshallFromFile(String filename) throws JAXBException, IOException {
     File file = new File(filename);
     if (file.exists()) {
-      return ((JAXBElement<T>) UNMARSHALLER.unmarshal(file)).getValue();
+      return unmarshall(file);
     } else {
       // load from classpath
       InputStream stream = Thread.currentThread().getContextClassLoader().getResourceAsStream(filename);
       if (stream == null) {
         throw new IOException("File not found:" + filename);
       }
-      return ((JAXBElement<T>) UNMARSHALLER.unmarshal(stream)).getValue();
+      return unmarshall(stream);
     }
   }
 }

lens-api/src/main/java/org/apache/lens/api/metastore/SchemaTraverser.java

@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.api.metastore;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.util.Map;
+import java.util.function.BiConsumer;
+
+import com.google.common.collect.Maps;
+
+/*
+ * Created on 07/03/17.
+ */
+public class SchemaTraverser implements Runnable {
+  final File parent;
+  final Map<String, Class<?>> types = Maps.newLinkedHashMap();
+  private final SchemaEntityProcessor action;
+  {
+    types.put("storages", XStorage.class);
+    types.put("cubes/base", XBaseCube.class);
+    types.put("cubes/derived", XDerivedCube.class);
+    types.put("dimensions", XDimension.class);
+    types.put("facts", XFactTable.class);
+    types.put("dimtables", XDimensionTable.class);
+    types.put("dimensiontables", XDimensionTable.class);
+    types.put("dimensiontables", XDimensionTable.class);
+    types.put("segmentations", XSegmentation.class);
+  }
+  private static final FilenameFilter XML_FILTER = (dir, name) -> name.endsWith(".xml");
+
+  public interface SchemaEntityProcessor extends BiConsumer<File, Class<?>> {
+  }
+
+  public SchemaTraverser(File parent, SchemaEntityProcessor action) {
+    this.parent = parent;
+    this.action = action;
+  }
+
+  @Override
+  public void run() {
+    for (Map.Entry<String, Class<?>> entry : types.entrySet()) {
+      File f = new File(parent, entry.getKey());
+      if (f.exists()) {
+        assert f.isDirectory();
+        File[] files = f.listFiles(XML_FILTER);
+        if (files != null) {
+          for (File entityFile : files) {
+            action.accept(entityFile.getAbsoluteFile(), entry.getValue());
+          }
+        }
+      }
+    }
+  }
+}

lens-api/src/main/resources/cube-0.1.xsd

@@ -390,7 +390,7 @@
       </xs:documentation>
     </xs:annotation>
     <xs:sequence>
-      <xs:element type="x_expr_column" name="expression" maxOccurs="unbounded" minOccurs="1"/>
+      <xs:element type="x_expr_column" name="expression" maxOccurs="unbounded" minOccurs="0"/>
     </xs:sequence>
   </xs:complexType>
 
@@ -707,7 +707,7 @@
 
   <xs:complexType name="x_columns">
     <xs:sequence>
-      <xs:element name="column" type="x_column" maxOccurs="unbounded" minOccurs="1"/>
+      <xs:element name="column" type="x_column" maxOccurs="unbounded" minOccurs="0"/>
     </xs:sequence>
   </xs:complexType>
 

lens-api/src/main/resources/lens-errors.conf

@@ -326,6 +326,19 @@ lensCubeErrorsForQuery = [
     errorMsg = "Could not find queried table or chain: %s"
   }
 
+  {
+    errorCode = 3034
+    httpStatusCode = ${BAD_REQUEST}
+    errorMsg = "%s does not have any facts that can cover the requested time range : %s and  queried measure set : %s"
+  }
+
+  {
+    errorCode = 3035
+    httpStatusCode = ${BAD_REQUEST}
+    errorMsg = "%s does not have any facts that can cover the queried measure set : %s"
+  }
+
+
 ]
 
 lensCubeErrorsForMetastore = [

lens-cli/src/main/java/org/apache/lens/cli/commands/LensSchemaCommands.java

@@ -18,11 +18,21 @@
  */
 package org.apache.lens.cli.commands;
 
-import java.io.*;
+import java.io.File;
+import java.io.FilenameFilter;
 import java.util.List;
+import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.apache.lens.api.metastore.SchemaTraverser;
+import org.apache.lens.api.metastore.XBaseCube;
+import org.apache.lens.api.metastore.XDerivedCube;
+import org.apache.lens.api.metastore.XDimension;
+import org.apache.lens.api.metastore.XDimensionTable;
+import org.apache.lens.api.metastore.XFactTable;
+import org.apache.lens.api.metastore.XSegmentation;
+import org.apache.lens.api.metastore.XStorage;
 import org.apache.lens.cli.commands.annotations.UserDocumentation;
 
 import org.springframework.beans.factory.annotation.Autowired;
@@ -35,6 +45,7 @@
 import org.springframework.util.Assert;
 
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 @Component
 @UserDocumentation(title = "Creating schema with one command",
@@ -84,15 +95,52 @@ public class LensSchemaCommands implements CommandMarker {
     logger.setLevel(Level.FINE);
   }
 
-  private static final FilenameFilter XML_FILTER = new FilenameFilter() {
-    @Override
-    public boolean accept(File dir, String name) {
-      return name.endsWith(".xml");
-    }
-  };
+  private static final FilenameFilter XML_FILTER = (dir, name) -> name.endsWith(".xml");
+  private static final Map<Class<?>, String> CREATE_COMMAND_MAP = Maps.newHashMap();
+  private static final Map<Class<?>, String> UPDATE_COMMAND_MAP = Maps.newHashMap();
+
   @Autowired
   private JLineShellComponent shell;
 
+  static {
+    CREATE_COMMAND_MAP.put(XStorage.class, "create storage --path %s");
+    UPDATE_COMMAND_MAP.put(XStorage.class, "update storage --name %s --path %s");
+    CREATE_COMMAND_MAP.put(XDimension.class, "create dimension --path %s");
+    UPDATE_COMMAND_MAP.put(XDimension.class, "update dimension --name %s --path %s");
+    CREATE_COMMAND_MAP.put(XBaseCube.class, "create cube --path %s");
+    UPDATE_COMMAND_MAP.put(XBaseCube.class, "update cube --name %s --path %s");
+    CREATE_COMMAND_MAP.put(XDerivedCube.class, "create cube --path %s");
+    UPDATE_COMMAND_MAP.put(XDerivedCube.class, "update cube --name %s --path %s");
+    CREATE_COMMAND_MAP.put(XDimensionTable.class, "create dimtable --path %s");
+    UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s");
+    CREATE_COMMAND_MAP.put(XDimensionTable.class, "create dimtable --path %s");
+    UPDATE_COMMAND_MAP.put(XDimensionTable.class, "update dimtable --dimtable_name %s --path %s");
+    CREATE_COMMAND_MAP.put(XFactTable.class, "create fact --path %s");
+    UPDATE_COMMAND_MAP.put(XFactTable.class, "update fact --fact_name %s --path %s");
+    CREATE_COMMAND_MAP.put(XSegmentation.class, "create segmentation --path %s");
+    UPDATE_COMMAND_MAP.put(XSegmentation.class, "update segmentation --name %s --path %s");
+  }
+
+  private final SchemaTraverser.SchemaEntityProcessor processor = (entityFile, type) -> {
+    String entityName = entityFile.getName().substring(0, entityFile.getName().length() - 4);
+    String entityPath = entityFile.getAbsolutePath();
+    String createCommand = String.format(CREATE_COMMAND_MAP.get(type), entityPath);
+    String entityType = createCommand.substring(8, createCommand.indexOf(" ", 9));
+    logger.fine(createCommand);
+    if (shell.executeScriptLine(createCommand)) {
+      logger.info("Created " + entityType + " " + entityName);
+    } else {
+      logger.warning("Create failed, trying update");
+      String updateCommand = String.format(UPDATE_COMMAND_MAP.get(type), entityName, entityPath);
+      logger.fine(updateCommand);
+      if (shell.executeScriptLine(updateCommand)) {
+        logger.info("Updated " + entityType + " " + entityName);
+      } else {
+        logger.severe("Couldn't create or update " + entityType + " " + entityName);
+      }
+    }
+  };
+
   @CliCommand(value = {"schema", "create schema"},
     help = "Parses the specified resource file and executes commands for "
       + "creation/updation of schema\nExpected structure is " + STRUCTURE)
@@ -108,55 +156,10 @@ public void script(
     // ignore result. it can fail if database already exists
     shell.executeCommand("create database " + database);
     if (shell.executeScriptLine("use " + database)) {
-      createOrUpdate(new File(schemaDirectory, "storages"), "storage",
-        "create storage --path %s", "update storage --name %s --path %s");
-      createOrUpdate(new File(schemaDirectory, "dimensions"), "dimension",
-        "create dimension --path %s", "update dimension --name %s --path %s");
-      createOrUpdate(new File(new File(schemaDirectory, "cubes"), "base"), "base cube",
-        "create cube --path %s", "update cube --name %s --path %s");
-      createOrUpdate(new File(new File(schemaDirectory, "cubes"), "derived"), "derived cube",
-        "create cube --path %s", "update cube --name %s --path %s");
-      createOrUpdate(new File(schemaDirectory, "dimensiontables"), "dimension table",
-        "create dimtable --path %s", "update dimtable --dimtable_name %s --path %s");
-      createOrUpdate(new File(schemaDirectory, "dimtables"), "dimension table",
-        "create dimtable --path %s", "update dimtable --dimtable_name %s --path %s");
-      createOrUpdate(new File(schemaDirectory, "facts"), "fact",
-        "create fact --path %s", "update fact --fact_name %s --path %s");
-      createOrUpdate(new File(schemaDirectory, "segmentations"), "fact",
-        "create segmentation --path %s", "update segmentation --name %s --path %s");
+      SchemaTraverser schemaTraverser = new SchemaTraverser(schemaDirectory, processor);
+      schemaTraverser.run();
     } else {
       throw new IllegalStateException("Switching to database " + database + " failed");
     }
   }
-
-  public List<File> createOrUpdate(File parent, String entityType, String createSyntax, String updateSyntax) {
-    List<File> failedFiles = Lists.newArrayList();
-    // Create/update entities
-    if (parent.exists()) {
-      Assert.isTrue(parent.isDirectory(), parent.toString() + " must be a directory");
-      for (File entityFile : parent.listFiles(XML_FILTER)) {
-        String entityName = entityFile.getName().substring(0, entityFile.getName().length() - 4);
-        String entityPath = entityFile.getAbsolutePath();
-        String createCommand = String.format(createSyntax, entityPath);
-        logger.fine(createCommand);
-        if (shell.executeScriptLine(createCommand)) {
-          logger.info("Created " + entityType + " " + entityName);
-        } else {
-          logger.warning("Create failed, trying update");
-          String updateCommand = String.format(updateSyntax, entityName, entityPath);
-          logger.fine(updateCommand);
-          if (shell.executeScriptLine(updateCommand)) {
-            logger.info("Updated " + entityType + " " + entityName);
-          } else {
-            logger.severe("Couldn't create or update " + entityType + " " + entityName);
-            failedFiles.add(entityFile);
-          }
-        }
-      }
-    }
-    if (!failedFiles.isEmpty()) {
-      logger.severe("Failed for " + entityType + ": " + failedFiles);
-    }
-    return failedFiles;
-  }
 }

lens-cube/src/main/java/org/apache/lens/cube/error/LensCubeErrorCode.java

@@ -57,6 +57,8 @@ public enum LensCubeErrorCode {
   STORAGE_UNION_DISABLED(3031, 1500),
   COULD_NOT_PARSE_EXPRESSION(3032, 1500),
   QUERIED_TABLE_NOT_FOUND(3033, 0),
+  NO_UNION_CANDIDATE_AVAILABLE(3034, 1501),
+  NO_JOIN_CANDIDATE_AVAILABLE(3035, 1502),
   // Error codes greater than 3100 are errors while doing a metastore operation.
   ERROR_IN_ENTITY_DEFINITION(3101, 100),
   TIMELINE_ABSENT(3102, 100),

lens-cube/src/main/java/org/apache/lens/cube/error/NoCandidateFactAvailableException.java

@@ -18,18 +18,27 @@
  */
 package org.apache.lens.cube.error;
 
-import org.apache.lens.cube.metadata.CubeFactTable;
+import org.apache.lens.cube.parse.CubeQueryContext;
 import org.apache.lens.cube.parse.PruneCauses;
+import org.apache.lens.cube.parse.StorageCandidate;
 import org.apache.lens.server.api.error.LensException;
 
+import lombok.Getter;
 
+
+/**
+ * Note: This class is mainly meant for test cases to assert the detailed reasons (stored in
+ * {@link #briefAndDetailedError}  leading to "No Candidate was found"
+ */
 public class NoCandidateFactAvailableException extends LensException {
 
-  private final PruneCauses<CubeFactTable> briefAndDetailedError;
+  @Getter
+  private final PruneCauses<StorageCandidate> briefAndDetailedError;
 
-  public NoCandidateFactAvailableException(PruneCauses<CubeFactTable> briefAndDetailedError) {
-    super(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(), briefAndDetailedError.getBriefCause());
-    this.briefAndDetailedError = briefAndDetailedError;
+  public NoCandidateFactAvailableException(CubeQueryContext cubeql) {
+    super(LensCubeErrorCode.NO_CANDIDATE_FACT_AVAILABLE.getLensErrorInfo(),
+        cubeql.getStoragePruningMsgs().getBriefCause());
+    this.briefAndDetailedError = cubeql.getStoragePruningMsgs();
   }
 
   public PruneCauses.BriefAndDetailedError getJsonMessage() {

lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java

@@ -111,21 +111,16 @@ private static void addCubeNames(String factName, Map<String, String> props, Str
 
   private Map<String, Map<UpdatePeriod, String>> getUpdatePeriodMap(String factName, Map<String, String> props) {
     Map<String, Map<UpdatePeriod, String>> ret = new HashMap<>();
-    for (Map.Entry entry : storageUpdatePeriods.entrySet()) {
-      String storage = (String) entry.getKey();
-      for (UpdatePeriod period : (Set<UpdatePeriod>) entry.getValue()) {
+    for (Map.Entry<String, Set<UpdatePeriod>> entry : storageUpdatePeriods.entrySet()) {
+      String storage = entry.getKey();
+      for (UpdatePeriod period : entry.getValue()) {
         String storagePrefixKey = MetastoreUtil
           .getUpdatePeriodStoragePrefixKey(factName.trim(), storage, period.getName());
         String storageTableNamePrefix = props.get(storagePrefixKey);
         if (storageTableNamePrefix == null) {
           storageTableNamePrefix = storage;
         }
-        Map<UpdatePeriod, String> mapOfUpdatePeriods = ret.get(storage);
-        if (mapOfUpdatePeriods == null) {
-          mapOfUpdatePeriods = new HashMap<>();
-          ret.put(storage, mapOfUpdatePeriods);
-        }
-        mapOfUpdatePeriods.put(period, storageTableNamePrefix);
+        ret.computeIfAbsent(storage, k -> new HashMap<>()).put(period, storageTableNamePrefix);
       }
     }
     return ret;