diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java index 05108f015..3dd8dbb82 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java @@ -27,6 +27,8 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +import lombok.NonNull; + /** * The Class LensDatabaseCommands. */ @@ -119,4 +121,18 @@ public APIResult doUpdate(String name, String path) { protected APIResult doDelete(String name, boolean cascade) { return getClient().dropDatabase(name, cascade); } + + /** + * Adds the jar. + * + * @param path the path + * @return the string + */ + @CliCommand(value = "add dbjar", help = "Add jar resource to the db") + public String addDBJar( + @CliOption(key = {"", "path"}, mandatory = true, help = " / ") + @NonNull String path) { + APIResult result = getClient().addDBJarResource(path); + return result.getMessage(); + } } diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java index ea0cd7630..8405bea2c 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java @@ -612,4 +612,8 @@ public List listResources(String type) { public Response getLogs(String logFile) { return this.connection.getLogs(logFile); } + + public APIResult addDBJarResource(String path) { + return this.connection.addResourceToDB("jar", path); + } } diff --git a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java index eeb473afa..19299b2f4 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java @@ -18,6 +18,7 @@ */ package org.apache.lens.client; +import java.io.File; import java.net.ConnectException; import java.util.Iterator; import java.util.List; @@ -37,10 +38,8 @@ import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver; import org.apache.lens.client.exceptions.LensClientServerConnectionException; -import org.glassfish.jersey.media.multipart.FormDataBodyPart; -import org.glassfish.jersey.media.multipart.FormDataContentDisposition; -import org.glassfish.jersey.media.multipart.FormDataMultiPart; -import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.glassfish.jersey.media.multipart.*; +import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; import org.glassfish.jersey.moxy.json.MoxyJsonFeature; import lombok.Getter; @@ -327,4 +326,40 @@ public String toString() { sb.append('}'); return sb.toString(); } + + /** + * Adds the resource to current DB. + * + * @param type the type + * @param resourcePath the resource path + * @return the API result + */ + public APIResult addResourceToDB(String type, String resourcePath) { + WebTarget target = getMetastoreWebTarget(); + FormDataMultiPart mp = new FormDataMultiPart(); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("type").build(), type)); + + File file = new File(resourcePath); + log.debug("uploading file path : " + file.getAbsolutePath() + "|size = " + file.length()); + final FormDataContentDisposition dispo = FormDataContentDisposition// + .name("file")// + .fileName("db_0.jar")// temp dummy name + .size(file.length())// + .build(); + + FileDataBodyPart filePart = new FileDataBodyPart("file", file); + filePart.setContentDisposition(dispo); + mp.bodyPart(filePart); + + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult result = target.path("databases/jar").queryParam("sessionid", this.sessionHandle).request() + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + + log.debug(result.getStatus() + " - " + result); + + return result; + } } diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java index c11fd831a..967ec8ba6 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java @@ -18,6 +18,7 @@ */ package org.apache.lens.server.api.metastore; +import java.io.InputStream; import java.util.Date; import java.util.List; @@ -570,4 +571,6 @@ void updatePartition(LensSessionHandle sessionid, String tblName, String storage void updatePartitions(LensSessionHandle sessionid, String tblName, String storageName, XPartitionList partitions) throws LensException; + + void addDBJar(LensSessionHandle sessionid, String type, InputStream is) throws LensException; } diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index a1acd1a7a..f2ef1c41e 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -20,6 +20,7 @@ import static org.apache.lens.server.metastore.JAXBUtils.*; +import java.io.*; import java.util.*; import javax.ws.rs.BadRequestException; @@ -37,14 +38,17 @@ import org.apache.lens.server.api.metastore.CubeMetastoreService; import org.apache.lens.server.session.LensSessionImpl; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.service.cli.CLIService; import com.google.common.collect.Lists; @@ -823,6 +827,109 @@ public void updatePartitions(LensSessionHandle sessionid, String tblName, String } } + @Override + public void addDBJar(LensSessionHandle sessionid, String type, InputStream is) throws LensException { + // Read list of databases in + FileSystem serverFs = null; + FileSystem jarOrderFs = null; + FSDataOutputStream fos = null; + try { + acquire(sessionid); + String currentDB = SessionState.get().getCurrentDatabase(); + + String baseDir = + getHiveConf().get(LensConfConstants.DATABASE_RESOURCE_DIR, LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR); + + String dbDir = baseDir + File.separator + currentDB; + log.info("Database specific resources at {}", dbDir); + + + Path resTopDirPath = new Path(dbDir); + serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf()); + if (!serverFs.exists(resTopDirPath)) { + log.warn("Database resource location does not exist. Database jar can't be uploaded", dbDir); + throw new LensException("Database resource location does not exist. Database jar can't be uploaded"); + } + + Path resJarOrderPath = new Path(dbDir, "jar_order"); + jarOrderFs = FileSystem.newInstance(resJarOrderPath.toUri(), getHiveConf()); + if (jarOrderFs.exists(resJarOrderPath)) { + log.warn("Database jar_order file exist - {}. Database jar can't be uploaded", resJarOrderPath); + throw new LensException("Database jar_order file exist. Database jar can't be uploaded"); + } + + String tempFileName = currentDB + "_uploading.jar"; + + Path uploadingPath = new Path(dbDir, tempFileName); + FileSystem uploadingFs = FileSystem.newInstance(uploadingPath.toUri(), getHiveConf()); + if (uploadingFs.exists(uploadingPath)) { + log.warn("Already uploading a file - {}. This Database jar can't be uploaded. Try later!", uploadingPath); + throw new LensException("Database jar file upload in progress . Database jar can't be uploaded. Try later!"); + } + + int lastIndex = 0; + + Path dbFolderPath = new Path(baseDir, currentDB); + FileStatus[] existingFiles = serverFs.listStatus(dbFolderPath); + for (FileStatus fs : existingFiles) { + String fPath = fs.getPath().getName(); + String[] tokens = fPath.split("_"); + + if (tokens.length > 1) { + int fIndex = Integer.parseInt(tokens[tokens.length - 1].substring(0, 1)); + if (fIndex > lastIndex) { + lastIndex = fIndex; + } + } + } + + int newIndex = lastIndex + 1; + + + Path resJarPath = new Path(baseDir, currentDB + File.separator + tempFileName); + log.info("new jar name : " + resJarPath.getName()); + fos = serverFs.create(resJarPath); + IOUtils.copy(is, fos); + fos.flush(); + + Path renamePath = new Path(baseDir, currentDB + File.separator + currentDB + "_" + newIndex + ".jar"); + serverFs.rename(resJarPath, renamePath); + + + } catch (FileNotFoundException e) { + log.error("FileNotFoundException", e); + throw new LensException(e); + } catch (IOException e) { + log.error("IOException", e); + throw new LensException(e); + } finally { + if (fos != null) { + try { + fos.close(); + } catch (IOException e) { + log.error("Error closing file system instance fos", e); + } + } + + if (serverFs != null) { + try { + serverFs.close(); + } catch (IOException e) { + log.error("Error closing file system instance serverFs", e); + } + } + + if (jarOrderFs != null) { + try { + jarOrderFs.close(); + } catch (IOException e) { + log.error("Error closing file system instance jarOrderFs", e); + } + } + release(sessionid); + } + } + @Override public int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, String storageName, XPartitionList partitions) throws LensException { @@ -1467,4 +1574,5 @@ public HealthStatus getHealthStatus() { : new HealthStatus(isHealthy, details.toString()); } + } diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index e7d545cfb..2d763ba32 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -20,6 +20,7 @@ import static org.apache.lens.api.APIResult.*; +import java.io.*; import java.util.List; import javax.ws.rs.*; @@ -40,6 +41,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.glassfish.jersey.media.multipart.FormDataParam; + import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; @@ -1240,7 +1243,7 @@ public StringList getAllDims(@QueryParam("sessionid") LensSessionHandle sessioni @POST @Path("/dimtables") public APIResult createDimensionTable(@QueryParam("sessionid") LensSessionHandle sessionid, - XDimensionTable dimensionTable) { + XDimensionTable dimensionTable) { checkSessionId(sessionid); try { getSvc().createDimensionTable(sessionid, dimensionTable); @@ -1540,7 +1543,7 @@ public APIResult addPartitionToDimStorage(@QueryParam("sessionid") LensSessionHa @Path("/dimtables/{dimTableName}/storages/{storage}/partition") public APIResult updatePartitionOfDimStorage(@QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("dimTableName") String dimTableName, - @PathParam("storage") String storage, + @PathParam("storage") String storage, XPartition partition) { checkSessionId(sessionid); checkNonNullArgs("Partition is null", partition); @@ -1700,4 +1703,36 @@ public StringList getPartitionTimelines(@QueryParam("sessionid") LensSessionHand throw exc; } } + + /** + * Add a resource to the current DB + *

+ *

+ * The returned @{link APIResult} will have status SUCCEEDED only if the add operation was successful for all + * services running in this Lens server. + *

+ * + * @param sessionid session handle object + * @param type The type of resource. Valid types are 'jar' + * @param fileInputStream stream of the resource. Local or HDFS path + * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if add was successful. {@link APIResult} with state + * {@link Status#PARTIAL}, if add succeeded only for some services. {@link APIResult} with state + * {@link Status#FAILED}, if add has failed + */ + @POST + @Path("databases/jar") + @Consumes({MediaType.MULTIPART_FORM_DATA}) + @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN}) + public APIResult addDBResource(@QueryParam("sessionid") LensSessionHandle sessionid, + @FormDataParam("type") String type, @FormDataParam("file") InputStream fileInputStream) { + + try { + getSvc().addDBJar(sessionid, type, fileInputStream); + } catch (LensException e) { + log.error("Error in adding resource to db", e); + return new APIResult(Status.FAILED, e.getMessage()); + } + return new APIResult(Status.SUCCEEDED, "Add resource succeeded"); + } + } diff --git a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java index 72f5c534b..b75054607 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java @@ -18,6 +18,7 @@ */ package org.apache.lens.server.session; +import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -47,6 +48,7 @@ public class DatabaseResourceService extends AbstractService { public static final String NAME = "database-resources"; private Map classLoaderCache; private Map> dbResEntryMap; + private List commonResMap; /** * The metrics service. @@ -72,11 +74,17 @@ public DatabaseResourceService(String name) { super(name); } + private String resTopDir = null; + @Override public synchronized void init(HiveConf hiveConf) { super.init(hiveConf); classLoaderCache = new HashMap<>(); dbResEntryMap = new HashMap<>(); + commonResMap = new LinkedList<>(); + + resTopDir = + hiveConf.get(LensConfConstants.DATABASE_RESOURCE_DIR, LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR); } @Override @@ -85,11 +93,44 @@ public synchronized void start() { try { log.info("Starting loading DB specific resources"); - loadDbResourceEntries(); - loadResources(); + + // If the base folder itself is not present, then return as we can't load any jars. + FileSystem serverFs = null; + try { + Path resTopDirPath = new Path(resTopDir); + serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf()); + if (!serverFs.exists(resTopDirPath)) { + incrCounter(LOAD_RESOURCES_ERRORS); + log.warn("DB base location does not exist.", resTopDir); + return; + } + } catch (Exception io) { + log.error("Error locating base directory", io); + throw new LensException(io); + } finally { + if (serverFs != null) { + try { + serverFs.close(); + } catch (IOException e) { + log.error("Error closing file system instance", e); + } + } + } + + + // Map common jars available for all DB folders + mapCommonResourceEntries(); + + // Map DB specific jar + mapDbResourceEntries(); + + // Load all the jars for all the DB's mapped in previous steps + loadMappedResources(); + + } catch (LensException e) { incrCounter(LOAD_RESOURCES_ERRORS); - log.warn("Failed to load DB resource mapping, resources must be added explicitly to session."); + log.error("Failed to load DB resource mapping, resources must be added explicitly to session.", e); } } @@ -99,22 +140,69 @@ public synchronized void stop() { super.stop(); classLoaderCache.clear(); dbResEntryMap.clear(); + commonResMap.clear(); + } + + private void mapCommonResourceEntries() { + // Check if order file is present in the directory + List jars = new ScannedPaths(new Path(resTopDir), "jar", false).getFinalPaths(); + for (String resPath : jars) { + Path jarFilePath = new Path(resPath); + commonResMap.add(new LensSessionImpl.ResourceEntry("jar", jarFilePath.toUri().toString())); + } + + } + + private void mapDbResourceEntries(String dbName) throws LensException { + // Read list of databases in + FileSystem serverFs = null; + + try { + + Path resDirPath = new Path(resTopDir, dbName); + serverFs = FileSystem.newInstance(resDirPath.toUri(), getHiveConf()); + resDirPath = serverFs.resolvePath(resDirPath); + if (!serverFs.exists(resDirPath)) { + incrCounter(LOAD_RESOURCES_ERRORS); + log.warn("Database resource location does not exist - {}. Database jars will not be available", resDirPath); + + // remove the db entry + dbResEntryMap.remove(dbName); + classLoaderCache.remove(dbName); + return; + } + + findResourcesInDir(serverFs, dbName, resDirPath); + log.debug("Found resources {}", dbResEntryMap); + } catch (IOException io) { + log.error("Error getting list of dbs to load resources from", io); + throw new LensException(io); + } finally { + if (serverFs != null) { + try { + serverFs.close(); + } catch (IOException e) { + log.error("Error closing file system instance", e); + } + } + } } - private void loadDbResourceEntries() throws LensException { + private void mapDbResourceEntries() throws LensException { // Read list of databases in FileSystem serverFs = null; try { - String resTopDir = + String baseDir = getHiveConf().get(LensConfConstants.DATABASE_RESOURCE_DIR, LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR); - log.info("Database specific resources at {}", resTopDir); + log.info("Database specific resources at {}", baseDir); - Path resTopDirPath = new Path(resTopDir); + Path resTopDirPath = new Path(baseDir); serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf()); + resTopDirPath = serverFs.resolvePath(resTopDirPath); if (!serverFs.exists(resTopDirPath)) { incrCounter(LOAD_RESOURCES_ERRORS); - log.warn("Database resource location does not exist - {}. Database jars will not be available", resTopDir); + log.warn("DB base location does not exist.", baseDir); return; } @@ -123,10 +211,15 @@ private void loadDbResourceEntries() throws LensException { Path dbDirPath = dbDir.getPath(); if (serverFs.isDirectory(dbDirPath)) { String dbName = dbDirPath.getName(); - // Get all resources for that db - findResourcesInDir(serverFs, dbName, dbDirPath); - } else { - log.warn("DB resource DIR is not a directory: {}", dbDirPath); + + Path dbJarOrderPath = new Path(baseDir, dbName + File.separator + "jar_order"); + if (serverFs.exists(dbJarOrderPath)) { + // old flow + mapDbResourceEntries(dbName); + } else { + // new flow + mapDbSpecificJar(resTopDirPath, dbName); + } } } @@ -134,6 +227,8 @@ private void loadDbResourceEntries() throws LensException { } catch (IOException io) { log.error("Error getting list of dbs to load resources from", io); throw new LensException(io); + } catch (Exception e) { + log.error("Exception : ", e); } finally { if (serverFs != null) { try { @@ -145,6 +240,56 @@ private void loadDbResourceEntries() throws LensException { } } + private void mapDbSpecificJar(Path baseDir, String dbName) throws Exception { + FileSystem serverFs = null; + try { + int lastIndex = 0; + + + Path dbFolderPath = new Path(baseDir, dbName); + + serverFs = FileSystem.newInstance(dbFolderPath.toUri(), getHiveConf()); + FileStatus[] existingFiles = serverFs.listStatus(dbFolderPath); + for (FileStatus fs : existingFiles) { + String fPath = fs.getPath().getName(); + String[] tokens = fPath.split("_"); + + if (tokens.length > 1) { + int fIndex = Integer.parseInt(tokens[tokens.length - 1].substring(0, 1)); + if (fIndex > lastIndex) { + lastIndex = fIndex; + } + } + } + + if (lastIndex > 0) { + // latest jar + Path latestJarPath = new Path(baseDir, dbName + File.separator + dbName + "_" + lastIndex + ".jar"); + addResourceEntry(new LensSessionImpl.ResourceEntry("jar", latestJarPath.toUri().toString()), dbName); + } + + + // add common jars + for (LensSessionImpl.ResourceEntry jar : commonResMap) { + addResourceEntry(jar, dbName); + } + + } catch (Exception io) { + log.error("Error getting db specific resource ", io); + throw new LensException(io); + } finally { + if (serverFs != null) { + try { + serverFs.close(); + } catch (IOException e) { + log.error("Error closing file system instance", e); + } + } + } + + } + + private void findResourcesInDir(FileSystem serverFs, String database, Path dbDirPath) throws IOException { // Check if order file is present in the directory List jars = new ScannedPaths(dbDirPath, "jar").getFinalPaths(); @@ -197,7 +342,7 @@ private void addResourceEntry(LensSessionImpl.ResourceEntry entry, String dbName /** * Load DB specific resources */ - public void loadResources() { + public void loadMappedResources() { for (String db : dbResEntryMap.keySet()) { try { createClassLoader(db); @@ -223,7 +368,7 @@ protected void createClassLoader(String database) throws LensException { * @return class loader updated as a result of adding any JARs */ protected synchronized ClassLoader loadDBJars(String database, Collection resources, - boolean addToCache) { + boolean addToCache) { ClassLoader classLoader = classLoaderCache.get(database); if (classLoader == null) { // No change since there are no static resources to be added diff --git a/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java b/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java index e48eab4e2..449f1d89c 100644 --- a/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java +++ b/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java @@ -46,12 +46,17 @@ public class ScannedPaths implements Iterable { private final Path path; private final String type; + private boolean recurse; /* The Chosen Ones */ - @Getter(lazy = true) private final List finalPaths = getMatchedPaths(path, type); + @Getter(lazy = true) private final List finalPaths = getMatchedPaths(path, type, recurse); public ScannedPaths(String path, String type) { - this(new Path(path), type); + this(new Path(path), type, true); + } + + public ScannedPaths(Path path, String type) { + this(path, type, true); } @@ -70,7 +75,7 @@ public Iterator iterator() { * * Updates finalPaths List with matched paths and returns an iterator for matched paths. */ - private List getMatchedPaths(Path pt, String type) { + private List getMatchedPaths(Path pt, String type, boolean recurse) { List finalPaths = new ArrayList<>(); InputStream resourceOrderIStream = null; FileSystem fs; @@ -102,8 +107,10 @@ private List getMatchedPaths(Path pt, String type) { statuses = fs.globStatus(new Path(pt, "*")); if (statuses != null) { for (FileStatus st : statuses) { - newMatches = getMatchedPaths(st.getPath(), type); - finalPaths.addAll(newMatches); + if (!st.isDirectory() || recurse) { + newMatches = getMatchedPaths(st.getPath(), type, recurse); + finalPaths.addAll(newMatches); + } } } } @@ -120,9 +127,9 @@ private List getMatchedPaths(Path pt, String type) { /** Get matched resources recursively for provided path/pattern **/ if (resource.startsWith("/") || resource.contains(":/")) { - newMatches = getMatchedPaths(new Path(resource), type); + newMatches = getMatchedPaths(new Path(resource), type, true); } else { - newMatches = getMatchedPaths(new Path(pt, resource), type); + newMatches = getMatchedPaths(new Path(pt, resource), type, true); } finalPaths.addAll(newMatches); } @@ -135,7 +142,7 @@ private List getMatchedPaths(Path pt, String type) { FileStatus[] statuses = fs.globStatus(Path.getPathWithoutSchemeAndAuthority(pt)); if (statuses != null) { for (FileStatus st : statuses) { - List newMatches = getMatchedPaths(st.getPath(), type); + List newMatches = getMatchedPaths(st.getPath(), type, recurse); finalPaths.addAll(newMatches); } } diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java index fb757e467..2bac71069 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java @@ -76,7 +76,7 @@ protected URI getUri() { } private boolean isPortAlreadyFound() { - return port != -1; + return port != -1 && port != 0; } public void setUp() throws Exception { diff --git a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java index b651b7916..56824da5a 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java @@ -62,6 +62,7 @@ public final class LensServerTestUtil { public static final String DB_WITH_JARS = "test_db_static_jars"; public static final String DB_WITH_JARS_2 = "test_db_static_jars_2"; + private LensServerTestUtil() { } @@ -93,8 +94,9 @@ public static void createTable(String tblName, WebTarget parent, LensSessionHand mt)); final QueryHandle handle = target.request(mt) - .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType>() {}).getData(); + .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), + new GenericType>() { + }).getData(); // wait till the query finishes LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) .get(LensQuery.class); @@ -105,7 +107,7 @@ public static void createTable(String tblName, WebTarget parent, LensSessionHand stat = ctx.getStatus(); Thread.sleep(1000); } - final String debugHelpMsg = "Query Handle:"+ctx.getQueryHandleString(); + final String debugHelpMsg = "Query Handle:" + ctx.getQueryHandleString(); assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, debugHelpMsg); assertTrue(ctx.getSubmissionTime() > 0, debugHelpMsg); assertTrue(ctx.getLaunchTime() > 0, debugHelpMsg); @@ -120,7 +122,7 @@ public static void createTable(String tblName, WebTarget parent, LensSessionHand } public static void loadData(String tblName, final String testDataFile, WebTarget parent, - LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { + LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); final WebTarget target = parent.path("queryapi/queries"); @@ -129,18 +131,19 @@ public static void loadData(String tblName, final String testDataFile, WebTarget String dataLoad = "LOAD DATA LOCAL INPATH '" + testDataFile + "' OVERWRITE INTO TABLE " + tblName; mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - mt)); + mt)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), dataLoad)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, - mt)); + mt)); final QueryHandle handle = target.request(mt).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType>() {}).getData(); + new GenericType>() { + }).getData(); // wait till the query finishes LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) - .get(LensQuery.class); + .get(LensQuery.class); QueryStatus stat = ctx.getStatus(); while (!stat.finished()) { ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) @@ -150,6 +153,7 @@ public static void loadData(String tblName, final String testDataFile, WebTarget } assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); } + /** * Load data. * @@ -160,7 +164,7 @@ public static void loadData(String tblName, final String testDataFile, WebTarget * @throws InterruptedException the interrupted exception */ public static void loadDataFromClasspath(String tblName, final String testDataFile, WebTarget parent, - LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { + LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { String absolutePath = LensServerTestUtil.class.getClassLoader().getResource(testDataFile).getPath(); loadData(tblName, absolutePath, parent, lensSessionId, mt); @@ -206,7 +210,8 @@ public static void dropTableWithConf(String tblName, WebTarget parent, LensSessi mt)); final QueryHandle handle = target.request(mt).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType>() {}).getData(); + new GenericType>() { + }).getData(); // wait till the query finishes LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) @@ -261,7 +266,7 @@ public static void createTestDatabaseResources(String[] testDatabases, HiveConf // nothing to setup return; } - File resDir = new File("target/resources"); + File resDir = new File(conf.get(LensConfConstants.DATABASE_RESOURCE_DIR, "target/resources/")); if (!resDir.exists()) { resDir.mkdir(); } @@ -302,4 +307,152 @@ public static void createTestDatabaseResources(String[] testDatabases, HiveConf } } } + + + /** + * Test case when db folder exists & jar_order file present ( Existing flow ) + * + * @throws Exception + */ + public static void createTestDbWithoutCommonJars(String[] dbs, HiveConf conf) throws Exception { + File srcJarDir = new File("target/testjars/"); + if (!srcJarDir.exists()) { + // nothing to setup + return; + } + File resDir = new File(conf.get(LensConfConstants.DATABASE_RESOURCE_DIR)); + if (!resDir.exists()) { + resDir.mkdir(); + } + + setUpDbWithJarOrderAndFiles(dbs[0], resDir, conf); + + setUpDbWithNoJarOrderAndVersionFiles(dbs[1], resDir, conf); + + setUpDbWithNoJarOrderAndNoFiles(dbs[2], resDir, conf); + } + + + public static void createTestDbWithCommonJars(String[] dbs, HiveConf conf) throws Exception { + File srcJarDir = new File("target/testjars/"); + if (!srcJarDir.exists()) { + // nothing to setup + return; + } + File resDir = new File(conf.get(LensConfConstants.DATABASE_RESOURCE_DIR)); + if (!resDir.exists()) { + resDir.mkdir(); + } + + File testJarFile = new File("target/testjars/test.jar"); + + // Copy common jars + FileUtils.copyFile(testJarFile, new File(resDir, "lens-ship.jar")); + FileUtils.copyFile(testJarFile, new File(resDir, "meta.jar")); + + + setUpDbWithJarOrderAndFiles(dbs[0], resDir, conf); + + setUpDbWithNoJarOrderAndVersionFiles(dbs[1], resDir, conf); + + setUpDbWithNoJarOrderAndNoFiles(dbs[2], resDir, conf); + } + + //********************************************* + // Setup DB : JAR_ORDER + SOME FILES + //********************************************* + private static void setUpDbWithJarOrderAndFiles(String db, File resDir, HiveConf conf) throws Exception { + Database database0 = new Database(); + database0.setName(db); + + Hive hive = Hive.get(conf); + hive.createDatabase(database0, true); + + File dbDir1 = new File(resDir, db); + if (!dbDir1.exists()) { + dbDir1.mkdir(); + } + + File dbDir0 = new File(resDir, db); + if (!dbDir0.exists()) { + dbDir0.mkdir(); + } + + String[] jarFilesOrder = { + "x_" + db + ".jar", + "y_" + db + ".jar", + "z_" + db + ".jar", + }; + + try { + + // We are explicitly specifying jar order + FileUtils.writeLines(new File(dbDir0, "jar_order"), Arrays.asList(jarFilesOrder[2], jarFilesOrder[1], + jarFilesOrder[0])); + + File testJarFile = new File("target/testjars/test.jar"); + + FileUtils.copyFile(testJarFile, new File(dbDir0, jarFilesOrder[0])); + FileUtils.copyFile(testJarFile, new File(dbDir0, jarFilesOrder[1])); + FileUtils.copyFile(testJarFile, new File(dbDir0, jarFilesOrder[2])); + } catch (FileNotFoundException fnf) { + log.error("File not found.", fnf); + } catch (Exception e) { + log.error("File not found.", e); + } + } + + + //********************************************* + // Setup DB : NO JAR_ORDER + VERSION FILES + //********************************************* + private static void setUpDbWithNoJarOrderAndVersionFiles(String db, File resDir, HiveConf conf) throws Exception { + + Database database0 = new Database(); + database0.setName(db); + + Hive hive = Hive.get(conf); + hive.createDatabase(database0, true); + + File dbDir1 = new File(resDir, db); + if (!dbDir1.exists()) { + dbDir1.mkdir(); + } + + String[] jarOrder = { + db + "_3.jar", + db + "_2.jar", + db + "_1.jar", + }; + + + try { + File testJarFile = new File("target/testjars/test.jar"); + + FileUtils.copyFile(testJarFile, new File(dbDir1, jarOrder[0])); + FileUtils.copyFile(testJarFile, new File(dbDir1, jarOrder[1])); + FileUtils.copyFile(testJarFile, new File(dbDir1, jarOrder[2])); + } catch (FileNotFoundException fnf) { + log.error("File not found.", fnf); + } + } + + //********************************************* + // Setup DB : NO JAR_ORDER + NO FILES + NO VERSION FILES + //********************************************* + private static void setUpDbWithNoJarOrderAndNoFiles(String db, File resDir, HiveConf conf) throws Exception { + + Database database0 = new Database(); + database0.setName(db); + + Hive hive = Hive.get(conf); + hive.createDatabase(database0, true); + + File dbDir1 = new File(resDir, db); + if (!dbDir1.exists()) { + dbDir1.mkdir(); + } + } + + } diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java new file mode 100644 index 000000000..3839c43b4 --- /dev/null +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.metastore; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; + +import java.io.File; +import java.util.HashMap; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; + +import org.apache.lens.api.APIResult; +import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.server.LensJerseyTest; +import org.apache.lens.server.LensServices; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.metastore.CubeMetastoreService; + +import org.apache.commons.io.FileUtils; + +import org.glassfish.jersey.media.multipart.FormDataBodyPart; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.MultiPart; +import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; +import org.glassfish.jersey.test.TestProperties; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +@Test(groups = "unit-test") +public class TestDatabaseService extends LensJerseyTest { + CubeMetastoreServiceImpl metastoreService; + LensSessionHandle lensSessionId; + String rootPath = null; + + + private void assertSuccess(APIResult result) { + assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED, String.valueOf(result)); + } + + @BeforeMethod + public void setUp() throws Exception { + super.setUp(); + rootPath = getServerConf().get(LensConfConstants.DATABASE_RESOURCE_DIR); + metastoreService = LensServices.get().getService(CubeMetastoreService.NAME); + lensSessionId = metastoreService.openSession("foo", "bar", new HashMap()); + } + + @AfterMethod + public void tearDown() throws Exception { + metastoreService.closeSession(lensSessionId); + super.tearDown(); + } + + @Override + protected Application configure() { + enable(TestProperties.LOG_TRAFFIC); + enable(TestProperties.DUMP_ENTITY); + return new MetastoreApp(); + } + + private String getCurrentDatabase(MediaType mediaType) throws Exception { + return target().path("metastore").path("databases/current") + .queryParam("sessionid", lensSessionId).request(mediaType).get(String.class); + } + + private FormDataMultiPart getFormData(MediaType mediaType) { + FormDataMultiPart mp = null; + try { + + mp = new FormDataMultiPart(); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("type").build(), "jar")); + + File file = new File("target/testjars/serde.jar"); + log.debug("uploading file path : " + file.getAbsolutePath() + "|size = " + file.length()); + final FormDataContentDisposition dispo = FormDataContentDisposition + .name("file") + .fileName("test.jar") + .size(file.length()) + .build(); + + FileDataBodyPart filePart = new FileDataBodyPart("file", file); + filePart.setContentDisposition(dispo); + mp.bodyPart(filePart); + } catch (Exception e) { + log.error("Error in getting form data", e); + } + + return mp; + + } + + /** + * Test case when no db folder exists + * + * @param mediaType + * @throws Exception + */ + @Test(dataProvider = "mediaTypeData") + public void testJarUpload(MediaType mediaType) throws Exception { + String dbName = "db1" + "_" + mediaType.getSubtype(); + + // create + APIResult result = target().path("metastore").path("databases") + .queryParam("sessionid", lensSessionId).request(mediaType).post(getEntityForString(dbName, mediaType), APIResult + .class); + assertNotNull(result); + assertSuccess(result); + + // set + WebTarget dbTarget = target().path("metastore").path("databases/current"); + + result = dbTarget.queryParam("sessionid", lensSessionId).request(mediaType) + .put(getEntityForString(dbName, mediaType), APIResult.class); + assertNotNull(result); + assertSuccess(result); + + + FormDataMultiPart mp = getFormData(mediaType); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult resultUpd = target().path("metastore").path("databases/jar"). + queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + log.debug(resultUpd.getStatus() + " " + resultUpd); + assertEquals(resultUpd.getMessage(), "Database resource location does not exist. Database jar can't be uploaded"); + + } + + + /** + * Test case when db folder exists & jar_order file present ( Existing flow ) + * + * @param mediaType + * @throws Exception + */ + @Test(dataProvider = "mediaTypeData") + public void testJarUploadWithJarOrderFileInDbFolder(MediaType mediaType) throws Exception { + String dbName = "db2" + "_" + mediaType.getSubtype() + "_" + System.currentTimeMillis(); + + // create + APIResult result = target().path("metastore").path("databases") + .queryParam("sessionid", lensSessionId).request(mediaType).post(getEntityForString(dbName, mediaType), APIResult + .class); + assertNotNull(result); + assertSuccess(result); + + // set + WebTarget dbTarget = target().path("metastore").path("databases/current"); + + result = dbTarget.queryParam("sessionid", lensSessionId).request(mediaType) + .put(getEntityForString(dbName, mediaType), APIResult.class); + assertNotNull(result); + assertSuccess(result); + + // Create DB folder + File dbFolder = new File("target/resources/" + dbName); + dbFolder.mkdirs(); + + File dbFolderJarOrder = new File("target/resources/" + dbName + File.separator + "jar_order"); + dbFolderJarOrder.createNewFile(); + + + FormDataMultiPart mp = getFormData(mediaType); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult resultUpd = target().path("metastore").path("databases/jar"). + queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + log.debug(resultUpd.getStatus() + " " + resultUpd); + assertEquals(resultUpd.getMessage(), "Database jar_order file exist. Database jar can't be uploaded"); + + cleanUp(dbFolder); + } + + + /** + * Test case when db folder exists & jar_order file NOT present & db_uploading.jar present. + * This restricts single upload ant any time. + * + * @param mediaType + * @throws Exception + */ + @Test(dataProvider = "mediaTypeData") + public void testJarUploadWithDbUploadingJarInFolder(MediaType mediaType) throws Exception { + String dbName = "db3" + "_" + mediaType.getSubtype() + "_" + System.currentTimeMillis(); + + // create + APIResult result = target().path("metastore").path("databases") + .queryParam("sessionid", lensSessionId).request(mediaType).post(getEntityForString(dbName, mediaType), APIResult + .class); + assertNotNull(result); + assertSuccess(result); + + // set + WebTarget dbTarget = target().path("metastore").path("databases/current"); + + result = dbTarget.queryParam("sessionid", lensSessionId).request(mediaType) + .put(getEntityForString(dbName, mediaType), APIResult.class); + assertNotNull(result); + assertSuccess(result); + + // Create DB folder + File dbFolder = new File("target/resources/" + dbName); + dbFolder.mkdirs(); + + File dbFolderJarOrder = new File("target/resources/" + dbName + File.separator + dbName + "_uploading.jar"); + dbFolderJarOrder.createNewFile(); + + FormDataMultiPart mp = getFormData(mediaType); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult resultUpd = target().path("metastore").path("databases/jar"). + queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + log.debug(resultUpd.getStatus() + " " + resultUpd); + assertEquals(resultUpd.getMessage(), "Database jar file upload in progress . Database jar can't be uploaded." + + " Try later!"); + + cleanUp(dbFolder); + } + + + /** + * Test case when db folder exists & jar_order file NOT present & db_uploading.jar NOT present. + * + * @param mediaType + * @throws Exception + */ + @Test(dataProvider = "mediaTypeData") + public void testJarUploadWithNoJarOrderInFolder(MediaType mediaType) throws Exception { + String dbName = "db4" + "_" + mediaType.getSubtype() + "_" + System.currentTimeMillis(); + + // create + APIResult result = target().path("metastore").path("databases") + .queryParam("sessionid", lensSessionId).request(mediaType).post(getEntityForString(dbName, mediaType), APIResult + .class); + assertNotNull(result); + assertSuccess(result); + + // set + WebTarget dbTarget = target().path("metastore").path("databases/current"); + + result = dbTarget.queryParam("sessionid", lensSessionId).request(mediaType) + .put(getEntityForString(dbName, mediaType), APIResult.class); + assertNotNull(result); + assertSuccess(result); + + // Create DB folder + File dbFolder = new File("target/resources/" + dbName); + dbFolder.mkdirs(); + + FormDataMultiPart mp = getFormData(mediaType); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult resultUpd = target().path("metastore").path("databases/jar"). + queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + log.debug(resultUpd.getStatus() + " " + resultUpd); + assertEquals(resultUpd.getStatus(), APIResult.Status.SUCCEEDED); + + cleanUp(dbFolder); + } + + /** + * Test case when db folder exists & jar_order file NOT present & db_uploading.jar NOT present and with existing jars. + * + * @param mediaType + * @throws Exception + */ + @Test(dataProvider = "mediaTypeData") + public void testJarUploadWithExistingJarsInFolder(MediaType mediaType) throws Exception { + String dbName = "db5" + "_" + mediaType.getSubtype() + "_" + System.currentTimeMillis(); + + // create + APIResult result = target().path("metastore").path("databases") + .queryParam("sessionid", lensSessionId).request(mediaType).post(getEntityForString(dbName, mediaType), APIResult + .class); + assertNotNull(result); + assertSuccess(result); + + // set + WebTarget dbTarget = target().path("metastore").path("databases/current"); + + result = dbTarget.queryParam("sessionid", lensSessionId).request(mediaType) + .put(getEntityForString(dbName, mediaType), APIResult.class); + assertNotNull(result); + assertSuccess(result); + + // Create DB folder + File dbFolder = new File("target/resources/" + dbName); + dbFolder.mkdirs(); + + File dbFolderJarOrder1 = new File("target/resources/" + dbName + File.separator + dbName + "_1.jar"); + dbFolderJarOrder1.createNewFile(); + + File dbFolderJarOrder2 = new File("target/resources/" + dbName + File.separator + dbName + "_2.jar"); + dbFolderJarOrder2.createNewFile(); + + FormDataMultiPart mp = getFormData(mediaType); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult resultUpd = target().path("metastore").path("databases/jar"). + queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + log.debug(resultUpd.getStatus() + " " + resultUpd); + assertEquals(resultUpd.getStatus(), APIResult.Status.SUCCEEDED); + + cleanUp(dbFolder); + } + + private void cleanUp(File f) { + try { + FileUtils.deleteDirectory(f); + } catch (Exception e) { + log.error("Error cleaning directory", e); + } + } + +} diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java index 1008faf8a..c60014116 100644 --- a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java +++ b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java @@ -48,25 +48,66 @@ public class TestDatabaseResourceService { private final String[] testDatabases = {DB1, DB2}; + private static final String JAR_ORDER_AND_FILES = DB_PFX + "jar_order_files_in_db"; + private static final String NO_JAR_ORDER_AND_VERSION_FILES = DB_PFX + "no_jar_order_and_version_files"; + private static final String NO_JAR_ORDER_NO_FILES = DB_PFX + "no_jar_order_no_files"; + + private final String[] testDatabases1 = {JAR_ORDER_AND_FILES, NO_JAR_ORDER_AND_VERSION_FILES, NO_JAR_ORDER_NO_FILES}; + private final HiveConf conf = new HiveConf(TestDatabaseResourceService.class); private DatabaseResourceService dbResService; + private final HiveConf conf1 = new HiveConf(TestDatabaseResourceService.class); + private DatabaseResourceService dbResService1; + + private final HiveConf conf2 = new HiveConf(TestDatabaseResourceService.class); + private DatabaseResourceService dbResService2; + @BeforeClass public void setup() throws Exception { - LensServerTestUtil.createTestDatabaseResources(testDatabases, conf); - // Start resource service. + + String prefix = System.getProperty("user.dir"); + conf.set(LensConfConstants.DATABASE_RESOURCE_DIR, "target/resources"); + conf1.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix + "/target/resources_without_common_jars"); + conf2.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix + "/target/resources_with_common_jars"); + + LensServerTestUtil.createTestDatabaseResources(testDatabases, conf); + LensServerTestUtil.createTestDbWithoutCommonJars(testDatabases1, conf1); + LensServerTestUtil.createTestDbWithCommonJars(testDatabases1, conf2); + dbResService = new DatabaseResourceService(DatabaseResourceService.NAME); dbResService.init(conf); dbResService.start(); + + dbResService1 = new DatabaseResourceService(DatabaseResourceService.NAME); + dbResService1.init(conf1); + dbResService1.start(); + + dbResService2 = new DatabaseResourceService(DatabaseResourceService.NAME); + dbResService2.init(conf2); + dbResService2.start(); + + } @AfterClass public void tearDown() throws Exception { - Hive hive = Hive.get(conf); + Hive hive0 = Hive.get(conf); for (String db : testDatabases) { - hive.dropDatabase(db, true, true); + hive0.dropDatabase(db, true, true); + } + + Hive hive1 = Hive.get(conf1); + for (String db : testDatabases1) { + hive1.dropDatabase(db, true, true); + } + + Hive hive2 = Hive.get(conf2); + for (String db : testDatabases1) { + hive2.dropDatabase(db, true, true); } + } @Test @@ -82,7 +123,7 @@ public void testClassLoaderCreated() throws Exception { private boolean isJarLoaded(ClassLoader loader, String db) throws Exception { URLClassLoader db1Loader = (URLClassLoader) loader; - for (URL url : db1Loader.getURLs()) { + for (URL url : db1Loader.getURLs()) { String jarFile = url.getPath(); if (jarFile.endsWith(db + ".jar")) { log.info("Found jar url " + url.toString()); @@ -132,11 +173,144 @@ public void verifyClassLoader() throws Exception { Class clz = Class.forName("ClassLoaderTestClass", true, getClass().getClassLoader()); Assert.fail("Expected class loading to fail"); } catch (Throwable th) { - log.error("Expected error " + th + " msg = "+th.getMessage(), th); + log.error("Expected error " + th + " msg = " + th.getMessage(), th); } // Should pass now Class clz = Class.forName(TEST_CLASS, true, dbResService.getClassLoader(DB1)); Assert.assertNotNull(clz); } + + + /************************************************** + * Test cases without common jars + **************************************************/ + @Test + public void testDbWithoutCommonJarsAndWithJarOrderAndFiles() throws Exception { + String db = testDatabases1[0]; + Collection actualOrder = dbResService1.getResourcesForDatabase(db); + List actualOrderList = new ArrayList(); + + for (LensSessionImpl.ResourceEntry res : actualOrder) { + actualOrderList.add(res.getLocation()); + } + + String[] jarFilesOrder = { + "z_" + db + ".jar", + "y_" + db + ".jar", + "x_" + db + ".jar", + }; + + // Verify order + for (int i = 0; i < jarFilesOrder.length; i++) { + Assert.assertTrue(actualOrderList.get(i).contains(jarFilesOrder[i]), + actualOrderList.get(i) + " > " + jarFilesOrder[i]); + } + } + + @Test + public void testDbWithoutCommonJarsAndWithNoJarOrderAndVersionFiles() throws Exception { + String db = testDatabases1[1]; + Collection actualOrder = dbResService1.getResourcesForDatabase(db); + List actualOrderList = new ArrayList(); + + for (LensSessionImpl.ResourceEntry res : actualOrder) { + actualOrderList.add(res.getLocation()); + } + + // Should pick the latest one + String[] jarFilesOrder = { + db + "_3.jar", + }; + + // Verify order + for (int i = 0; i < jarFilesOrder.length; i++) { + Assert.assertTrue(actualOrderList.get(i).contains(jarFilesOrder[i]), + actualOrderList.get(i) + " > " + jarFilesOrder[i]); + } + } + + @Test + public void testDbWithoutCommonJarsAndNoJarOrderAndNoFiles() throws Exception { + String db = testDatabases1[2]; + Collection actualOrder = dbResService1.getResourcesForDatabase(db); + Assert.assertNull(actualOrder); + + } + + /************************************************** + * Test cases without common jars + **************************************************/ + + @Test + public void testDbWithCommonJarsAndWithJarOrderAndFiles() throws Exception { + String db = testDatabases1[0]; + Collection actualOrder = dbResService2.getResourcesForDatabase(db); + List actualOrderList = new ArrayList(); + + for (LensSessionImpl.ResourceEntry res : actualOrder) { + actualOrderList.add(res.getLocation()); + } + + String[] jarFilesOrder = { + "z_" + db + ".jar", + "y_" + db + ".jar", + "x_" + db + ".jar", + }; + + // Verify order + for (int i = 0; i < jarFilesOrder.length; i++) { + Assert.assertTrue(actualOrderList.get(i).contains(jarFilesOrder[i]), + actualOrderList.get(i) + " > " + jarFilesOrder[i]); + } + } + + @Test + public void testDbWithCommonJarsAndWithNoJarOrderAndVersionFiles() throws Exception { + String db = testDatabases1[1]; + Collection actualOrder = dbResService2.getResourcesForDatabase(db); + List actualOrderList = new ArrayList(); + + for (LensSessionImpl.ResourceEntry res : actualOrder) { + actualOrderList.add(res.getLocation()); + } + + // Should pick the latest one + String[] jarFilesOrder = { + db + "_3.jar", + "lens-ship.jar", + "meta.jar", + }; + + // Verify order + for (int i = 0; i < jarFilesOrder.length; i++) { + Assert.assertTrue(actualOrderList.get(i).contains(jarFilesOrder[i]), + actualOrderList.get(i) + " > " + jarFilesOrder[i]); + } + } + + @Test + public void testDbWithCommonJarsAndNoJarOrderAndNoFiles() throws Exception { + String db = testDatabases1[2]; + Collection actualOrder = dbResService2.getResourcesForDatabase(db); + List actualOrderList = new ArrayList(); + + for (LensSessionImpl.ResourceEntry res : actualOrder) { + actualOrderList.add(res.getLocation()); + } + + // Should pick the latest one + String[] jarFilesOrder = { + "lens-ship.jar", + "meta.jar", + }; + + // Verify order + for (int i = 0; i < jarFilesOrder.length; i++) { + Assert.assertTrue(actualOrderList.get(i).contains(jarFilesOrder[i]), + actualOrderList.get(i) + " > " + jarFilesOrder[i]); + } + } + + } diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml index c3187a8de..6a2660e5e 100644 --- a/lens-server/src/test/resources/lens-site.xml +++ b/lens-server/src/test/resources/lens-site.xml @@ -45,6 +45,12 @@ Enable metrics to be reported on console + + lens.server.database.resource.dir + ${project.build.directory}/resources/ + database resource directory + + lens.server.persist.location target/persist-dir