From 137dcd5919d51a07287f70f12ad9747ba1aa6db3 Mon Sep 17 00:00:00 2001 From: badri Date: Thu, 3 Mar 2016 12:48:27 +0530 Subject: [PATCH 1/7] Feature implemented both on client and server. Needs to be tested --- .../cli/commands/LensDatabaseCommands.java | 15 ++++ .../org/apache/lens/client/LensClient.java | 4 + .../apache/lens/client/LensConnection.java | 25 ++++++ .../api/metastore/CubeMetastoreService.java | 3 + .../metastore/CubeMetastoreServiceImpl.java | 87 +++++++++++++++++++ .../server/metastore/MetastoreResource.java | 63 ++++++++++++++ 6 files changed, 197 insertions(+) diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java index 05108f015..f3087084d 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java @@ -27,6 +27,8 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; +import lombok.NonNull; + /** * The Class LensDatabaseCommands. */ @@ -119,4 +121,17 @@ public APIResult doUpdate(String name, String path) { protected APIResult doDelete(String name, boolean cascade) { return getClient().dropDatabase(name, cascade); } + + /** + * Adds the jar. + * + * @param path the path + * @return the string + */ + @CliCommand(value = "add dbjar", help = "Add jar resource to the db") + public String addDBJar( + @CliOption(key = {"", "path"}, mandatory = true, help = " / ") @NonNull String path) { + APIResult result = getClient().addDBJarResource(path); + return result.getMessage(); + } } diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java index ea0cd7630..8405bea2c 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java @@ -612,4 +612,8 @@ public List listResources(String type) { public Response getLogs(String logFile) { return this.connection.getLogs(logFile); } + + public APIResult addDBJarResource(String path) { + return this.connection.addResourceToDB("jar", path); + } } diff --git a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java index eeb473afa..bdbfacc25 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java @@ -18,6 +18,7 @@ */ package org.apache.lens.client; +import java.io.File; import java.net.ConnectException; import java.util.Iterator; import java.util.List; @@ -41,6 +42,7 @@ import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataMultiPart; import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; import org.glassfish.jersey.moxy.json.MoxyJsonFeature; import lombok.Getter; @@ -327,4 +329,27 @@ public String toString() { sb.append('}'); return sb.toString(); } + + /** + * Adds the resource to current DB. + * + * @param type the type + * @param resourcePath the resource path + * @return the API result + */ + public APIResult addResourceToDB(String type, String resourcePath) { + WebTarget target = getMetastoreWebTarget(); + FormDataMultiPart mp = new FormDataMultiPart(); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), this.sessionHandle, + MediaType.APPLICATION_XML_TYPE)); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("type").build(), type)); + + FileDataBodyPart filePart = new FileDataBodyPart("file", new File(resourcePath)); + filePart.setContentDisposition(FormDataContentDisposition.name("file").fileName("db_0.jar").build()); + mp.bodyPart(filePart); + + APIResult result = target.path("databases/jar").request() + .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class); + return result; + } } diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java index c11fd831a..967ec8ba6 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java @@ -18,6 +18,7 @@ */ package org.apache.lens.server.api.metastore; +import java.io.InputStream; import java.util.Date; import java.util.List; @@ -570,4 +571,6 @@ void updatePartition(LensSessionHandle sessionid, String tblName, String storage void updatePartitions(LensSessionHandle sessionid, String tblName, String storageName, XPartitionList partitions) throws LensException; + + void addDBJar(LensSessionHandle sessionid, String type, InputStream is) throws LensException; } diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index a1acd1a7a..ba98abdd1 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -20,6 +20,7 @@ import static org.apache.lens.server.metastore.JAXBUtils.*; +import java.io.*; import java.util.*; import javax.ws.rs.BadRequestException; @@ -37,14 +38,17 @@ import org.apache.lens.server.api.metastore.CubeMetastoreService; import org.apache.lens.server.session.LensSessionImpl; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.service.cli.CLIService; import com.google.common.collect.Lists; @@ -823,6 +827,88 @@ public void updatePartitions(LensSessionHandle sessionid, String tblName, String } } + @Override + public void addDBJar(LensSessionHandle sessionid, String type, InputStream is) throws LensException { + // Read list of databases in + FileSystem serverFs = null; + FileSystem jarOrderFs = null; + FSDataOutputStream fos = null; + try { + acquire(sessionid); + String currentDB = SessionState.get().getCurrentDatabase(); + + String resTopDir = + getHiveConf().get(currentDB, LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR); + log.info("Database specific resources at {}", resTopDir); + + Path resTopDirPath = new Path(resTopDir); + serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf()); + if (!serverFs.exists(resTopDirPath)) { + log.warn("Database resource location does not exist - {}. Database jar can't be uploaded", resTopDir); + throw new LensException("Database resource location does not exist - {}. Database jar can't be uploaded"); + } + + Path resJarOrderPath = new Path(resTopDir, currentDB + File.separator + "jar_order"); + jarOrderFs = FileSystem.newInstance(resJarOrderPath.toUri(), getHiveConf()); + if (jarOrderFs.exists(resJarOrderPath)) { + log.warn("Database jar_order file exist - {}. Database jar can't be uploaded", resTopDir); + throw new LensException("Database jar_order file exist - {}. Database jar can't be uploaded"); + } + + int lastIndex = 0; + + Path dbFolderPath = new Path(resTopDir, currentDB); + FileStatus[] existingFiles = serverFs.listStatus(dbFolderPath); + for (FileStatus fs : existingFiles) { + String fPath = fs.getPath().getName(); + String[] tokens = fPath.split("_"); + + if (tokens.length > 1) { + int fIndex = Integer.parseInt(tokens[1].substring(0, 1)); + if (fIndex > lastIndex) + lastIndex = fIndex; + } + } + + int newIndex = lastIndex + 1; + + Path resJarPath = new Path(resTopDir, currentDB + File.separator +currentDB + "_" + newIndex + ".jar"); + log.info("new jar name : " + resJarPath.getName()); + fos = serverFs.create(resJarPath); + IOUtils.copy(is, fos); + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (fos != null) { + try { + fos.close(); + } catch (IOException e) { + log.error("Error closing file system instance fos", e); + } + } + + if (serverFs != null) { + try { + serverFs.close(); + } catch (IOException e) { + log.error("Error closing file system instance serverFs", e); + } + } + + if (jarOrderFs != null) { + try { + jarOrderFs.close(); + } catch (IOException e) { + log.error("Error closing file system instance jarOrderFs", e); + } + } + release(sessionid); + } + } + @Override public int addPartitionsToDimStorage(LensSessionHandle sessionid, String dimTblName, String storageName, XPartitionList partitions) throws LensException { @@ -1467,4 +1553,5 @@ public HealthStatus getHealthStatus() { : new HealthStatus(isHealthy, details.toString()); } + } diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index e7d545cfb..248c1dd51 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -20,6 +20,7 @@ import static org.apache.lens.api.APIResult.*; +import java.io.*; import java.util.List; import javax.ws.rs.*; @@ -35,11 +36,15 @@ import org.apache.lens.server.LensServices; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.CubeMetastoreService; +import org.apache.lens.server.util.ScannedPaths; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.glassfish.jersey.media.multipart.FormDataParam; + import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; @@ -1700,4 +1705,62 @@ public StringList getPartitionTimelines(@QueryParam("sessionid") LensSessionHand throw exc; } } + + /** + * Validate jar + *

+ *

+ * The returned @{link APIResult} will have status SUCCEEDED only if the add operation was successful for all + * services running in this Lens server. + *

+ * + * @param size size of the jar + * @param type The type of resource. Valid types are 'jar' + * @param path path of the resource. Local or HDFS path + * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if add was successful. {@link APIResult} with state + * {@link Status#PARTIAL}, if add succeeded only for some services. {@link APIResult} with state + * {@link Status#FAILED}, if add has failed + */ + @PUT + @Path("databases/validateDBJar") + @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN}) + public APIResult validateDBResource(@QueryParam("size") String size, + @QueryParam("type") String type, @QueryParam("path") String path) { + + + return new APIResult(Status.SUCCEEDED, "Add resource succeeded"); + } + + + /** + * Add a resource to the current DB + *

+ *

+ * The returned @{link APIResult} will have status SUCCEEDED only if the add operation was successful for all + * services running in this Lens server. + *

+ * + * @param sessionid session handle object + * @param type The type of resource. Valid types are 'jar' + * @param fileInputStream stream of the resource. Local or HDFS path + * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if add was successful. {@link APIResult} with state + * {@link Status#PARTIAL}, if add succeeded only for some services. {@link APIResult} with state + * {@link Status#FAILED}, if add has failed + */ + @POST + @Path("databases/jar") + @Consumes({MediaType.MULTIPART_FORM_DATA}) + @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN}) + public APIResult addDBResource(@FormDataParam("sessionid") LensSessionHandle sessionid, + @FormDataParam("type") String type, @FormDataParam("file") InputStream fileInputStream) { + + try { + getSvc().addDBJar(sessionid,type,fileInputStream); + } catch (LensException e) { + e.printStackTrace(); + return new APIResult(Status.FAILED, "Add resource failed"); + } + return new APIResult(Status.SUCCEEDED, "Add resource succeeded"); + } + } From 5fd54f7243c537c37c0575aa21a8dbd8b9cceb51 Mon Sep 17 00:00:00 2001 From: badri Date: Thu, 10 Mar 2016 15:20:55 +0530 Subject: [PATCH 2/7] testing done --- .../apache/lens/client/LensConnection.java | 32 +- .../metastore/CubeMetastoreServiceImpl.java | 43 ++- .../server/metastore/MetastoreResource.java | 4 +- .../apache/lens/server/LensJerseyTest.java | 2 +- .../server/metastore/TestDatabaseService.java | 350 ++++++++++++++++++ lens-server/src/test/resources/lens-site.xml | 6 + 6 files changed, 412 insertions(+), 25 deletions(-) create mode 100644 lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java diff --git a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java index bdbfacc25..6fdcf67e8 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java @@ -38,10 +38,7 @@ import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver; import org.apache.lens.client.exceptions.LensClientServerConnectionException; -import org.glassfish.jersey.media.multipart.FormDataBodyPart; -import org.glassfish.jersey.media.multipart.FormDataContentDisposition; -import org.glassfish.jersey.media.multipart.FormDataMultiPart; -import org.glassfish.jersey.media.multipart.MultiPartFeature; +import org.glassfish.jersey.media.multipart.*; import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; import org.glassfish.jersey.moxy.json.MoxyJsonFeature; @@ -340,16 +337,31 @@ public String toString() { public APIResult addResourceToDB(String type, String resourcePath) { WebTarget target = getMetastoreWebTarget(); FormDataMultiPart mp = new FormDataMultiPart(); - mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), this.sessionHandle, - MediaType.APPLICATION_XML_TYPE)); +// mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), this.sessionHandle, +// MediaType.APPLICATION_XML_TYPE)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("type").build(), type)); - FileDataBodyPart filePart = new FileDataBodyPart("file", new File(resourcePath)); - filePart.setContentDisposition(FormDataContentDisposition.name("file").fileName("db_0.jar").build()); + File file = new File(resourcePath); + log.debug("uploading file path : " + file.getAbsolutePath() + "|size = " + file.length()); + final FormDataContentDisposition dispo = FormDataContentDisposition// + .name("file")// + .fileName("db_0.jar")// + .size(file.length())// + .build(); + + FileDataBodyPart filePart = new FileDataBodyPart("file", file); + filePart.setContentDisposition(dispo); mp.bodyPart(filePart); - APIResult result = target.path("databases/jar").request() - .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult result = target.path("databases/jar").queryParam("sessionid", this.sessionHandle).request() + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + + System.out.println(result.getStatus() + " " + result); + return result; } } diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index ba98abdd1..f210ff7c9 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -38,6 +38,7 @@ import org.apache.lens.server.api.metastore.CubeMetastoreService; import org.apache.lens.server.session.LensSessionImpl; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -837,34 +838,46 @@ public void addDBJar(LensSessionHandle sessionid, String type, InputStream is) t acquire(sessionid); String currentDB = SessionState.get().getCurrentDatabase(); - String resTopDir = - getHiveConf().get(currentDB, LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR); - log.info("Database specific resources at {}", resTopDir); + String baseDir = + getHiveConf().get(LensConfConstants.DATABASE_RESOURCE_DIR, LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR); - Path resTopDirPath = new Path(resTopDir); + String dbDir = baseDir+File.separator+currentDB; + log.info("Database specific resources at {}", dbDir); + + + Path resTopDirPath = new Path(dbDir); serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf()); if (!serverFs.exists(resTopDirPath)) { - log.warn("Database resource location does not exist - {}. Database jar can't be uploaded", resTopDir); - throw new LensException("Database resource location does not exist - {}. Database jar can't be uploaded"); + log.warn("Database resource location does not exist. Database jar can't be uploaded", dbDir); + throw new LensException("Database resource location does not exist. Database jar can't be uploaded"); } - Path resJarOrderPath = new Path(resTopDir, currentDB + File.separator + "jar_order"); + Path resJarOrderPath = new Path(dbDir, "jar_order"); jarOrderFs = FileSystem.newInstance(resJarOrderPath.toUri(), getHiveConf()); if (jarOrderFs.exists(resJarOrderPath)) { - log.warn("Database jar_order file exist - {}. Database jar can't be uploaded", resTopDir); - throw new LensException("Database jar_order file exist - {}. Database jar can't be uploaded"); + log.warn("Database jar_order file exist - {}. Database jar can't be uploaded", resJarOrderPath); + throw new LensException("Database jar_order file exist. Database jar can't be uploaded"); + } + + String tempFileName = currentDB+"_uploading.jar"; + + Path uploadingPath = new Path(dbDir, tempFileName); + FileSystem uploadingFs = FileSystem.newInstance(uploadingPath.toUri(), getHiveConf()); + if (uploadingFs.exists(uploadingPath)) { + log.warn("Already uploading a file - {}. This Database jar can't be uploaded. Try later!", uploadingPath); + throw new LensException("Database jar file upload in progress . Database jar can't be uploaded. Try later!"); } int lastIndex = 0; - Path dbFolderPath = new Path(resTopDir, currentDB); + Path dbFolderPath = new Path(baseDir, currentDB); FileStatus[] existingFiles = serverFs.listStatus(dbFolderPath); for (FileStatus fs : existingFiles) { String fPath = fs.getPath().getName(); String[] tokens = fPath.split("_"); if (tokens.length > 1) { - int fIndex = Integer.parseInt(tokens[1].substring(0, 1)); + int fIndex = Integer.parseInt(tokens[tokens.length - 1].substring(0, 1)); if (fIndex > lastIndex) lastIndex = fIndex; } @@ -872,10 +885,16 @@ public void addDBJar(LensSessionHandle sessionid, String type, InputStream is) t int newIndex = lastIndex + 1; - Path resJarPath = new Path(resTopDir, currentDB + File.separator +currentDB + "_" + newIndex + ".jar"); + + Path resJarPath = new Path(baseDir, currentDB + File.separator +tempFileName); log.info("new jar name : " + resJarPath.getName()); fos = serverFs.create(resJarPath); IOUtils.copy(is, fos); + fos.flush(); + + Path renamePath = new Path(baseDir, currentDB + File.separator +currentDB + "_" + newIndex+ ".jar"); + serverFs.rename(resJarPath,renamePath); + } catch (FileNotFoundException e) { e.printStackTrace(); diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index 248c1dd51..4c48fe106 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -1751,14 +1751,14 @@ public APIResult validateDBResource(@QueryParam("size") String size, @Path("databases/jar") @Consumes({MediaType.MULTIPART_FORM_DATA}) @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN}) - public APIResult addDBResource(@FormDataParam("sessionid") LensSessionHandle sessionid, + public APIResult addDBResource(@QueryParam("sessionid") LensSessionHandle sessionid, @FormDataParam("type") String type, @FormDataParam("file") InputStream fileInputStream) { try { getSvc().addDBJar(sessionid,type,fileInputStream); } catch (LensException e) { e.printStackTrace(); - return new APIResult(Status.FAILED, "Add resource failed"); + return new APIResult(Status.FAILED, e.getMessage()); } return new APIResult(Status.SUCCEEDED, "Add resource succeeded"); } diff --git a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java index fb757e467..2bac71069 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensJerseyTest.java @@ -76,7 +76,7 @@ protected URI getUri() { } private boolean isPortAlreadyFound() { - return port != -1; + return port != -1 && port != 0; } public void setUp() throws Exception { diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java new file mode 100644 index 000000000..fc9a8b2ce --- /dev/null +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java @@ -0,0 +1,350 @@ +package org.apache.lens.server.metastore; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; + +import javax.mail.Folder; +import javax.ws.rs.BadRequestException; +import javax.ws.rs.NotFoundException; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; + +import org.apache.lens.api.APIResult; +import org.apache.lens.api.LensSessionHandle; +import org.apache.lens.api.metastore.ObjectFactory; +import org.apache.lens.server.LensJerseyTest; +import org.apache.lens.server.LensServices; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.metastore.CubeMetastoreService; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hive.conf.HiveConf; + +import org.glassfish.jersey.media.multipart.FormDataBodyPart; +import org.glassfish.jersey.media.multipart.FormDataContentDisposition; +import org.glassfish.jersey.media.multipart.FormDataMultiPart; +import org.glassfish.jersey.media.multipart.MultiPart; +import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; +import org.glassfish.jersey.test.TestProperties; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import lombok.extern.slf4j.Slf4j; + +/** + * Created by badrinath.kulkarni on 3/8/16. + */ +@Slf4j +@Test(groups = "unit-test") +public class TestDatabaseService extends LensJerseyTest { + CubeMetastoreServiceImpl metastoreService; + LensSessionHandle lensSessionId; + String rootPath = null; + + + private void assertSuccess(APIResult result) { + assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED, String.valueOf(result)); + } + + @BeforeMethod + public void setUp() throws Exception { + super.setUp(); + rootPath = getServerConf().get(LensConfConstants.DATABASE_RESOURCE_DIR); + metastoreService = LensServices.get().getService(CubeMetastoreService.NAME); + lensSessionId = metastoreService.openSession("foo", "bar", new HashMap()); + } + + @AfterMethod + public void tearDown() throws Exception { + metastoreService.closeSession(lensSessionId); + super.tearDown(); + } + + @Override + protected Application configure() { + enable(TestProperties.LOG_TRAFFIC); + enable(TestProperties.DUMP_ENTITY); + return new MetastoreApp(); + } + + private String getCurrentDatabase(MediaType mediaType) throws Exception { + return target().path("metastore").path("databases/current") + .queryParam("sessionid", lensSessionId).request(mediaType).get(String.class); + } + + private FormDataMultiPart getFormData(MediaType mediaType) { + FormDataMultiPart mp = null; + try { + + mp = new FormDataMultiPart(); + mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("type").build(), "jar")); + + File file = new File("target/testjars/serde.jar"); + log.debug("uploading file path : " + file.getAbsolutePath() + "|size = " + file.length()); + final FormDataContentDisposition dispo = FormDataContentDisposition + .name("file") + .fileName("test.jar") + .size(file.length()) + .build(); + + FileDataBodyPart filePart = new FileDataBodyPart("file", file); + filePart.setContentDisposition(dispo); + mp.bodyPart(filePart); + + + } catch (Exception e) { + e.printStackTrace(); + } finally { + + } + + return mp; + + } + + /** + * Test case when no db folder exists + * + * @param mediaType + * @throws Exception + */ + @Test(dataProvider = "mediaTypeData") + public void testJarUpload(MediaType mediaType) throws Exception { + String dbName = "db1" + "_" + mediaType.getSubtype(); + + // create + APIResult result = target().path("metastore").path("databases") + .queryParam("sessionid", lensSessionId).request(mediaType).post(getEntityForString(dbName, mediaType), APIResult + .class); + assertNotNull(result); + assertSuccess(result); + + // set + WebTarget dbTarget = target().path("metastore").path("databases/current"); + + result = dbTarget.queryParam("sessionid", lensSessionId).request(mediaType) + .put(getEntityForString(dbName, mediaType), APIResult.class); + assertNotNull(result); + assertSuccess(result); + + + FormDataMultiPart mp = getFormData(mediaType); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult resultUpd = target().path("metastore").path("databases/jar"). + queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + System.out.println(resultUpd.getStatus() + " " + resultUpd); + assertEquals(resultUpd.getMessage(), "Database resource location does not exist. Database jar can't be uploaded"); + + + } + + + /** + * Test case when db folder exists & jar_order file present ( Existing flow ) + * + * @param mediaType + * @throws Exception + */ + @Test(dataProvider = "mediaTypeData") + public void testJarUploadWithJarOrderFileInDbFolder(MediaType mediaType) throws Exception { + String dbName = "db2" + "_" + mediaType.getSubtype() + "_" + System.currentTimeMillis(); + + // create + APIResult result = target().path("metastore").path("databases") + .queryParam("sessionid", lensSessionId).request(mediaType).post(getEntityForString(dbName, mediaType), APIResult + .class); + assertNotNull(result); + assertSuccess(result); + + // set + WebTarget dbTarget = target().path("metastore").path("databases/current"); + + result = dbTarget.queryParam("sessionid", lensSessionId).request(mediaType) + .put(getEntityForString(dbName, mediaType), APIResult.class); + assertNotNull(result); + assertSuccess(result); + + // Create DB folder + File dbFolder = new File("target/resources/" + dbName); + dbFolder.mkdirs(); + + File dbFolderJarOrder = new File("target/resources/" + dbName + File.separator + "jar_order"); + dbFolderJarOrder.createNewFile(); + + + FormDataMultiPart mp = getFormData(mediaType); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult resultUpd = target().path("metastore").path("databases/jar"). + queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + System.out.println(resultUpd.getStatus() + " " + resultUpd); + assertEquals(resultUpd.getMessage(), "Database jar_order file exist. Database jar can't be uploaded"); + + cleanUp(dbFolder); + } + + + /** + * Test case when db folder exists & jar_order file NOT present & db_uploading.jar present. + * This restricts single upload ant any time. + * + * @param mediaType + * @throws Exception + */ + @Test(dataProvider = "mediaTypeData") + public void testJarUploadWithDbUploadingJarInFolder(MediaType mediaType) throws Exception { + String dbName = "db3" + "_" + mediaType.getSubtype() + "_" + System.currentTimeMillis(); + + // create + APIResult result = target().path("metastore").path("databases") + .queryParam("sessionid", lensSessionId).request(mediaType).post(getEntityForString(dbName, mediaType), APIResult + .class); + assertNotNull(result); + assertSuccess(result); + + // set + WebTarget dbTarget = target().path("metastore").path("databases/current"); + + result = dbTarget.queryParam("sessionid", lensSessionId).request(mediaType) + .put(getEntityForString(dbName, mediaType), APIResult.class); + assertNotNull(result); + assertSuccess(result); + + // Create DB folder + File dbFolder = new File("target/resources/" + dbName); + dbFolder.mkdirs(); + + File dbFolderJarOrder = new File("target/resources/" + dbName + File.separator + dbName+"_uploading.jar"); + dbFolderJarOrder.createNewFile(); + + FormDataMultiPart mp = getFormData(mediaType); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult resultUpd = target().path("metastore").path("databases/jar"). + queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + System.out.println(resultUpd.getStatus() + " " + resultUpd); + assertEquals(resultUpd.getMessage(), "Database jar file upload in progress . Database jar can't be uploaded. Try later!"); + + cleanUp(dbFolder); + } + + + /** + * Test case when db folder exists & jar_order file NOT present & db_uploading.jar NOT present. + * + * @param mediaType + * @throws Exception + */ + @Test(dataProvider = "mediaTypeData") + public void testJarUploadWithNoJarOrderInFolder(MediaType mediaType) throws Exception { + String dbName = "db4" + "_" + mediaType.getSubtype() + "_" + System.currentTimeMillis(); + + // create + APIResult result = target().path("metastore").path("databases") + .queryParam("sessionid", lensSessionId).request(mediaType).post(getEntityForString(dbName, mediaType), APIResult + .class); + assertNotNull(result); + assertSuccess(result); + + // set + WebTarget dbTarget = target().path("metastore").path("databases/current"); + + result = dbTarget.queryParam("sessionid", lensSessionId).request(mediaType) + .put(getEntityForString(dbName, mediaType), APIResult.class); + assertNotNull(result); + assertSuccess(result); + + // Create DB folder + File dbFolder = new File("target/resources/" + dbName); + dbFolder.mkdirs(); + + FormDataMultiPart mp = getFormData(mediaType); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult resultUpd = target().path("metastore").path("databases/jar"). + queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + System.out.println(resultUpd.getStatus() + " " + resultUpd); + assertEquals(resultUpd.getStatus(), APIResult.Status.SUCCEEDED); + + cleanUp(dbFolder); + } + + /** + * Test case when db folder exists & jar_order file NOT present & db_uploading.jar NOT present and with existing jars. + * + * @param mediaType + * @throws Exception + */ + @Test(dataProvider = "mediaTypeData") + public void testJarUploadWithExistingJarsInFolder(MediaType mediaType) throws Exception { + String dbName = "db5" + "_" + mediaType.getSubtype() + "_" + System.currentTimeMillis(); + + // create + APIResult result = target().path("metastore").path("databases") + .queryParam("sessionid", lensSessionId).request(mediaType).post(getEntityForString(dbName, mediaType), APIResult + .class); + assertNotNull(result); + assertSuccess(result); + + // set + WebTarget dbTarget = target().path("metastore").path("databases/current"); + + result = dbTarget.queryParam("sessionid", lensSessionId).request(mediaType) + .put(getEntityForString(dbName, mediaType), APIResult.class); + assertNotNull(result); + assertSuccess(result); + + // Create DB folder + File dbFolder = new File("target/resources/" + dbName); + dbFolder.mkdirs(); + + File dbFolderJarOrder1 = new File("target/resources/" + dbName + File.separator + dbName + "_1.jar"); + dbFolderJarOrder1.createNewFile(); + + File dbFolderJarOrder2 = new File("target/resources/" + dbName + File.separator + dbName + "_2.jar"); + dbFolderJarOrder2.createNewFile(); + + FormDataMultiPart mp = getFormData(mediaType); + MultiPart multiPart = new MultiPart(); + multiPart.setMediaType(MediaType.MULTIPART_FORM_DATA_TYPE); + + + APIResult resultUpd = target().path("metastore").path("databases/jar"). + queryParam("sessionid", lensSessionId).request(mediaType) + .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); + System.out.println(resultUpd.getStatus() + " " + resultUpd); + assertEquals(resultUpd.getStatus(), APIResult.Status.SUCCEEDED); + + cleanUp(dbFolder); + } + + private void cleanUp(File f) { + try { + //FileUtils.deleteDirectory(f); + } catch (Exception e) { + e.printStackTrace(); + } + } + +} diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml index c3187a8de..6a2660e5e 100644 --- a/lens-server/src/test/resources/lens-site.xml +++ b/lens-server/src/test/resources/lens-site.xml @@ -45,6 +45,12 @@ Enable metrics to be reported on console + + lens.server.database.resource.dir + ${project.build.directory}/resources/ + database resource directory + + lens.server.persist.location target/persist-dir From 30c48b873020a41df618bd06f2760febc3928542 Mon Sep 17 00:00:00 2001 From: badri Date: Wed, 16 Mar 2016 18:10:31 +0530 Subject: [PATCH 3/7] Load jars on start --- .../session/DatabaseResourceService.java | 170 ++++++++++++++++-- .../apache/lens/server/util/ScannedPaths.java | 21 ++- 2 files changed, 172 insertions(+), 19 deletions(-) diff --git a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java index 72f5c534b..e188b55b7 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java @@ -18,6 +18,7 @@ */ package org.apache.lens.server.session; +import java.io.File; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; @@ -47,6 +48,7 @@ public class DatabaseResourceService extends AbstractService { public static final String NAME = "database-resources"; private Map classLoaderCache; private Map> dbResEntryMap; + private List commonResMap; /** * The metrics service. @@ -72,11 +74,17 @@ public DatabaseResourceService(String name) { super(name); } + private String resTopDir = null; + @Override public synchronized void init(HiveConf hiveConf) { super.init(hiveConf); classLoaderCache = new HashMap<>(); dbResEntryMap = new HashMap<>(); + commonResMap = new LinkedList<>(); + + resTopDir = + hiveConf.get(LensConfConstants.DATABASE_RESOURCE_DIR, LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR); } @Override @@ -85,8 +93,43 @@ public synchronized void start() { try { log.info("Starting loading DB specific resources"); - loadDbResourceEntries(); - loadResources(); + + // If the base folder itself is not present, then return as we can't load any jars. + FileSystem serverFs = null; + try { + Path resTopDirPath = new Path(resTopDir); + serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf()); + if (!serverFs.exists(resTopDirPath)) { + incrCounter(LOAD_RESOURCES_ERRORS); + log.warn("DB base location does not exist.", resTopDir); + return; + } + } catch (Exception io) { + log.error("Error locating base directory", io); + throw new LensException(io); + } finally { + if (serverFs != null) { + try { + serverFs.close(); + } catch (IOException e) { + log.error("Error closing file system instance", e); + } + } + } + + + // Map common jars available for all DB folders + mapCommonResourceEntries(); + + // Map DB specific jar + mapDbResourceEntries(); + + // Load all the jars for all the DB's mapped in previous steps + loadMappedResources(); + + + + } catch (LensException e) { incrCounter(LOAD_RESOURCES_ERRORS); log.warn("Failed to load DB resource mapping, resources must be added explicitly to session."); @@ -99,22 +142,67 @@ public synchronized void stop() { super.stop(); classLoaderCache.clear(); dbResEntryMap.clear(); + commonResMap.clear(); } - private void loadDbResourceEntries() throws LensException { + private void mapCommonResourceEntries() { + // Check if order file is present in the directory + List jars = new ScannedPaths(new Path(resTopDir), "jar", false).getFinalPaths(); + for (String resPath : jars) { + Path jarFilePath = new Path(resPath); + commonResMap.add(new LensSessionImpl.ResourceEntry("jar", jarFilePath.toUri().toString())); + } + + } + + private void mapDbResourceEntries(String dbName) throws LensException { // Read list of databases in FileSystem serverFs = null; try { - String resTopDir = + + Path resDirPath = new Path(resTopDir, dbName); + serverFs = FileSystem.newInstance(resDirPath.toUri(), getHiveConf()); + if (!serverFs.exists(resDirPath)) { + incrCounter(LOAD_RESOURCES_ERRORS); + log.warn("Database resource location does not exist - {}. Database jars will not be available", resDirPath); + + // remove the db entry + dbResEntryMap.remove(dbName); + classLoaderCache.remove(dbName); + return; + } + + findResourcesInDir(serverFs, dbName, resDirPath); + log.debug("Found resources {}", dbResEntryMap); + } catch (IOException io) { + log.error("Error getting list of dbs to load resources from", io); + throw new LensException(io); + } finally { + if (serverFs != null) { + try { + serverFs.close(); + } catch (IOException e) { + log.error("Error closing file system instance", e); + } + } + } + } + + private void mapDbResourceEntries() throws LensException { + // Read list of databases in + FileSystem serverFs = null; + + try { + String baseDir = getHiveConf().get(LensConfConstants.DATABASE_RESOURCE_DIR, LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR); - log.info("Database specific resources at {}", resTopDir); + log.info("Database specific resources at {}", baseDir); - Path resTopDirPath = new Path(resTopDir); + Path resTopDirPath = new Path(baseDir); serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf()); if (!serverFs.exists(resTopDirPath)) { incrCounter(LOAD_RESOURCES_ERRORS); - log.warn("Database resource location does not exist - {}. Database jars will not be available", resTopDir); + log.warn("DB base location does not exist.", baseDir); return; } @@ -123,9 +211,21 @@ private void loadDbResourceEntries() throws LensException { Path dbDirPath = dbDir.getPath(); if (serverFs.isDirectory(dbDirPath)) { String dbName = dbDirPath.getName(); - // Get all resources for that db - findResourcesInDir(serverFs, dbName, dbDirPath); + + Path dbJarOrderPath = new Path(baseDir, dbName+ File.separator+"jar_order"); + if(serverFs.exists(dbJarOrderPath)){ + // old flow + mapDbResourceEntries(dbName); + }else{ + // new flow + mapDbSpecificJar(baseDir, dbName); + } + + } else { + + // It has to be a common jar, so load it. + log.warn("DB resource DIR is not a directory: {}", dbDirPath); } } @@ -134,6 +234,53 @@ private void loadDbResourceEntries() throws LensException { } catch (IOException io) { log.error("Error getting list of dbs to load resources from", io); throw new LensException(io); + }catch (Exception e){ + log.error("Exception : ",e); + } + finally { + if (serverFs != null) { + try { + serverFs.close(); + } catch (IOException e) { + log.error("Error closing file system instance", e); + } + } + } + } + + private void mapDbSpecificJar(String baseDir, String dbName) throws Exception{ + FileSystem serverFs = null; + try{ + int lastIndex = 0; + + + Path dbFolderPath = new Path(baseDir, dbName); + + serverFs = FileSystem.newInstance(dbFolderPath.toUri(), getHiveConf()); + FileStatus[] existingFiles = serverFs.listStatus(dbFolderPath); + for (FileStatus fs : existingFiles) { + String fPath = fs.getPath().getName(); + String[] tokens = fPath.split("_"); + + if (tokens.length > 1) { + int fIndex = Integer.parseInt(tokens[tokens.length - 1].substring(0, 1)); + if (fIndex > lastIndex) + lastIndex = fIndex; + } + } + + // latest jar + Path latestJarPath = new Path(baseDir, dbName + File.separator +dbName + "_" + lastIndex+ ".jar"); + addResourceEntry(new LensSessionImpl.ResourceEntry("jar", latestJarPath.toUri().toString()), dbName); + + // add common jars + for(LensSessionImpl.ResourceEntry jar : commonResMap){ + addResourceEntry(jar,dbName); + } + + }catch (Exception io) { + log.error("Error getting list of dbs to load resources from", io); + throw new Exception(io); } finally { if (serverFs != null) { try { @@ -143,8 +290,11 @@ private void loadDbResourceEntries() throws LensException { } } } + } + + private void findResourcesInDir(FileSystem serverFs, String database, Path dbDirPath) throws IOException { // Check if order file is present in the directory List jars = new ScannedPaths(dbDirPath, "jar").getFinalPaths(); @@ -197,7 +347,7 @@ private void addResourceEntry(LensSessionImpl.ResourceEntry entry, String dbName /** * Load DB specific resources */ - public void loadResources() { + public void loadMappedResources() { for (String db : dbResEntryMap.keySet()) { try { createClassLoader(db); diff --git a/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java b/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java index e48eab4e2..4007f0a70 100644 --- a/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java +++ b/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java @@ -46,12 +46,13 @@ public class ScannedPaths implements Iterable { private final Path path; private final String type; + private boolean recurse; /* The Chosen Ones */ - @Getter(lazy = true) private final List finalPaths = getMatchedPaths(path, type); + @Getter(lazy = true) private final List finalPaths = getMatchedPaths(path, type, recurse); - public ScannedPaths(String path, String type) { - this(new Path(path), type); + public ScannedPaths(Path path, String type) { + this(path, type, true); } @@ -70,7 +71,7 @@ public Iterator iterator() { * * Updates finalPaths List with matched paths and returns an iterator for matched paths. */ - private List getMatchedPaths(Path pt, String type) { + private List getMatchedPaths(Path pt, String type, boolean recurse) { List finalPaths = new ArrayList<>(); InputStream resourceOrderIStream = null; FileSystem fs; @@ -102,8 +103,10 @@ private List getMatchedPaths(Path pt, String type) { statuses = fs.globStatus(new Path(pt, "*")); if (statuses != null) { for (FileStatus st : statuses) { - newMatches = getMatchedPaths(st.getPath(), type); - finalPaths.addAll(newMatches); + if (!st.isDirectory() || recurse) { + newMatches = getMatchedPaths(st.getPath(), type, recurse); + finalPaths.addAll(newMatches); + } } } } @@ -120,9 +123,9 @@ private List getMatchedPaths(Path pt, String type) { /** Get matched resources recursively for provided path/pattern **/ if (resource.startsWith("/") || resource.contains(":/")) { - newMatches = getMatchedPaths(new Path(resource), type); + newMatches = getMatchedPaths(new Path(resource), type, true); } else { - newMatches = getMatchedPaths(new Path(pt, resource), type); + newMatches = getMatchedPaths(new Path(pt, resource), type, true); } finalPaths.addAll(newMatches); } @@ -135,7 +138,7 @@ private List getMatchedPaths(Path pt, String type) { FileStatus[] statuses = fs.globStatus(Path.getPathWithoutSchemeAndAuthority(pt)); if (statuses != null) { for (FileStatus st : statuses) { - List newMatches = getMatchedPaths(st.getPath(), type); + List newMatches = getMatchedPaths(st.getPath(), type, recurse); finalPaths.addAll(newMatches); } } From 9f02e03dd91f0afc8feb045ccc085f29e67005a5 Mon Sep 17 00:00:00 2001 From: badri Date: Fri, 18 Mar 2016 17:25:40 +0530 Subject: [PATCH 4/7] Test cases implemented. --- .../apache/lens/client/LensConnection.java | 6 +- .../metastore/CubeMetastoreServiceImpl.java | 2 + .../session/DatabaseResourceService.java | 28 +-- .../apache/lens/server/util/ScannedPaths.java | 4 + .../lens/server/LensServerTestUtil.java | 175 ++++++++++++++- .../session/TestDatabaseResourceService.java | 201 +++++++++++++++++- 6 files changed, 378 insertions(+), 38 deletions(-) diff --git a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java index 6fdcf67e8..19299b2f4 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensConnection.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensConnection.java @@ -337,15 +337,13 @@ public String toString() { public APIResult addResourceToDB(String type, String resourcePath) { WebTarget target = getMetastoreWebTarget(); FormDataMultiPart mp = new FormDataMultiPart(); -// mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), this.sessionHandle, -// MediaType.APPLICATION_XML_TYPE)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("type").build(), type)); File file = new File(resourcePath); log.debug("uploading file path : " + file.getAbsolutePath() + "|size = " + file.length()); final FormDataContentDisposition dispo = FormDataContentDisposition// .name("file")// - .fileName("db_0.jar")// + .fileName("db_0.jar")// temp dummy name .size(file.length())// .build(); @@ -360,7 +358,7 @@ public APIResult addResourceToDB(String type, String resourcePath) { APIResult result = target.path("databases/jar").queryParam("sessionid", this.sessionHandle).request() .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); - System.out.println(result.getStatus() + " " + result); + log.debug(result.getStatus() + " - " + result); return result; } diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index f210ff7c9..ba75aec07 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -898,8 +898,10 @@ public void addDBJar(LensSessionHandle sessionid, String type, InputStream is) t } catch (FileNotFoundException e) { e.printStackTrace(); + throw new LensException(e); } catch (IOException e) { e.printStackTrace(); + throw new LensException(e); } finally { if (fos != null) { try { diff --git a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java index e188b55b7..877974ff0 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java @@ -128,8 +128,6 @@ public synchronized void start() { loadMappedResources(); - - } catch (LensException e) { incrCounter(LOAD_RESOURCES_ERRORS); log.warn("Failed to load DB resource mapping, resources must be added explicitly to session."); @@ -212,21 +210,14 @@ private void mapDbResourceEntries() throws LensException { if (serverFs.isDirectory(dbDirPath)) { String dbName = dbDirPath.getName(); - Path dbJarOrderPath = new Path(baseDir, dbName+ File.separator+"jar_order"); - if(serverFs.exists(dbJarOrderPath)){ + Path dbJarOrderPath = new Path(baseDir, dbName + File.separator + "jar_order"); + if (serverFs.exists(dbJarOrderPath)) { // old flow mapDbResourceEntries(dbName); - }else{ + } else { // new flow mapDbSpecificJar(baseDir, dbName); } - - - } else { - - // It has to be a common jar, so load it. - - log.warn("DB resource DIR is not a directory: {}", dbDirPath); } } @@ -269,9 +260,12 @@ private void mapDbSpecificJar(String baseDir, String dbName) throws Exception{ } } - // latest jar - Path latestJarPath = new Path(baseDir, dbName + File.separator +dbName + "_" + lastIndex+ ".jar"); - addResourceEntry(new LensSessionImpl.ResourceEntry("jar", latestJarPath.toUri().toString()), dbName); + if(lastIndex > 0){ + // latest jar + Path latestJarPath = new Path(baseDir, dbName + File.separator +dbName + "_" + lastIndex+ ".jar"); + addResourceEntry(new LensSessionImpl.ResourceEntry("jar", latestJarPath.toUri().toString()), dbName); + } + // add common jars for(LensSessionImpl.ResourceEntry jar : commonResMap){ @@ -279,8 +273,8 @@ private void mapDbSpecificJar(String baseDir, String dbName) throws Exception{ } }catch (Exception io) { - log.error("Error getting list of dbs to load resources from", io); - throw new Exception(io); + log.error("Error getting db specific resource ", io); + throw new LensException(io); } finally { if (serverFs != null) { try { diff --git a/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java b/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java index 4007f0a70..449f1d89c 100644 --- a/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java +++ b/lens-server/src/main/java/org/apache/lens/server/util/ScannedPaths.java @@ -51,6 +51,10 @@ public class ScannedPaths implements Iterable { /* The Chosen Ones */ @Getter(lazy = true) private final List finalPaths = getMatchedPaths(path, type, recurse); + public ScannedPaths(String path, String type) { + this(new Path(path), type, true); + } + public ScannedPaths(Path path, String type) { this(path, type, true); } diff --git a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java index b651b7916..0a34a6afc 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java @@ -62,6 +62,7 @@ public final class LensServerTestUtil { public static final String DB_WITH_JARS = "test_db_static_jars"; public static final String DB_WITH_JARS_2 = "test_db_static_jars_2"; + private LensServerTestUtil() { } @@ -93,8 +94,9 @@ public static void createTable(String tblName, WebTarget parent, LensSessionHand mt)); final QueryHandle handle = target.request(mt) - .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType>() {}).getData(); + .post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), + new GenericType>() { + }).getData(); // wait till the query finishes LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) .get(LensQuery.class); @@ -105,7 +107,7 @@ public static void createTable(String tblName, WebTarget parent, LensSessionHand stat = ctx.getStatus(); Thread.sleep(1000); } - final String debugHelpMsg = "Query Handle:"+ctx.getQueryHandleString(); + final String debugHelpMsg = "Query Handle:" + ctx.getQueryHandleString(); assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, debugHelpMsg); assertTrue(ctx.getSubmissionTime() > 0, debugHelpMsg); assertTrue(ctx.getLaunchTime() > 0, debugHelpMsg); @@ -120,7 +122,7 @@ public static void createTable(String tblName, WebTarget parent, LensSessionHand } public static void loadData(String tblName, final String testDataFile, WebTarget parent, - LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { + LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { LensConf conf = new LensConf(); conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false"); final WebTarget target = parent.path("queryapi/queries"); @@ -129,18 +131,19 @@ public static void loadData(String tblName, final String testDataFile, WebTarget String dataLoad = "LOAD DATA LOCAL INPATH '" + testDataFile + "' OVERWRITE INTO TABLE " + tblName; mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId, - mt)); + mt)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), dataLoad)); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute")); mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf, - mt)); + mt)); final QueryHandle handle = target.request(mt).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType>() {}).getData(); + new GenericType>() { + }).getData(); // wait till the query finishes LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) - .get(LensQuery.class); + .get(LensQuery.class); QueryStatus stat = ctx.getStatus(); while (!stat.finished()) { ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) @@ -150,6 +153,7 @@ public static void loadData(String tblName, final String testDataFile, WebTarget } assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL); } + /** * Load data. * @@ -160,7 +164,7 @@ public static void loadData(String tblName, final String testDataFile, WebTarget * @throws InterruptedException the interrupted exception */ public static void loadDataFromClasspath(String tblName, final String testDataFile, WebTarget parent, - LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { + LensSessionHandle lensSessionId, MediaType mt) throws InterruptedException { String absolutePath = LensServerTestUtil.class.getClassLoader().getResource(testDataFile).getPath(); loadData(tblName, absolutePath, parent, lensSessionId, mt); @@ -206,7 +210,8 @@ public static void dropTableWithConf(String tblName, WebTarget parent, LensSessi mt)); final QueryHandle handle = target.request(mt).post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE), - new GenericType>() {}).getData(); + new GenericType>() { + }).getData(); // wait till the query finishes LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request(mt) @@ -261,7 +266,7 @@ public static void createTestDatabaseResources(String[] testDatabases, HiveConf // nothing to setup return; } - File resDir = new File("target/resources"); + File resDir = new File(conf.get(LensConfConstants.DATABASE_RESOURCE_DIR)); if (!resDir.exists()) { resDir.mkdir(); } @@ -302,4 +307,152 @@ public static void createTestDatabaseResources(String[] testDatabases, HiveConf } } } + + + /** + * Test case when db folder exists & jar_order file present ( Existing flow ) + * + * @throws Exception + */ + public static void createTestDbWithoutCommonJars(String[] dbs, HiveConf conf) throws Exception { + File srcJarDir = new File("target/testjars/"); + if (!srcJarDir.exists()) { + // nothing to setup + return; + } + File resDir = new File(conf.get(LensConfConstants.DATABASE_RESOURCE_DIR)); + if (!resDir.exists()) { + resDir.mkdir(); + } + + setUpDbWithJarOrderAndFiles(dbs[0], resDir, conf); + + setUpDbWithNoJarOrderAndVersionFiles(dbs[1], resDir, conf); + + setUpDbWithNoJarOrderAndNoFiles(dbs[2], resDir, conf); + } + + + public static void createTestDbWithCommonJars(String[] dbs, HiveConf conf) throws Exception { + File srcJarDir = new File("target/testjars/"); + if (!srcJarDir.exists()) { + // nothing to setup + return; + } + File resDir = new File(conf.get(LensConfConstants.DATABASE_RESOURCE_DIR)); + if (!resDir.exists()) { + resDir.mkdir(); + } + + File testJarFile = new File("target/testjars/test.jar"); + + // Copy common jars + FileUtils.copyFile(testJarFile, new File(resDir, "lens-ship.jar")); + FileUtils.copyFile(testJarFile, new File(resDir, "meta.jar")); + + + setUpDbWithJarOrderAndFiles(dbs[0], resDir, conf); + + setUpDbWithNoJarOrderAndVersionFiles(dbs[1], resDir, conf); + + setUpDbWithNoJarOrderAndNoFiles(dbs[2], resDir, conf); + } + + //********************************************* + // Setup DB : JAR_ORDER + SOME FILES + //********************************************* + private static void setUpDbWithJarOrderAndFiles(String db, File resDir, HiveConf conf) throws Exception { + Database database0 = new Database(); + database0.setName(db); + + Hive hive = Hive.get(conf); + hive.createDatabase(database0, true); + + File dbDir1 = new File(resDir, db); + if (!dbDir1.exists()) { + dbDir1.mkdir(); + } + + File dbDir0 = new File(resDir, db); + if (!dbDir0.exists()) { + dbDir0.mkdir(); + } + + String[] jarFilesOrder = { + "x_" + db + ".jar", + "y_" + db + ".jar", + "z_" + db + ".jar", + }; + + try { + + // We are explicitly specifying jar order + FileUtils.writeLines(new File(dbDir0, "jar_order"), Arrays.asList(jarFilesOrder[2], jarFilesOrder[1], + jarFilesOrder[0])); + + File testJarFile = new File("target/testjars/test.jar"); + + FileUtils.copyFile(testJarFile, new File(dbDir0, jarFilesOrder[0])); + FileUtils.copyFile(testJarFile, new File(dbDir0, jarFilesOrder[1])); + FileUtils.copyFile(testJarFile, new File(dbDir0, jarFilesOrder[2])); + } catch (FileNotFoundException fnf) { + log.error("File not found.", fnf); + } catch (Exception e) { + log.error("File not found.", e); + } + } + + + //********************************************* + // Setup DB : NO JAR_ORDER + VERSION FILES + //********************************************* + private static void setUpDbWithNoJarOrderAndVersionFiles(String db, File resDir, HiveConf conf) throws Exception { + + Database database0 = new Database(); + database0.setName(db); + + Hive hive = Hive.get(conf); + hive.createDatabase(database0, true); + + File dbDir1 = new File(resDir, db); + if (!dbDir1.exists()) { + dbDir1.mkdir(); + } + + String[] jarOrder = { + db + "_3.jar", + db + "_2.jar", + db + "_1.jar" + }; + + + try { + File testJarFile = new File("target/testjars/test.jar"); + + FileUtils.copyFile(testJarFile, new File(dbDir1, jarOrder[0])); + FileUtils.copyFile(testJarFile, new File(dbDir1, jarOrder[1])); + FileUtils.copyFile(testJarFile, new File(dbDir1, jarOrder[2])); + } catch (FileNotFoundException fnf) { + log.error("File not found.", fnf); + } + } + + //********************************************* + // Setup DB : NO JAR_ORDER + NO FILES + NO VERSION FILES + //********************************************* + private static void setUpDbWithNoJarOrderAndNoFiles(String db, File resDir, HiveConf conf) throws Exception { + + Database database0 = new Database(); + database0.setName(db); + + Hive hive = Hive.get(conf); + hive.createDatabase(database0, true); + + File dbDir1 = new File(resDir, db); + if (!dbDir1.exists()) { + dbDir1.mkdir(); + } + } + + } diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java index 1008faf8a..7bea001aa 100644 --- a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java +++ b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java @@ -18,6 +18,7 @@ */ package org.apache.lens.server.session; +import java.io.File; import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; @@ -27,6 +28,7 @@ import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.api.LensConfConstants; +import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -46,27 +48,81 @@ public class TestDatabaseResourceService { private static final String DB1 = DB_PFX + "db1"; private static final String DB2 = DB_PFX + "db2"; +// //tested using dbResService1 +// private static final String DB_TEST_JarUploadWhenDbFolderExists = "db1_" + System.currentTimeMillis(); +// private static final String DB_TEST_JarUploadWithJarOrderFileInDbFolder = "db2_" + System.currentTimeMillis(); +// +// //tested using dbResService2 +// private static final String DB_TEST_JarUploadWithNoJarOrderInDbFolder = "db3_" + System.currentTimeMillis(); +// private static final String DB_TEST_JarUploadWithNoJarOrderAndCommonJarsInBaseFolder = "db4_" + System.currentTimeMillis(); +// private static final String DB_TEST_JarUploadWithJarOrderAndCommonJarsInBaseFolder = "db5_" + System.currentTimeMillis(); +// private final String[] testDatabases = {DB1, DB2}; + private static final String JAR_ORDER_AND_FILES = DB_PFX + "jar_order_files_in_db"; + private static final String NO_JAR_ORDER_AND_VERSION_FILES = DB_PFX + "no_jar_order_and_version_files"; + private static final String NO_JAR_ORDER_NO_FILES = DB_PFX + "no_jar_order_no_files"; + + private final String[] testDatabases1 = {JAR_ORDER_AND_FILES, NO_JAR_ORDER_AND_VERSION_FILES,NO_JAR_ORDER_NO_FILES}; + +// private final String[] testDatabases1 = {DB_TEST_JarUploadWhenDbFolderExists, DB_TEST_JarUploadWithJarOrderFileInDbFolder}; +// +// private final String[] testDatabases2 = {DB_TEST_JarUploadWithNoJarOrderInDbFolder, DB_TEST_JarUploadWithNoJarOrderAndCommonJarsInBaseFolder, DB_TEST_JarUploadWithJarOrderAndCommonJarsInBaseFolder}; + private final HiveConf conf = new HiveConf(TestDatabaseResourceService.class); private DatabaseResourceService dbResService; + private final HiveConf conf1 = new HiveConf(TestDatabaseResourceService.class); + private DatabaseResourceService dbResService1; + + private final HiveConf conf2 = new HiveConf(TestDatabaseResourceService.class); + private DatabaseResourceService dbResService2; + @BeforeClass public void setup() throws Exception { + + String prefix = "file://"+System.getProperty("user.dir"); + + conf.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix+"/target/resources"); + conf1.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix+"/target/resources_without_common_jars"); + conf2.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix+"/target/resources_with_common_jars"); + LensServerTestUtil.createTestDatabaseResources(testDatabases, conf); - // Start resource service. - conf.set(LensConfConstants.DATABASE_RESOURCE_DIR, "target/resources"); + LensServerTestUtil.createTestDbWithoutCommonJars(testDatabases1, conf1); + LensServerTestUtil.createTestDbWithCommonJars(testDatabases1, conf2); + dbResService = new DatabaseResourceService(DatabaseResourceService.NAME); dbResService.init(conf); dbResService.start(); + + dbResService1 = new DatabaseResourceService(DatabaseResourceService.NAME); + dbResService1.init(conf1); + dbResService1.start(); + + dbResService2 = new DatabaseResourceService(DatabaseResourceService.NAME); + dbResService2.init(conf2); + dbResService2.start(); + + } @AfterClass public void tearDown() throws Exception { - Hive hive = Hive.get(conf); + Hive hive0 = Hive.get(conf); for (String db : testDatabases) { - hive.dropDatabase(db, true, true); + hive0.dropDatabase(db, true, true); + } + + Hive hive1 = Hive.get(conf1); + for (String db : testDatabases1) { + hive1.dropDatabase(db, true, true); } + + Hive hive2 = Hive.get(conf2); + for (String db : testDatabases1) { + hive2.dropDatabase(db, true, true); + } + } @Test @@ -82,7 +138,7 @@ public void testClassLoaderCreated() throws Exception { private boolean isJarLoaded(ClassLoader loader, String db) throws Exception { URLClassLoader db1Loader = (URLClassLoader) loader; - for (URL url : db1Loader.getURLs()) { + for (URL url : db1Loader.getURLs()) { String jarFile = url.getPath(); if (jarFile.endsWith(db + ".jar")) { log.info("Found jar url " + url.toString()); @@ -132,11 +188,144 @@ public void verifyClassLoader() throws Exception { Class clz = Class.forName("ClassLoaderTestClass", true, getClass().getClassLoader()); Assert.fail("Expected class loading to fail"); } catch (Throwable th) { - log.error("Expected error " + th + " msg = "+th.getMessage(), th); + log.error("Expected error " + th + " msg = " + th.getMessage(), th); } // Should pass now Class clz = Class.forName(TEST_CLASS, true, dbResService.getClassLoader(DB1)); Assert.assertNotNull(clz); } + + + /************************************************** + * Test cases without common jars + **************************************************/ + @Test + public void testDbWithoutCommonJarsAndWithJarOrderAndFiles()throws Exception { + String db = testDatabases1[0]; + Collection actualOrder = dbResService1.getResourcesForDatabase(db); + List actualOrderList = new ArrayList(); + + for (LensSessionImpl.ResourceEntry res : actualOrder) { + actualOrderList.add(res.getLocation()); + } + + String[] jarFilesOrder = { + "z_" + db + ".jar", + "y_" + db + ".jar", + "x_" + db + ".jar", + }; + + // Verify order + for (int i = 0; i < jarFilesOrder.length; i++) { + Assert.assertTrue(actualOrderList.get(i).contains(jarFilesOrder[i]), + actualOrderList.get(i) + " > " + jarFilesOrder[i]); + } + } + + @Test + public void testDbWithoutCommonJarsAndWithNoJarOrderAndVersionFiles()throws Exception { + String db = testDatabases1[1]; + Collection actualOrder = dbResService1.getResourcesForDatabase(db); + List actualOrderList = new ArrayList(); + + for (LensSessionImpl.ResourceEntry res : actualOrder) { + actualOrderList.add(res.getLocation()); + } + + // Should pick the latest one + String[] jarFilesOrder = { + db + "_3.jar", + }; + + // Verify order + for (int i = 0; i < jarFilesOrder.length; i++) { + Assert.assertTrue(actualOrderList.get(i).contains(jarFilesOrder[i]), + actualOrderList.get(i) + " > " + jarFilesOrder[i]); + } + } + + @Test + public void testDbWithoutCommonJarsAndNoJarOrderAndNoFiles()throws Exception { + String db = testDatabases1[2]; + Collection actualOrder = dbResService1.getResourcesForDatabase(db); + Assert.assertNull(actualOrder); + + } + + /************************************************** + * Test cases without common jars + **************************************************/ + + @Test + public void testDbWithCommonJarsAndWithJarOrderAndFiles()throws Exception { + String db = testDatabases1[0]; + Collection actualOrder = dbResService2.getResourcesForDatabase(db); + List actualOrderList = new ArrayList(); + + for (LensSessionImpl.ResourceEntry res : actualOrder) { + actualOrderList.add(res.getLocation()); + } + + String[] jarFilesOrder = { + "z_" + db + ".jar", + "y_" + db + ".jar", + "x_" + db + ".jar", + }; + + // Verify order + for (int i = 0; i < jarFilesOrder.length; i++) { + Assert.assertTrue(actualOrderList.get(i).contains(jarFilesOrder[i]), + actualOrderList.get(i) + " > " + jarFilesOrder[i]); + } + } + + @Test + public void testDbWithCommonJarsAndWithNoJarOrderAndVersionFiles()throws Exception { + String db = testDatabases1[1]; + Collection actualOrder = dbResService2.getResourcesForDatabase(db); + List actualOrderList = new ArrayList(); + + for (LensSessionImpl.ResourceEntry res : actualOrder) { + actualOrderList.add(res.getLocation()); + } + + // Should pick the latest one + String[] jarFilesOrder = { + db + "_3.jar", + "lens-ship.jar", + "meta.jar", + }; + + // Verify order + for (int i = 0; i < jarFilesOrder.length; i++) { + Assert.assertTrue(actualOrderList.get(i).contains(jarFilesOrder[i]), + actualOrderList.get(i) + " > " + jarFilesOrder[i]); + } + } + + @Test + public void testDbWithCommonJarsAndNoJarOrderAndNoFiles()throws Exception { + String db = testDatabases1[2]; + Collection actualOrder = dbResService2.getResourcesForDatabase(db); + List actualOrderList = new ArrayList(); + + for (LensSessionImpl.ResourceEntry res : actualOrder) { + actualOrderList.add(res.getLocation()); + } + + // Should pick the latest one + String[] jarFilesOrder = { + "lens-ship.jar", + "meta.jar", + }; + + // Verify order + for (int i = 0; i < jarFilesOrder.length; i++) { + Assert.assertTrue(actualOrderList.get(i).contains(jarFilesOrder[i]), + actualOrderList.get(i) + " > " + jarFilesOrder[i]); + } + } + + } From f3327c1ff94837792947d96230dbcae46c3c572e Mon Sep 17 00:00:00 2001 From: badri Date: Fri, 18 Mar 2016 18:04:56 +0530 Subject: [PATCH 5/7] Code formatted. Error fixed. --- .../server/metastore/MetastoreResource.java | 60 ++++++------------- .../lens/server/LensServerTestUtil.java | 2 +- .../server/metastore/TestDatabaseService.java | 13 ++-- .../session/TestDatabaseResourceService.java | 13 ---- 4 files changed, 24 insertions(+), 64 deletions(-) diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index 4c48fe106..d56d6d182 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -405,17 +405,17 @@ public StringList getAllCubes(@QueryParam("sessionid") LensSessionHandle session checkSessionId(sessionid); try { switch (cubeTypes) { - case "all": - return new StringList(getSvc().getAllCubeNames(sessionid)); - case "base": - return new StringList(getSvc().getAllBaseCubeNames(sessionid)); - case "derived": - return new StringList(getSvc().getAllDerivedCubeNames(sessionid)); - case "queryable": - return new StringList(getSvc().getAllQueryableCubeNames(sessionid)); - default: - throw new BadRequestException("Invalid type " + cubeTypes + " Accepted" - + " values are 'all' or 'base' or 'derived' or 'queryable'"); + case "all": + return new StringList(getSvc().getAllCubeNames(sessionid)); + case "base": + return new StringList(getSvc().getAllBaseCubeNames(sessionid)); + case "derived": + return new StringList(getSvc().getAllDerivedCubeNames(sessionid)); + case "queryable": + return new StringList(getSvc().getAllQueryableCubeNames(sessionid)); + default: + throw new BadRequestException("Invalid type " + cubeTypes + " Accepted" + + " values are 'all' or 'base' or 'derived' or 'queryable'"); } } catch (LensException e) { log.error("Error getting cube names", e); @@ -1245,7 +1245,7 @@ public StringList getAllDims(@QueryParam("sessionid") LensSessionHandle sessioni @POST @Path("/dimtables") public APIResult createDimensionTable(@QueryParam("sessionid") LensSessionHandle sessionid, - XDimensionTable dimensionTable) { + XDimensionTable dimensionTable) { checkSessionId(sessionid); try { getSvc().createDimensionTable(sessionid, dimensionTable); @@ -1545,7 +1545,7 @@ public APIResult addPartitionToDimStorage(@QueryParam("sessionid") LensSessionHa @Path("/dimtables/{dimTableName}/storages/{storage}/partition") public APIResult updatePartitionOfDimStorage(@QueryParam("sessionid") LensSessionHandle sessionid, @PathParam("dimTableName") String dimTableName, - @PathParam("storage") String storage, + @PathParam("storage") String storage, XPartition partition) { checkSessionId(sessionid); checkNonNullArgs("Partition is null", partition); @@ -1707,46 +1707,20 @@ public StringList getPartitionTimelines(@QueryParam("sessionid") LensSessionHand } /** - * Validate jar + * Add a resource to the current DB *

*

* The returned @{link APIResult} will have status SUCCEEDED only if the add operation was successful for all * services running in this Lens server. *

* - * @param size size of the jar + * @param sessionid session handle object * @param type The type of resource. Valid types are 'jar' - * @param path path of the resource. Local or HDFS path + * @param fileInputStream stream of the resource. Local or HDFS path * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if add was successful. {@link APIResult} with state * {@link Status#PARTIAL}, if add succeeded only for some services. {@link APIResult} with state * {@link Status#FAILED}, if add has failed */ - @PUT - @Path("databases/validateDBJar") - @Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML, MediaType.TEXT_PLAIN}) - public APIResult validateDBResource(@QueryParam("size") String size, - @QueryParam("type") String type, @QueryParam("path") String path) { - - - return new APIResult(Status.SUCCEEDED, "Add resource succeeded"); - } - - - /** - * Add a resource to the current DB - *

- *

- * The returned @{link APIResult} will have status SUCCEEDED only if the add operation was successful for all - * services running in this Lens server. - *

- * - * @param sessionid session handle object - * @param type The type of resource. Valid types are 'jar' - * @param fileInputStream stream of the resource. Local or HDFS path - * @return {@link APIResult} with state {@link Status#SUCCEEDED}, if add was successful. {@link APIResult} with state - * {@link Status#PARTIAL}, if add succeeded only for some services. {@link APIResult} with state - * {@link Status#FAILED}, if add has failed - */ @POST @Path("databases/jar") @Consumes({MediaType.MULTIPART_FORM_DATA}) @@ -1755,7 +1729,7 @@ public APIResult addDBResource(@QueryParam("sessionid") LensSessionHandle sessio @FormDataParam("type") String type, @FormDataParam("file") InputStream fileInputStream) { try { - getSvc().addDBJar(sessionid,type,fileInputStream); + getSvc().addDBJar(sessionid, type, fileInputStream); } catch (LensException e) { e.printStackTrace(); return new APIResult(Status.FAILED, e.getMessage()); diff --git a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java index 0a34a6afc..65486ffab 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java @@ -266,7 +266,7 @@ public static void createTestDatabaseResources(String[] testDatabases, HiveConf // nothing to setup return; } - File resDir = new File(conf.get(LensConfConstants.DATABASE_RESOURCE_DIR)); + File resDir = new File(conf.get(LensConfConstants.DATABASE_RESOURCE_DIR, "target/resources/")); if (!resDir.exists()) { resDir.mkdir(); } diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java index fc9a8b2ce..4470379df 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java @@ -144,10 +144,9 @@ public void testJarUpload(MediaType mediaType) throws Exception { APIResult resultUpd = target().path("metastore").path("databases/jar"). queryParam("sessionid", lensSessionId).request(mediaType) .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); - System.out.println(resultUpd.getStatus() + " " + resultUpd); + log.debug(resultUpd.getStatus() + " " + resultUpd); assertEquals(resultUpd.getMessage(), "Database resource location does not exist. Database jar can't be uploaded"); - } @@ -192,7 +191,7 @@ public void testJarUploadWithJarOrderFileInDbFolder(MediaType mediaType) throws APIResult resultUpd = target().path("metastore").path("databases/jar"). queryParam("sessionid", lensSessionId).request(mediaType) .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); - System.out.println(resultUpd.getStatus() + " " + resultUpd); + log.debug(resultUpd.getStatus() + " " + resultUpd); assertEquals(resultUpd.getMessage(), "Database jar_order file exist. Database jar can't be uploaded"); cleanUp(dbFolder); @@ -229,7 +228,7 @@ public void testJarUploadWithDbUploadingJarInFolder(MediaType mediaType) throws File dbFolder = new File("target/resources/" + dbName); dbFolder.mkdirs(); - File dbFolderJarOrder = new File("target/resources/" + dbName + File.separator + dbName+"_uploading.jar"); + File dbFolderJarOrder = new File("target/resources/" + dbName + File.separator + dbName + "_uploading.jar"); dbFolderJarOrder.createNewFile(); FormDataMultiPart mp = getFormData(mediaType); @@ -240,7 +239,7 @@ public void testJarUploadWithDbUploadingJarInFolder(MediaType mediaType) throws APIResult resultUpd = target().path("metastore").path("databases/jar"). queryParam("sessionid", lensSessionId).request(mediaType) .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); - System.out.println(resultUpd.getStatus() + " " + resultUpd); + log.debug(resultUpd.getStatus() + " " + resultUpd); assertEquals(resultUpd.getMessage(), "Database jar file upload in progress . Database jar can't be uploaded. Try later!"); cleanUp(dbFolder); @@ -284,7 +283,7 @@ public void testJarUploadWithNoJarOrderInFolder(MediaType mediaType) throws Exce APIResult resultUpd = target().path("metastore").path("databases/jar"). queryParam("sessionid", lensSessionId).request(mediaType) .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); - System.out.println(resultUpd.getStatus() + " " + resultUpd); + log.debug(resultUpd.getStatus() + " " + resultUpd); assertEquals(resultUpd.getStatus(), APIResult.Status.SUCCEEDED); cleanUp(dbFolder); @@ -333,7 +332,7 @@ public void testJarUploadWithExistingJarsInFolder(MediaType mediaType) throws Ex APIResult resultUpd = target().path("metastore").path("databases/jar"). queryParam("sessionid", lensSessionId).request(mediaType) .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); - System.out.println(resultUpd.getStatus() + " " + resultUpd); + log.debug(resultUpd.getStatus() + " " + resultUpd); assertEquals(resultUpd.getStatus(), APIResult.Status.SUCCEEDED); cleanUp(dbFolder); diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java index 7bea001aa..728fede5e 100644 --- a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java +++ b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java @@ -48,15 +48,6 @@ public class TestDatabaseResourceService { private static final String DB1 = DB_PFX + "db1"; private static final String DB2 = DB_PFX + "db2"; -// //tested using dbResService1 -// private static final String DB_TEST_JarUploadWhenDbFolderExists = "db1_" + System.currentTimeMillis(); -// private static final String DB_TEST_JarUploadWithJarOrderFileInDbFolder = "db2_" + System.currentTimeMillis(); -// -// //tested using dbResService2 -// private static final String DB_TEST_JarUploadWithNoJarOrderInDbFolder = "db3_" + System.currentTimeMillis(); -// private static final String DB_TEST_JarUploadWithNoJarOrderAndCommonJarsInBaseFolder = "db4_" + System.currentTimeMillis(); -// private static final String DB_TEST_JarUploadWithJarOrderAndCommonJarsInBaseFolder = "db5_" + System.currentTimeMillis(); -// private final String[] testDatabases = {DB1, DB2}; private static final String JAR_ORDER_AND_FILES = DB_PFX + "jar_order_files_in_db"; @@ -65,10 +56,6 @@ public class TestDatabaseResourceService { private final String[] testDatabases1 = {JAR_ORDER_AND_FILES, NO_JAR_ORDER_AND_VERSION_FILES,NO_JAR_ORDER_NO_FILES}; -// private final String[] testDatabases1 = {DB_TEST_JarUploadWhenDbFolderExists, DB_TEST_JarUploadWithJarOrderFileInDbFolder}; -// -// private final String[] testDatabases2 = {DB_TEST_JarUploadWithNoJarOrderInDbFolder, DB_TEST_JarUploadWithNoJarOrderAndCommonJarsInBaseFolder, DB_TEST_JarUploadWithJarOrderAndCommonJarsInBaseFolder}; - private final HiveConf conf = new HiveConf(TestDatabaseResourceService.class); private DatabaseResourceService dbResService; From c50fbc43c34596191e07fca03b809ed08da09a4f Mon Sep 17 00:00:00 2001 From: badri Date: Fri, 18 Mar 2016 19:28:20 +0530 Subject: [PATCH 6/7] checkstyle errors fixed --- .../cli/commands/LensDatabaseCommands.java | 3 +- .../metastore/CubeMetastoreServiceImpl.java | 18 ++++---- .../server/metastore/MetastoreResource.java | 26 ++++++------ .../session/DatabaseResourceService.java | 33 ++++++++------- .../lens/server/LensServerTestUtil.java | 2 +- .../server/metastore/TestDatabaseService.java | 42 +++++++++++-------- .../session/TestDatabaseResourceService.java | 24 +++++------ 7 files changed, 76 insertions(+), 72 deletions(-) diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java index f3087084d..3dd8dbb82 100644 --- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java +++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensDatabaseCommands.java @@ -130,7 +130,8 @@ protected APIResult doDelete(String name, boolean cascade) { */ @CliCommand(value = "add dbjar", help = "Add jar resource to the db") public String addDBJar( - @CliOption(key = {"", "path"}, mandatory = true, help = " / ") @NonNull String path) { + @CliOption(key = {"", "path"}, mandatory = true, help = " / ") + @NonNull String path) { APIResult result = getClient().addDBJarResource(path); return result.getMessage(); } diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java index ba75aec07..f2ef1c41e 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/CubeMetastoreServiceImpl.java @@ -38,7 +38,6 @@ import org.apache.lens.server.api.metastore.CubeMetastoreService; import org.apache.lens.server.session.LensSessionImpl; -import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -841,7 +840,7 @@ public void addDBJar(LensSessionHandle sessionid, String type, InputStream is) t String baseDir = getHiveConf().get(LensConfConstants.DATABASE_RESOURCE_DIR, LensConfConstants.DEFAULT_DATABASE_RESOURCE_DIR); - String dbDir = baseDir+File.separator+currentDB; + String dbDir = baseDir + File.separator + currentDB; log.info("Database specific resources at {}", dbDir); @@ -859,7 +858,7 @@ public void addDBJar(LensSessionHandle sessionid, String type, InputStream is) t throw new LensException("Database jar_order file exist. Database jar can't be uploaded"); } - String tempFileName = currentDB+"_uploading.jar"; + String tempFileName = currentDB + "_uploading.jar"; Path uploadingPath = new Path(dbDir, tempFileName); FileSystem uploadingFs = FileSystem.newInstance(uploadingPath.toUri(), getHiveConf()); @@ -878,29 +877,30 @@ public void addDBJar(LensSessionHandle sessionid, String type, InputStream is) t if (tokens.length > 1) { int fIndex = Integer.parseInt(tokens[tokens.length - 1].substring(0, 1)); - if (fIndex > lastIndex) + if (fIndex > lastIndex) { lastIndex = fIndex; + } } } int newIndex = lastIndex + 1; - Path resJarPath = new Path(baseDir, currentDB + File.separator +tempFileName); + Path resJarPath = new Path(baseDir, currentDB + File.separator + tempFileName); log.info("new jar name : " + resJarPath.getName()); fos = serverFs.create(resJarPath); IOUtils.copy(is, fos); fos.flush(); - Path renamePath = new Path(baseDir, currentDB + File.separator +currentDB + "_" + newIndex+ ".jar"); - serverFs.rename(resJarPath,renamePath); + Path renamePath = new Path(baseDir, currentDB + File.separator + currentDB + "_" + newIndex + ".jar"); + serverFs.rename(resJarPath, renamePath); } catch (FileNotFoundException e) { - e.printStackTrace(); + log.error("FileNotFoundException", e); throw new LensException(e); } catch (IOException e) { - e.printStackTrace(); + log.error("IOException", e); throw new LensException(e); } finally { if (fos != null) { diff --git a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java index d56d6d182..2d763ba32 100644 --- a/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java +++ b/lens-server/src/main/java/org/apache/lens/server/metastore/MetastoreResource.java @@ -36,9 +36,7 @@ import org.apache.lens.server.LensServices; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.CubeMetastoreService; -import org.apache.lens.server.util.ScannedPaths; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -405,17 +403,17 @@ public StringList getAllCubes(@QueryParam("sessionid") LensSessionHandle session checkSessionId(sessionid); try { switch (cubeTypes) { - case "all": - return new StringList(getSvc().getAllCubeNames(sessionid)); - case "base": - return new StringList(getSvc().getAllBaseCubeNames(sessionid)); - case "derived": - return new StringList(getSvc().getAllDerivedCubeNames(sessionid)); - case "queryable": - return new StringList(getSvc().getAllQueryableCubeNames(sessionid)); - default: - throw new BadRequestException("Invalid type " + cubeTypes + " Accepted" - + " values are 'all' or 'base' or 'derived' or 'queryable'"); + case "all": + return new StringList(getSvc().getAllCubeNames(sessionid)); + case "base": + return new StringList(getSvc().getAllBaseCubeNames(sessionid)); + case "derived": + return new StringList(getSvc().getAllDerivedCubeNames(sessionid)); + case "queryable": + return new StringList(getSvc().getAllQueryableCubeNames(sessionid)); + default: + throw new BadRequestException("Invalid type " + cubeTypes + " Accepted" + + " values are 'all' or 'base' or 'derived' or 'queryable'"); } } catch (LensException e) { log.error("Error getting cube names", e); @@ -1731,7 +1729,7 @@ public APIResult addDBResource(@QueryParam("sessionid") LensSessionHandle sessio try { getSvc().addDBJar(sessionid, type, fileInputStream); } catch (LensException e) { - e.printStackTrace(); + log.error("Error in adding resource to db", e); return new APIResult(Status.FAILED, e.getMessage()); } return new APIResult(Status.SUCCEEDED, "Add resource succeeded"); diff --git a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java index 877974ff0..b75054607 100644 --- a/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java +++ b/lens-server/src/main/java/org/apache/lens/server/session/DatabaseResourceService.java @@ -130,7 +130,7 @@ public synchronized void start() { } catch (LensException e) { incrCounter(LOAD_RESOURCES_ERRORS); - log.warn("Failed to load DB resource mapping, resources must be added explicitly to session."); + log.error("Failed to load DB resource mapping, resources must be added explicitly to session.", e); } } @@ -161,6 +161,7 @@ private void mapDbResourceEntries(String dbName) throws LensException { Path resDirPath = new Path(resTopDir, dbName); serverFs = FileSystem.newInstance(resDirPath.toUri(), getHiveConf()); + resDirPath = serverFs.resolvePath(resDirPath); if (!serverFs.exists(resDirPath)) { incrCounter(LOAD_RESOURCES_ERRORS); log.warn("Database resource location does not exist - {}. Database jars will not be available", resDirPath); @@ -198,6 +199,7 @@ private void mapDbResourceEntries() throws LensException { Path resTopDirPath = new Path(baseDir); serverFs = FileSystem.newInstance(resTopDirPath.toUri(), getHiveConf()); + resTopDirPath = serverFs.resolvePath(resTopDirPath); if (!serverFs.exists(resTopDirPath)) { incrCounter(LOAD_RESOURCES_ERRORS); log.warn("DB base location does not exist.", baseDir); @@ -216,7 +218,7 @@ private void mapDbResourceEntries() throws LensException { mapDbResourceEntries(dbName); } else { // new flow - mapDbSpecificJar(baseDir, dbName); + mapDbSpecificJar(resTopDirPath, dbName); } } } @@ -225,10 +227,9 @@ private void mapDbResourceEntries() throws LensException { } catch (IOException io) { log.error("Error getting list of dbs to load resources from", io); throw new LensException(io); - }catch (Exception e){ - log.error("Exception : ",e); - } - finally { + } catch (Exception e) { + log.error("Exception : ", e); + } finally { if (serverFs != null) { try { serverFs.close(); @@ -239,9 +240,9 @@ private void mapDbResourceEntries() throws LensException { } } - private void mapDbSpecificJar(String baseDir, String dbName) throws Exception{ + private void mapDbSpecificJar(Path baseDir, String dbName) throws Exception { FileSystem serverFs = null; - try{ + try { int lastIndex = 0; @@ -255,24 +256,25 @@ private void mapDbSpecificJar(String baseDir, String dbName) throws Exception{ if (tokens.length > 1) { int fIndex = Integer.parseInt(tokens[tokens.length - 1].substring(0, 1)); - if (fIndex > lastIndex) + if (fIndex > lastIndex) { lastIndex = fIndex; + } } } - if(lastIndex > 0){ + if (lastIndex > 0) { // latest jar - Path latestJarPath = new Path(baseDir, dbName + File.separator +dbName + "_" + lastIndex+ ".jar"); + Path latestJarPath = new Path(baseDir, dbName + File.separator + dbName + "_" + lastIndex + ".jar"); addResourceEntry(new LensSessionImpl.ResourceEntry("jar", latestJarPath.toUri().toString()), dbName); } // add common jars - for(LensSessionImpl.ResourceEntry jar : commonResMap){ - addResourceEntry(jar,dbName); + for (LensSessionImpl.ResourceEntry jar : commonResMap) { + addResourceEntry(jar, dbName); } - }catch (Exception io) { + } catch (Exception io) { log.error("Error getting db specific resource ", io); throw new LensException(io); } finally { @@ -288,7 +290,6 @@ private void mapDbSpecificJar(String baseDir, String dbName) throws Exception{ } - private void findResourcesInDir(FileSystem serverFs, String database, Path dbDirPath) throws IOException { // Check if order file is present in the directory List jars = new ScannedPaths(dbDirPath, "jar").getFinalPaths(); @@ -367,7 +368,7 @@ protected void createClassLoader(String database) throws LensException { * @return class loader updated as a result of adding any JARs */ protected synchronized ClassLoader loadDBJars(String database, Collection resources, - boolean addToCache) { + boolean addToCache) { ClassLoader classLoader = classLoaderCache.get(database); if (classLoader == null) { // No change since there are no static resources to be added diff --git a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java index 65486ffab..56824da5a 100644 --- a/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java +++ b/lens-server/src/test/java/org/apache/lens/server/LensServerTestUtil.java @@ -422,7 +422,7 @@ private static void setUpDbWithNoJarOrderAndVersionFiles(String db, File resDir, String[] jarOrder = { db + "_3.jar", db + "_2.jar", - db + "_1.jar" + db + "_1.jar", }; diff --git a/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java b/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java index 4470379df..3839c43b4 100644 --- a/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java +++ b/lens-server/src/test/java/org/apache/lens/server/metastore/TestDatabaseService.java @@ -1,16 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.lens.server.metastore; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.fail; import java.io.File; -import java.io.IOException; import java.util.HashMap; -import javax.mail.Folder; -import javax.ws.rs.BadRequestException; -import javax.ws.rs.NotFoundException; import javax.ws.rs.client.Entity; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.Application; @@ -18,14 +31,12 @@ import org.apache.lens.api.APIResult; import org.apache.lens.api.LensSessionHandle; -import org.apache.lens.api.metastore.ObjectFactory; import org.apache.lens.server.LensJerseyTest; import org.apache.lens.server.LensServices; import org.apache.lens.server.api.LensConfConstants; import org.apache.lens.server.api.metastore.CubeMetastoreService; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.hive.conf.HiveConf; import org.glassfish.jersey.media.multipart.FormDataBodyPart; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; @@ -39,9 +50,7 @@ import lombok.extern.slf4j.Slf4j; -/** - * Created by badrinath.kulkarni on 3/8/16. - */ + @Slf4j @Test(groups = "unit-test") public class TestDatabaseService extends LensJerseyTest { @@ -98,12 +107,8 @@ private FormDataMultiPart getFormData(MediaType mediaType) { FileDataBodyPart filePart = new FileDataBodyPart("file", file); filePart.setContentDisposition(dispo); mp.bodyPart(filePart); - - } catch (Exception e) { - e.printStackTrace(); - } finally { - + log.error("Error in getting form data", e); } return mp; @@ -240,7 +245,8 @@ public void testJarUploadWithDbUploadingJarInFolder(MediaType mediaType) throws queryParam("sessionid", lensSessionId).request(mediaType) .post(Entity.entity(mp, multiPart.getMediaType()), APIResult.class); log.debug(resultUpd.getStatus() + " " + resultUpd); - assertEquals(resultUpd.getMessage(), "Database jar file upload in progress . Database jar can't be uploaded. Try later!"); + assertEquals(resultUpd.getMessage(), "Database jar file upload in progress . Database jar can't be uploaded." + + " Try later!"); cleanUp(dbFolder); } @@ -340,9 +346,9 @@ public void testJarUploadWithExistingJarsInFolder(MediaType mediaType) throws Ex private void cleanUp(File f) { try { - //FileUtils.deleteDirectory(f); + FileUtils.deleteDirectory(f); } catch (Exception e) { - e.printStackTrace(); + log.error("Error cleaning directory", e); } } diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java index 728fede5e..95eb6a8d8 100644 --- a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java +++ b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java @@ -18,7 +18,6 @@ */ package org.apache.lens.server.session; -import java.io.File; import java.net.URL; import java.net.URLClassLoader; import java.util.ArrayList; @@ -28,7 +27,6 @@ import org.apache.lens.server.LensServerTestUtil; import org.apache.lens.server.api.LensConfConstants; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -54,7 +52,7 @@ public class TestDatabaseResourceService { private static final String NO_JAR_ORDER_AND_VERSION_FILES = DB_PFX + "no_jar_order_and_version_files"; private static final String NO_JAR_ORDER_NO_FILES = DB_PFX + "no_jar_order_no_files"; - private final String[] testDatabases1 = {JAR_ORDER_AND_FILES, NO_JAR_ORDER_AND_VERSION_FILES,NO_JAR_ORDER_NO_FILES}; + private final String[] testDatabases1 = {JAR_ORDER_AND_FILES, NO_JAR_ORDER_AND_VERSION_FILES, NO_JAR_ORDER_NO_FILES}; private final HiveConf conf = new HiveConf(TestDatabaseResourceService.class); private DatabaseResourceService dbResService; @@ -68,11 +66,11 @@ public class TestDatabaseResourceService { @BeforeClass public void setup() throws Exception { - String prefix = "file://"+System.getProperty("user.dir"); + String prefix = System.getProperty("user.dir"); - conf.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix+"/target/resources"); - conf1.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix+"/target/resources_without_common_jars"); - conf2.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix+"/target/resources_with_common_jars"); + conf.set(LensConfConstants.DATABASE_RESOURCE_DIR, "/target/resources"); + conf1.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix + "/target/resources_without_common_jars"); + conf2.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix + "/target/resources_with_common_jars"); LensServerTestUtil.createTestDatabaseResources(testDatabases, conf); LensServerTestUtil.createTestDbWithoutCommonJars(testDatabases1, conf1); @@ -188,7 +186,7 @@ public void verifyClassLoader() throws Exception { * Test cases without common jars **************************************************/ @Test - public void testDbWithoutCommonJarsAndWithJarOrderAndFiles()throws Exception { + public void testDbWithoutCommonJarsAndWithJarOrderAndFiles() throws Exception { String db = testDatabases1[0]; Collection actualOrder = dbResService1.getResourcesForDatabase(db); List actualOrderList = new ArrayList(); @@ -211,7 +209,7 @@ public void testDbWithoutCommonJarsAndWithJarOrderAndFiles()throws Exception { } @Test - public void testDbWithoutCommonJarsAndWithNoJarOrderAndVersionFiles()throws Exception { + public void testDbWithoutCommonJarsAndWithNoJarOrderAndVersionFiles() throws Exception { String db = testDatabases1[1]; Collection actualOrder = dbResService1.getResourcesForDatabase(db); List actualOrderList = new ArrayList(); @@ -233,7 +231,7 @@ public void testDbWithoutCommonJarsAndWithNoJarOrderAndVersionFiles()throws Exce } @Test - public void testDbWithoutCommonJarsAndNoJarOrderAndNoFiles()throws Exception { + public void testDbWithoutCommonJarsAndNoJarOrderAndNoFiles() throws Exception { String db = testDatabases1[2]; Collection actualOrder = dbResService1.getResourcesForDatabase(db); Assert.assertNull(actualOrder); @@ -245,7 +243,7 @@ public void testDbWithoutCommonJarsAndNoJarOrderAndNoFiles()throws Exception { **************************************************/ @Test - public void testDbWithCommonJarsAndWithJarOrderAndFiles()throws Exception { + public void testDbWithCommonJarsAndWithJarOrderAndFiles() throws Exception { String db = testDatabases1[0]; Collection actualOrder = dbResService2.getResourcesForDatabase(db); List actualOrderList = new ArrayList(); @@ -268,7 +266,7 @@ public void testDbWithCommonJarsAndWithJarOrderAndFiles()throws Exception { } @Test - public void testDbWithCommonJarsAndWithNoJarOrderAndVersionFiles()throws Exception { + public void testDbWithCommonJarsAndWithNoJarOrderAndVersionFiles() throws Exception { String db = testDatabases1[1]; Collection actualOrder = dbResService2.getResourcesForDatabase(db); List actualOrderList = new ArrayList(); @@ -292,7 +290,7 @@ public void testDbWithCommonJarsAndWithNoJarOrderAndVersionFiles()throws Excepti } @Test - public void testDbWithCommonJarsAndNoJarOrderAndNoFiles()throws Exception { + public void testDbWithCommonJarsAndNoJarOrderAndNoFiles() throws Exception { String db = testDatabases1[2]; Collection actualOrder = dbResService2.getResourcesForDatabase(db); List actualOrderList = new ArrayList(); From 6db08fe19053e94e864a4f8852f74d6ca1a84333 Mon Sep 17 00:00:00 2001 From: badri Date: Fri, 18 Mar 2016 20:14:26 +0530 Subject: [PATCH 7/7] fixed slash error --- .../apache/lens/server/session/TestDatabaseResourceService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java index 95eb6a8d8..c60014116 100644 --- a/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java +++ b/lens-server/src/test/java/org/apache/lens/server/session/TestDatabaseResourceService.java @@ -68,7 +68,7 @@ public void setup() throws Exception { String prefix = System.getProperty("user.dir"); - conf.set(LensConfConstants.DATABASE_RESOURCE_DIR, "/target/resources"); + conf.set(LensConfConstants.DATABASE_RESOURCE_DIR, "target/resources"); conf1.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix + "/target/resources_without_common_jars"); conf2.set(LensConfConstants.DATABASE_RESOURCE_DIR, prefix + "/target/resources_with_common_jars");