Optimized Nested Table Locations check in sre. #27

pom.xml

@@ -22,7 +22,7 @@
 
     <groupId>com.cloudera.utils.hive</groupId>
     <artifactId>hive-sre</artifactId>
-    <version>2.4.2.0</version>
+    <version>2.4.3.0</version>
 
     <name>hive-sre</name>
 

src/main/resources/MYSQL/hive_sre_queries.yaml

@@ -496,9 +496,16 @@ WITH
                               DB.NAME AS ROOT_DB
                             , TBL.TBL_NAME AS ROOT_TBL
                             , S.LOCATION               AS ROOT_LOCATION
-                            , CONCAT(S.LOCATION, \"/%\") AS SEARCH_LOCATION
+                            , CONCAT(S.LOCATION, '/%') AS SEARCH_LOCATION
               FROM DBS DB
               INNER JOIN TBLS TBL ON DB.DB_ID = TBL.DB_ID
               INNER JOIN SDS S ON TBL.SD_ID = S.SD_ID
+              WHERE 
+                DB.NAME LIKE ?
         ) SUB ON S.LOCATION LIKE SUB.SEARCH_LOCATION
       "
+    parameters:
+      dbs:
+        initial:  "%"
+        sqlType:  12
+        location: 1

src/main/resources/ORACLE/hive_sre_queries.yaml

@@ -540,9 +540,16 @@ WHERE
                               DB.NAME ROOT_DB
                             , TBL.TBL_NAME ROOT_TBL
                             , S.LOCATION               ROOT_LOCATION
-                            , CONCAT(S.LOCATION, \"/%\") SEARCH_LOCATION
+                            , CONCAT(S.LOCATION, '/%') SEARCH_LOCATION
               FROM DBS DB
               INNER JOIN TBLS TBL ON DB.DB_ID = TBL.DB_ID
               INNER JOIN SDS S ON TBL.SD_ID = S.SD_ID
+              WHERE
+                DB.NAME LIKE ?
         ) SUB ON S.LOCATION LIKE SUB.SEARCH_LOCATION
       "
+    parameters:
+      dbs:
+        initial:  "%"
+        sqlType:  12
+        location: 1

src/main/resources/POSTGRES/hive_sre_queries.yaml

@@ -442,6 +442,11 @@ ORDER BY \"DB_NAME\", \"TBL_TYPE\"
       , \"TYPE_COUNT\"
     FROM \"ANTI_PATTERN\"
     "
+    parameters:
+      dbs:
+        initial:  "%"
+        sqlType:  12
+        location: 1
   anti_pattern_base_dir_detail:
     statement: "
     WITH \"BASE_DIR_COUNT\" AS (
@@ -492,9 +497,16 @@ ORDER BY \"DB_NAME\", \"TBL_TYPE\"
                               \"DB\".\"NAME\" AS \"ROOT_DB\"
                             , \"TBL\".\"TBL_NAME\" AS \"ROOT_TBL\"
                             , \"S\".\"LOCATION\"               AS \"ROOT_LOCATION\"
-                            , CONCAT(\"S\".\"LOCATION\", \"/%\") AS \"SEARCH_LOCATION\"
+                            , CONCAT(\"S\".\"LOCATION\", '/%') AS \"SEARCH_LOCATION\"
               FROM \"DBS\" \"DB\"
               INNER JOIN \"TBLS\" \"TBL\" ON \"DB\".\"DB_ID\" = \"TBL\".\"DB_ID\"
               INNER JOIN \"SDS\" \"S\" ON \"TBL\".\"SD_ID\" = \"S\".\"SD_ID\"
-        ) \"SUB\" ON \"S\".\"LOCATION\" LIKE \"SUB\".\"SEARCH_LOCATION\"
+              WHERE
+                \"DB\".\"NAME\" LIKE ?
+              ) \"SUB\" ON \"S\".\"LOCATION\" LIKE \"SUB\".\"SEARCH_LOCATION\"
       "
+    parameters:
+      dbs:
+        initial:  "%"
+        sqlType:  12
+        location: 1

src/main/resources/procs/hive_sre_procs.yaml

@@ -368,8 +368,9 @@ processes:
     successDescription:       "Nested Table Directory Listing"
     errorFilename:            "hive_nested_table_locations_err.txt"
     successFilename:          "hive_nested_table_locations.md"
-    listingQuery:             "nested_table_locations"
+    dbListingQuery:           "db_tbl_count"
     listingColumns:           [ "name" ,"tbl_name" , "location", "root_db", "root_tbl", "root_location"]
+    pathsListingQuery:        "nested_table_locations"
     skipCommandCheck:
       displayName:            "Hive Nested Table Locations"
       title:                  "# Hive Nested Table Locations - Unique Base Location w/ Counts (v.${Implementation-Version})\n"

src/test/java/com/cloudera/utils/hive/config/SreSQLMySqlTest.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2022. Cloudera, Inc. All Rights Reserved
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package com.cloudera.utils.hive.config;
+
+import com.cloudera.utils.hive.Sre;
+import org.junit.Test;
+
+public class SreSQLMySqlTest {
+
+    @Test
+    public void u3MYSQL_SQLTest() {
+            String cfg = System.getProperty("user.home") + System.getProperty("file.separator") +
+                    ".hive-sre/cfg" + System.getProperty("file.separator") +
+                    "default-" + Metastore.DB_TYPE.MYSQL + ".yaml";
+            Sre.main(new String[]{"u3", "-tsql", "-all", "-cfg", cfg,"-o", "/tmp/sre-sql-test"});
+    }
+
+    @Test
+    public void sreMYSQL_SQLTest() {
+        String cfg = System.getProperty("user.home") + System.getProperty("file.separator") +
+                ".hive-sre/cfg" + System.getProperty("file.separator") +
+                "default-" + Metastore.DB_TYPE.MYSQL + ".yaml";
+        Sre.main(new String[]{"sre", "-tsql", "-all", "-cfg", cfg,"-o", "/tmp/sre-sql-test"});
+    }
+
+}

src/test/java/com/cloudera/utils/hive/config/SreSQLOracleTest.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2022. Cloudera, Inc. All Rights Reserved
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package com.cloudera.utils.hive.config;
+
+import com.cloudera.utils.hive.Sre;
+import org.junit.Test;
+
+public class SreSQLOracleTest {
+
+    @Test
+    public void u3ORACLE_SQLTest() {
+        String cfg = System.getProperty("user.home") + System.getProperty("file.separator") +
+                ".hive-sre/cfg" + System.getProperty("file.separator") +
+                "default-" + Metastore.DB_TYPE.ORACLE + ".yaml";
+        Sre.main(new String[]{"u3", "-tsql", "-all", "-cfg", cfg,"-o", "/tmp/sre-sql-test"});
+    }
+
+    @Test
+    public void sreORACLE_SQLTest() {
+        String cfg = System.getProperty("user.home") + System.getProperty("file.separator") +
+                ".hive-sre/cfg" + System.getProperty("file.separator") +
+                "default-" + Metastore.DB_TYPE.ORACLE + ".yaml";
+        Sre.main(new String[]{"sre", "-tsql", "-all", "-cfg", cfg,"-o", "/tmp/sre-sql-test"});
+    }
+
+}

src/test/java/com/cloudera/utils/hive/config/SreSQLPostgreSQLTest.java

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2022. Cloudera, Inc. All Rights Reserved
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *        http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package com.cloudera.utils.hive.config;
+
+import com.cloudera.utils.hive.Sre;
+import org.junit.Test;
+
+public class SreSQLPostgreSQLTest {
+
+    @Test
+    public void u3POSTGRESQL_SQLTest() {
+        String cfg = System.getProperty("user.home") + System.getProperty("file.separator") +
+                ".hive-sre/cfg" + System.getProperty("file.separator") +
+                "default-" + Metastore.DB_TYPE.POSTGRES + ".yaml";
+        Sre.main(new String[]{"u3", "-tsql", "-all", "-cfg", cfg,"-o", "/tmp/sre-sql-test"});
+    }
+
+    @Test
+    public void srePOSTGRESQL_SQLTest() {
+        String cfg = System.getProperty("user.home") + System.getProperty("file.separator") +
+                ".hive-sre/cfg" + System.getProperty("file.separator") +
+                "default-" + Metastore.DB_TYPE.POSTGRES + ".yaml";
+        Sre.main(new String[]{"sre", "-tsql", "-all", "-cfg", cfg,"-o", "/tmp/sre-sql-test"});
+    }
+
+}