From 72f5d55aff661f02e72c502f7fd50acfb6361b43 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Thu, 10 Nov 2022 14:13:37 +0100 Subject: [PATCH 01/20] add ToDo to remove RemoteTagTranslater later --- .../org/heigit/ohsome/ohsomeapi/oshdb/RemoteTagTranslator.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/RemoteTagTranslator.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/RemoteTagTranslator.java index ad3a586e..7ca6e48d 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/RemoteTagTranslator.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/RemoteTagTranslator.java @@ -4,6 +4,9 @@ import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; + +// TODO: Remove + /** * A tag-translator that can be used in map/flatMap/aggregateBy functions that may be executed on * remote machines. From 8919b1ee6dc0c9f2a39de74049d31ad48cb06afd Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Thu, 10 Nov 2022 14:35:42 +0100 Subject: [PATCH 02/20] extract Application preRun method to utility class --- .../heigit/ohsome/ohsomeapi/Application.java | 132 +--------------- .../ohsomeapi/utils/ApplicationUtils.java | 143 ++++++++++++++++++ 2 files changed, 145 insertions(+), 130 deletions(-) create mode 100644 src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java index cb768197..ea7a744c 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java @@ -1,21 +1,11 @@ package org.heigit.ohsome.ohsomeapi; -import com.zaxxer.hikari.HikariConfig; import java.io.IOException; -import java.sql.DriverManager; import java.sql.SQLException; import java.util.Properties; import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException; -import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages; -import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData; -import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData; -import org.heigit.ohsome.ohsomeapi.oshdb.RemoteTagTranslator; -import org.heigit.ohsome.ohsomeapi.utils.RequestUtils; -import org.heigit.ohsome.oshdb.api.db.OSHDBH2; -import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; -import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; +import org.heigit.ohsome.ohsomeapi.utils.ApplicationUtils; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; -import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; import org.springframework.boot.DefaultApplicationArguments; @@ -84,125 +74,7 @@ public static void main(String[] args) { */ public static void preRun(ApplicationArguments args) throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { - final String dbProperty = "database.db"; - boolean multithreading = true; - boolean caching = false; - String dbPrefix = null; - long timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS; - int numberOfClusterNodes = DEFAULT_NUMBER_OF_CLUSTER_NODES; - int numberOfDataExtractionThreads = DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS; - // only used when tests are executed directly in Eclipse - if (System.getProperty(dbProperty) != null) { - DbConnData.db = new OSHDBH2(System.getProperty(dbProperty)); - } - for (String paramName : args.getOptionNames()) { - switch (paramName) { - case dbProperty: - DbConnData.db = new OSHDBH2(args.getOptionValues(paramName).get(0)); - break; - case "database.jdbc": - String[] jdbcParam = args.getOptionValues(paramName).get(0).split(";"); - DbConnData.db = new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]); - break; - case "database.ignite": - if (DbConnData.db != null) { - break; - } - DbConnData.db = new OSHDBIgnite(args.getOptionValues(paramName).get(0)); - break; - case "database.keytables": - DbConnData.keytables = new OSHDBH2(args.getOptionValues(paramName).get(0)); - break; - case "database.keytables.jdbc": - jdbcParam = args.getOptionValues(paramName).get(0).split(";"); - DbConnData.keytables = - new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]); - DbConnData.mapTagTranslator = new RemoteTagTranslator(() -> { - try { - Class.forName(jdbcParam[0]); - return new TagTranslator( - DriverManager.getConnection(jdbcParam[1], jdbcParam[2], jdbcParam[3])); - } catch (ClassNotFoundException e) { - throw new RuntimeException("A class with this specific name could not be found"); - } catch (OSHDBKeytablesNotFoundException | SQLException e) { - throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); - } - }); - HikariConfig hikariConfig = new HikariConfig(); - hikariConfig.setJdbcUrl(jdbcParam[1]); - hikariConfig.setUsername(jdbcParam[2]); - hikariConfig.setPassword(jdbcParam[3]); - hikariConfig.setMaximumPoolSize(numberOfDataExtractionThreads); - DbConnData.keytablesDbPoolConfig = hikariConfig; - break; - case "database.multithreading": - if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("false")) { - multithreading = false; - } - break; - case "database.caching": - if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("true")) { - caching = true; - } - break; - case "database.prefix": - dbPrefix = args.getOptionValues(paramName).get(0); - break; - case "database.timeout": - timeoutInMilliseconds = Long.parseLong(args.getOptionValues(paramName).get(0)); - break; - case "cluster.servernodes.count": - numberOfClusterNodes = Integer.parseInt(args.getOptionValues(paramName).get(0)); - break; - case "cluster.dataextraction.threadcount": - numberOfDataExtractionThreads = Integer.parseInt(args.getOptionValues(paramName).get(0)); - break; - default: - break; - } - } - if (DbConnData.db == null) { - throw new RuntimeException( - "You have to define one of the following three database parameters: '--database.db', " - + "'--database.ignite', or '--database.jdbc'."); - } - ProcessingData.setTimeout(timeoutInMilliseconds / 1000.0); - DbConnData.db.timeoutInMilliseconds(timeoutInMilliseconds); - ProcessingData.setNumberOfClusterNodes(numberOfClusterNodes); - ProcessingData.setNumberOfDataExtractionThreads(numberOfDataExtractionThreads); - if (DbConnData.db instanceof OSHDBJdbc) { - DbConnData.db = ((OSHDBJdbc) DbConnData.db).multithreading(multithreading); - } - if (DbConnData.db instanceof OSHDBH2) { - DbConnData.db = ((OSHDBH2) DbConnData.db).inMemory(caching); - } - if (DbConnData.keytables != null) { - DbConnData.tagTranslator = new TagTranslator(DbConnData.keytables.getConnection()); - } else { - if (!(DbConnData.db instanceof OSHDBJdbc)) { - throw new DatabaseAccessException("Missing keytables."); - } - DbConnData.tagTranslator = new TagTranslator(((OSHDBJdbc) DbConnData.db).getConnection()); - } - RequestUtils.extractOSHDBMetadata(); - if (DbConnData.mapTagTranslator == null) { - DbConnData.mapTagTranslator = new RemoteTagTranslator(DbConnData.tagTranslator); - } - if (DbConnData.db instanceof OSHDBIgnite) { - RemoteTagTranslator mtt = DbConnData.mapTagTranslator; - ((OSHDBIgnite) DbConnData.db).onClose(() -> { - try { - if (mtt.wasEvaluated()) { - mtt.get().getConnection().close(); - } - } catch (SQLException e) { - throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); - } - }); - } - if (dbPrefix != null) { - DbConnData.db = DbConnData.db.prefix(dbPrefix); - } + ApplicationUtils.preRun(args); } @Override diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java new file mode 100644 index 00000000..20f2b708 --- /dev/null +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java @@ -0,0 +1,143 @@ +package org.heigit.ohsome.ohsomeapi.utils; + +import com.zaxxer.hikari.HikariConfig; +import java.io.IOException; +import java.sql.DriverManager; +import java.sql.SQLException; +import org.heigit.ohsome.ohsomeapi.Application; +import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException; +import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages; +import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData; +import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData; +import org.heigit.ohsome.ohsomeapi.oshdb.RemoteTagTranslator; +import org.heigit.ohsome.oshdb.api.db.OSHDBH2; +import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; +import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; +import org.springframework.boot.ApplicationArguments; + +public class ApplicationUtils { + public static void preRun(ApplicationArguments args) + throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { + final String dbProperty = "database.db"; + boolean multithreading = true; + boolean caching = false; + String dbPrefix = null; + long timeoutInMilliseconds = Application.DEFAULT_TIMEOUT_IN_MILLISECONDS; + int numberOfClusterNodes = Application.DEFAULT_NUMBER_OF_CLUSTER_NODES; + int numberOfDataExtractionThreads = Application.DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS; + // only used when tests are executed directly in Eclipse + if (System.getProperty(dbProperty) != null) { + DbConnData.db = new OSHDBH2(System.getProperty(dbProperty)); + } + for (String paramName : args.getOptionNames()) { + switch (paramName) { + case dbProperty: + DbConnData.db = new OSHDBH2(args.getOptionValues(paramName).get(0)); + break; + case "database.jdbc": + String[] jdbcParam = args.getOptionValues(paramName).get(0).split(";"); + DbConnData.db = new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]); + break; + case "database.ignite": + if (DbConnData.db != null) { + break; + } + DbConnData.db = new OSHDBIgnite(args.getOptionValues(paramName).get(0)); + break; + case "database.keytables": + DbConnData.keytables = new OSHDBH2(args.getOptionValues(paramName).get(0)); + break; + case "database.keytables.jdbc": + jdbcParam = args.getOptionValues(paramName).get(0).split(";"); + DbConnData.keytables = + new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]); + DbConnData.mapTagTranslator = new RemoteTagTranslator(() -> { + try { + Class.forName(jdbcParam[0]); + return new TagTranslator( + DriverManager.getConnection(jdbcParam[1], jdbcParam[2], jdbcParam[3])); + } catch (ClassNotFoundException e) { + throw new RuntimeException("A class with this specific name could not be found"); + } catch (OSHDBKeytablesNotFoundException | SQLException e) { + throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); + } + }); + HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl(jdbcParam[1]); + hikariConfig.setUsername(jdbcParam[2]); + hikariConfig.setPassword(jdbcParam[3]); + hikariConfig.setMaximumPoolSize(numberOfDataExtractionThreads); + DbConnData.keytablesDbPoolConfig = hikariConfig; + break; + case "database.multithreading": + if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("false")) { + multithreading = false; + } + break; + case "database.caching": + if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("true")) { + caching = true; + } + break; + case "database.prefix": + dbPrefix = args.getOptionValues(paramName).get(0); + break; + case "database.timeout": + timeoutInMilliseconds = Long.parseLong(args.getOptionValues(paramName).get(0)); + break; + case "cluster.servernodes.count": + numberOfClusterNodes = Integer.parseInt(args.getOptionValues(paramName).get(0)); + break; + case "cluster.dataextraction.threadcount": + numberOfDataExtractionThreads = Integer.parseInt(args.getOptionValues(paramName).get(0)); + break; + default: + break; + } + } + if (DbConnData.db == null) { + throw new RuntimeException( + "You have to define one of the following three database parameters: '--database.db', " + + "'--database.ignite', or '--database.jdbc'."); + } + ProcessingData.setTimeout(timeoutInMilliseconds / 1000.0); + DbConnData.db.timeoutInMilliseconds(timeoutInMilliseconds); + ProcessingData.setNumberOfClusterNodes(numberOfClusterNodes); + ProcessingData.setNumberOfDataExtractionThreads(numberOfDataExtractionThreads); + if (DbConnData.db instanceof OSHDBJdbc) { + DbConnData.db = ((OSHDBJdbc) DbConnData.db).multithreading(multithreading); + } + if (DbConnData.db instanceof OSHDBH2) { + DbConnData.db = ((OSHDBH2) DbConnData.db).inMemory(caching); + } + if (DbConnData.keytables != null) { + DbConnData.tagTranslator = new TagTranslator(DbConnData.keytables.getConnection()); + } else { + if (!(DbConnData.db instanceof OSHDBJdbc)) { + throw new DatabaseAccessException("Missing keytables."); + } + DbConnData.tagTranslator = new TagTranslator(((OSHDBJdbc) DbConnData.db).getConnection()); + } + RequestUtils.extractOSHDBMetadata(); + if (DbConnData.mapTagTranslator == null) { + DbConnData.mapTagTranslator = new RemoteTagTranslator(DbConnData.tagTranslator); + } + if (DbConnData.db instanceof OSHDBIgnite) { + RemoteTagTranslator mtt = DbConnData.mapTagTranslator; + ((OSHDBIgnite) DbConnData.db).onClose(() -> { + try { + if (mtt.wasEvaluated()) { + mtt.get().getConnection().close(); + } + } catch (SQLException e) { + throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); + } + }); + } + if (dbPrefix != null) { + DbConnData.db = DbConnData.db.prefix(dbPrefix); + } + } +} \ No newline at end of file From d5c9f67d8dd182f58bcd7c08c20bdc14b1443c5c Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Thu, 10 Nov 2022 14:55:55 +0100 Subject: [PATCH 03/20] add private constructor to utility class --- .../org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java index 20f2b708..c2878449 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java @@ -18,6 +18,9 @@ import org.springframework.boot.ApplicationArguments; public class ApplicationUtils { + + private ApplicationUtils() {} + public static void preRun(ApplicationArguments args) throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { final String dbProperty = "database.db"; From 8ab5ea245c411d1f51fc6708b09b66d8ce2de486 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 11:16:12 +0100 Subject: [PATCH 04/20] ApplicationUtils: inlined dbProperty and removed property check for eclipse --- .../heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java index c2878449..ea7fd21f 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java @@ -23,20 +23,15 @@ private ApplicationUtils() {} public static void preRun(ApplicationArguments args) throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { - final String dbProperty = "database.db"; boolean multithreading = true; boolean caching = false; String dbPrefix = null; long timeoutInMilliseconds = Application.DEFAULT_TIMEOUT_IN_MILLISECONDS; int numberOfClusterNodes = Application.DEFAULT_NUMBER_OF_CLUSTER_NODES; int numberOfDataExtractionThreads = Application.DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS; - // only used when tests are executed directly in Eclipse - if (System.getProperty(dbProperty) != null) { - DbConnData.db = new OSHDBH2(System.getProperty(dbProperty)); - } for (String paramName : args.getOptionNames()) { switch (paramName) { - case dbProperty: + case "database.db": DbConnData.db = new OSHDBH2(args.getOptionValues(paramName).get(0)); break; case "database.jdbc": From 86b228c476a770e7f8037350ce598beb94d83fe9 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 17:42:41 +0100 Subject: [PATCH 05/20] ApplicationUtils: moved parameter parsing to separate method --- .../ohsomeapi/utils/ApplicationUtils.java | 136 ++++++++++++------ 1 file changed, 92 insertions(+), 44 deletions(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java index ea7fd21f..9c1b8571 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java @@ -19,55 +19,54 @@ public class ApplicationUtils { - private ApplicationUtils() {} + private boolean multithreading = true; + private boolean caching = false; + private String dbPrefix = null; + private long timeoutInMilliseconds = Application.DEFAULT_TIMEOUT_IN_MILLISECONDS; + private int numberOfClusterNodes = Application.DEFAULT_NUMBER_OF_CLUSTER_NODES; + private int numberOfDataExtractionThreads = Application.DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS; + private String databaseClassName; + private String databaseUser; + private String databasePassword; + private String databaseType; + private String databaseUrl; + private String keytablesClassName; + private String keytablesUser; + private String keytablesPassword; + private String keytablesType; + private String keytablesUrl; - public static void preRun(ApplicationArguments args) - throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { - boolean multithreading = true; - boolean caching = false; - String dbPrefix = null; - long timeoutInMilliseconds = Application.DEFAULT_TIMEOUT_IN_MILLISECONDS; - int numberOfClusterNodes = Application.DEFAULT_NUMBER_OF_CLUSTER_NODES; - int numberOfDataExtractionThreads = Application.DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS; + public ApplicationUtils(ApplicationArguments args) { for (String paramName : args.getOptionNames()) { switch (paramName) { + // TODO change to "database.h2" for a future stable version case "database.db": - DbConnData.db = new OSHDBH2(args.getOptionValues(paramName).get(0)); + databaseType = "h2"; + databaseUrl = args.getOptionValues(paramName).get(0); break; case "database.jdbc": + databaseType = "jdbc"; String[] jdbcParam = args.getOptionValues(paramName).get(0).split(";"); - DbConnData.db = new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]); + databaseClassName = jdbcParam[0]; + databaseUrl = jdbcParam[1]; + databaseUser = jdbcParam[2]; + databasePassword = jdbcParam[3]; break; case "database.ignite": - if (DbConnData.db != null) { - break; - } - DbConnData.db = new OSHDBIgnite(args.getOptionValues(paramName).get(0)); + databaseType = "ignite"; + databaseUrl = args.getOptionValues(paramName).get(0); break; case "database.keytables": - DbConnData.keytables = new OSHDBH2(args.getOptionValues(paramName).get(0)); + keytablesType = "h2"; + keytablesUrl = args.getOptionValues(paramName).get(0); break; case "database.keytables.jdbc": - jdbcParam = args.getOptionValues(paramName).get(0).split(";"); - DbConnData.keytables = - new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]); - DbConnData.mapTagTranslator = new RemoteTagTranslator(() -> { - try { - Class.forName(jdbcParam[0]); - return new TagTranslator( - DriverManager.getConnection(jdbcParam[1], jdbcParam[2], jdbcParam[3])); - } catch (ClassNotFoundException e) { - throw new RuntimeException("A class with this specific name could not be found"); - } catch (OSHDBKeytablesNotFoundException | SQLException e) { - throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); - } - }); - HikariConfig hikariConfig = new HikariConfig(); - hikariConfig.setJdbcUrl(jdbcParam[1]); - hikariConfig.setUsername(jdbcParam[2]); - hikariConfig.setPassword(jdbcParam[3]); - hikariConfig.setMaximumPoolSize(numberOfDataExtractionThreads); - DbConnData.keytablesDbPoolConfig = hikariConfig; + keytablesType = "jdbc"; + String[] keytablesJdbcParam = args.getOptionValues(paramName).get(0).split(";"); + keytablesClassName = keytablesJdbcParam[0]; + keytablesUrl = keytablesJdbcParam[1]; + keytablesUser = keytablesJdbcParam[2]; + keytablesPassword = keytablesJdbcParam[3]; break; case "database.multithreading": if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("false")) { @@ -95,20 +94,69 @@ public static void preRun(ApplicationArguments args) break; } } + } + + // refactor remainder back to Application.java + public static void preRun(ApplicationArguments args) + throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { + var config = new ApplicationUtils(args); + switch (config.databaseType) { + case "h2": + DbConnData.db = new OSHDBH2(config.databaseUrl); + break; + case "jdbc": + DbConnData.db = new OSHDBJdbc(config.databaseClassName, config.databaseUrl, + config.databaseUser, config.databasePassword); + break; + case "ignite": + DbConnData.db = new OSHDBIgnite(config.databaseUrl); + break; + default: + throw new IllegalStateException("Unexpected value: " + config.databaseType); + } + switch (config.keytablesType) { + case "h2": + DbConnData.db = new OSHDBH2(config.keytablesUrl); + break; + case "jdbc": + DbConnData.db = new OSHDBJdbc(config.keytablesClassName, config.keytablesUrl, + config.keytablesUser, config.keytablesPassword); + DbConnData.mapTagTranslator = new RemoteTagTranslator(() -> { + try { + Class.forName(config.keytablesClassName); + return new TagTranslator( + DriverManager.getConnection(config.keytablesUrl, config.keytablesUser, + config.keytablesPassword)); + } catch (ClassNotFoundException e) { + throw new RuntimeException("A class with this specific name could not be found"); + } catch (OSHDBKeytablesNotFoundException | SQLException e) { + throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); + } + }); + HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl(config.keytablesUrl); + hikariConfig.setUsername(config.keytablesUser); + hikariConfig.setPassword(config.keytablesPassword); + hikariConfig.setMaximumPoolSize(config.numberOfDataExtractionThreads); + DbConnData.keytablesDbPoolConfig = hikariConfig; + break; + default: + throw new IllegalStateException("Unexpected value: " + config.keytablesType); + } if (DbConnData.db == null) { throw new RuntimeException( "You have to define one of the following three database parameters: '--database.db', " + "'--database.ignite', or '--database.jdbc'."); } - ProcessingData.setTimeout(timeoutInMilliseconds / 1000.0); - DbConnData.db.timeoutInMilliseconds(timeoutInMilliseconds); - ProcessingData.setNumberOfClusterNodes(numberOfClusterNodes); - ProcessingData.setNumberOfDataExtractionThreads(numberOfDataExtractionThreads); + ProcessingData.setTimeout(config.timeoutInMilliseconds / 1000.0); + DbConnData.db.timeoutInMilliseconds(config.timeoutInMilliseconds); + ProcessingData.setNumberOfClusterNodes(config.numberOfClusterNodes); + ProcessingData.setNumberOfDataExtractionThreads(config.numberOfDataExtractionThreads); if (DbConnData.db instanceof OSHDBJdbc) { - DbConnData.db = ((OSHDBJdbc) DbConnData.db).multithreading(multithreading); + DbConnData.db = ((OSHDBJdbc) DbConnData.db).multithreading(config.multithreading); } if (DbConnData.db instanceof OSHDBH2) { - DbConnData.db = ((OSHDBH2) DbConnData.db).inMemory(caching); + DbConnData.db = ((OSHDBH2) DbConnData.db).inMemory(config.caching); } if (DbConnData.keytables != null) { DbConnData.tagTranslator = new TagTranslator(DbConnData.keytables.getConnection()); @@ -134,8 +182,8 @@ public static void preRun(ApplicationArguments args) } }); } - if (dbPrefix != null) { - DbConnData.db = DbConnData.db.prefix(dbPrefix); + if (config.dbPrefix != null) { + DbConnData.db = DbConnData.db.prefix(config.dbPrefix); } } } \ No newline at end of file From 91026470bd9f1c033d8e0dc4b2805d3e6d881953 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 17:46:41 +0100 Subject: [PATCH 06/20] ApplicationUtils rename to ConfigureApplication --- .../lombok/org/heigit/ohsome/ohsomeapi/Application.java | 4 ++-- .../{ApplicationUtils.java => ConfigureApplication.java} | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) rename src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/{ApplicationUtils.java => ConfigureApplication.java} (98%) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java index ea7a744c..2f5e7ee7 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java @@ -4,7 +4,7 @@ import java.sql.SQLException; import java.util.Properties; import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException; -import org.heigit.ohsome.ohsomeapi.utils.ApplicationUtils; +import org.heigit.ohsome.ohsomeapi.utils.ConfigureApplication; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; @@ -74,7 +74,7 @@ public static void main(String[] args) { */ public static void preRun(ApplicationArguments args) throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { - ApplicationUtils.preRun(args); + ConfigureApplication.preRun(args); } @Override diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java similarity index 98% rename from src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java rename to src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index 9c1b8571..e8ad1fde 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ApplicationUtils.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -17,7 +17,7 @@ import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.springframework.boot.ApplicationArguments; -public class ApplicationUtils { +public class ConfigureApplication { private boolean multithreading = true; private boolean caching = false; @@ -36,7 +36,7 @@ public class ApplicationUtils { private String keytablesType; private String keytablesUrl; - public ApplicationUtils(ApplicationArguments args) { + public ConfigureApplication(ApplicationArguments args) { for (String paramName : args.getOptionNames()) { switch (paramName) { // TODO change to "database.h2" for a future stable version @@ -99,7 +99,7 @@ public ApplicationUtils(ApplicationArguments args) { // refactor remainder back to Application.java public static void preRun(ApplicationArguments args) throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { - var config = new ApplicationUtils(args); + var config = new ConfigureApplication(args); switch (config.databaseType) { case "h2": DbConnData.db = new OSHDBH2(config.databaseUrl); From dc45eef19c80e8e6f0e2bbf72ba9ac997ee89541 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 17:50:13 +0100 Subject: [PATCH 07/20] ConfigureApplication: rename preRun to parseArguments --- src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java | 2 +- .../org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java index 2f5e7ee7..b74d91fc 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java @@ -74,7 +74,7 @@ public static void main(String[] args) { */ public static void preRun(ApplicationArguments args) throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { - ConfigureApplication.preRun(args); + ConfigureApplication.parseArguments(args); } @Override diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index e8ad1fde..385c71dd 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -97,7 +97,7 @@ public ConfigureApplication(ApplicationArguments args) { } // refactor remainder back to Application.java - public static void preRun(ApplicationArguments args) + public static void parseArguments(ApplicationArguments args) throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { var config = new ConfigureApplication(args); switch (config.databaseType) { From 9db53ba5ab34d4e0674ca37e7e06be6e3d8965e0 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 17:50:49 +0100 Subject: [PATCH 08/20] ConfigureApplication: constructor is now private --- .../org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index 385c71dd..64a0ff1b 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -36,7 +36,7 @@ public class ConfigureApplication { private String keytablesType; private String keytablesUrl; - public ConfigureApplication(ApplicationArguments args) { + private ConfigureApplication(ApplicationArguments args) { for (String paramName : args.getOptionNames()) { switch (paramName) { // TODO change to "database.h2" for a future stable version From 7bd5011786d5871f35e73d0fc5df24242c1b666f Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 17:59:18 +0100 Subject: [PATCH 09/20] ConfigureApplication: replace databaseType with enum --- .../ohsomeapi/utils/ConfigureApplication.java | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index 64a0ff1b..4e1ea22b 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -19,6 +19,10 @@ public class ConfigureApplication { + private enum DatabaseType { + H2, JDBC, IGNITE, NONE + } + private boolean multithreading = true; private boolean caching = false; private String dbPrefix = null; @@ -28,12 +32,12 @@ public class ConfigureApplication { private String databaseClassName; private String databaseUser; private String databasePassword; - private String databaseType; + private DatabaseType databaseType = DatabaseType.NONE; private String databaseUrl; private String keytablesClassName; private String keytablesUser; private String keytablesPassword; - private String keytablesType; + private DatabaseType keytablesType = DatabaseType.NONE; private String keytablesUrl; private ConfigureApplication(ApplicationArguments args) { @@ -41,11 +45,11 @@ private ConfigureApplication(ApplicationArguments args) { switch (paramName) { // TODO change to "database.h2" for a future stable version case "database.db": - databaseType = "h2"; + databaseType = DatabaseType.H2; databaseUrl = args.getOptionValues(paramName).get(0); break; case "database.jdbc": - databaseType = "jdbc"; + databaseType = DatabaseType.JDBC; String[] jdbcParam = args.getOptionValues(paramName).get(0).split(";"); databaseClassName = jdbcParam[0]; databaseUrl = jdbcParam[1]; @@ -53,15 +57,15 @@ private ConfigureApplication(ApplicationArguments args) { databasePassword = jdbcParam[3]; break; case "database.ignite": - databaseType = "ignite"; + databaseType = DatabaseType.IGNITE; databaseUrl = args.getOptionValues(paramName).get(0); break; case "database.keytables": - keytablesType = "h2"; + keytablesType = DatabaseType.H2; keytablesUrl = args.getOptionValues(paramName).get(0); break; case "database.keytables.jdbc": - keytablesType = "jdbc"; + keytablesType = DatabaseType.JDBC; String[] keytablesJdbcParam = args.getOptionValues(paramName).get(0).split(";"); keytablesClassName = keytablesJdbcParam[0]; keytablesUrl = keytablesJdbcParam[1]; @@ -101,24 +105,24 @@ public static void parseArguments(ApplicationArguments args) throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { var config = new ConfigureApplication(args); switch (config.databaseType) { - case "h2": + case H2: DbConnData.db = new OSHDBH2(config.databaseUrl); break; - case "jdbc": + case JDBC: DbConnData.db = new OSHDBJdbc(config.databaseClassName, config.databaseUrl, config.databaseUser, config.databasePassword); break; - case "ignite": + case IGNITE: DbConnData.db = new OSHDBIgnite(config.databaseUrl); break; default: throw new IllegalStateException("Unexpected value: " + config.databaseType); } switch (config.keytablesType) { - case "h2": + case H2: DbConnData.db = new OSHDBH2(config.keytablesUrl); break; - case "jdbc": + case JDBC: DbConnData.db = new OSHDBJdbc(config.keytablesClassName, config.keytablesUrl, config.keytablesUser, config.keytablesPassword); DbConnData.mapTagTranslator = new RemoteTagTranslator(() -> { From d6e662f246e7cf33d0039a46cea50608b144db0b Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 18:01:08 +0100 Subject: [PATCH 10/20] ConfigureApplication: keytables is only necessary for IGNITE --- .../heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index 4e1ea22b..efcf8743 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -145,6 +145,9 @@ public static void parseArguments(ApplicationArguments args) DbConnData.keytablesDbPoolConfig = hikariConfig; break; default: + if (config.databaseType != DatabaseType.IGNITE) { + break; + } throw new IllegalStateException("Unexpected value: " + config.keytablesType); } if (DbConnData.db == null) { From b1de45335e8b949bef91b67befa52dd3e944b07d Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 18:17:51 +0100 Subject: [PATCH 11/20] ConfigureApplication: configure keytables first --- .../ohsomeapi/utils/ConfigureApplication.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index efcf8743..b8714db6 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -104,20 +104,6 @@ private ConfigureApplication(ApplicationArguments args) { public static void parseArguments(ApplicationArguments args) throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { var config = new ConfigureApplication(args); - switch (config.databaseType) { - case H2: - DbConnData.db = new OSHDBH2(config.databaseUrl); - break; - case JDBC: - DbConnData.db = new OSHDBJdbc(config.databaseClassName, config.databaseUrl, - config.databaseUser, config.databasePassword); - break; - case IGNITE: - DbConnData.db = new OSHDBIgnite(config.databaseUrl); - break; - default: - throw new IllegalStateException("Unexpected value: " + config.databaseType); - } switch (config.keytablesType) { case H2: DbConnData.db = new OSHDBH2(config.keytablesUrl); @@ -150,6 +136,20 @@ public static void parseArguments(ApplicationArguments args) } throw new IllegalStateException("Unexpected value: " + config.keytablesType); } + switch (config.databaseType) { + case H2: + DbConnData.db = new OSHDBH2(config.databaseUrl); + break; + case JDBC: + DbConnData.db = new OSHDBJdbc(config.databaseClassName, config.databaseUrl, + config.databaseUser, config.databasePassword); + break; + case IGNITE: + DbConnData.db = new OSHDBIgnite(config.databaseUrl); + break; + default: + throw new IllegalStateException("Unexpected value: " + config.databaseType); + } if (DbConnData.db == null) { throw new RuntimeException( "You have to define one of the following three database parameters: '--database.db', " From 0261e950e0a85f405d599548e7918bb12cf0c6c4 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 20:07:45 +0100 Subject: [PATCH 12/20] Update OSHDB dependency to 1.0.0-SNAPSHOT apply changes from OSHDB PR 470 https://github.com/GIScience/oshdb/pull/470/ --- CHANGELOG.md | 4 + README.md | 1 - pom.xml | 3 +- .../executor/ElementsRequestExecutor.java | 35 +++--- .../ohsomeapi/executor/ExecutionUtils.java | 33 +---- .../executor/UsersRequestExecutor.java | 12 +- .../inputprocessing/InputProcessingUtils.java | 9 +- .../inputprocessing/InputProcessor.java | 12 +- .../ohsome/ohsomeapi/oshdb/DbConnData.java | 6 +- .../ohsomeapi/utils/ConfigureApplication.java | 115 +++++++----------- 10 files changed, 90 insertions(+), 140 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09fc1626..fa009e27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ Changelog ## 1.8.0-SNAPSHOT (current master) +### Breaking Changes +* remove caching command line parameter (TODO) +* update to OSHDB 1.0.0-SNAPSHOT (TODO) + ## 1.7.0 diff --git a/README.md b/README.md index 6d56a59b..e66c23c8 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,6 @@ Now you should have a running local API, which is ready for receiving requests u *Note:* * additionally you can add optional run-parameters: * to disable multithreading: `--database.multithreading=false` - * to enable in-memory-caching: `--database.caching=true` (caution.. enabling this option requires quite some memory, but makes processing much faster) * if you want to run the maven project in your IDE, you need to set the paths to your data in the run configurations * in Eclipse: *Run As --> Run Configurations --> (x)= Arguments --> Program arguments: 'enter the parameters here'* * if you want to get information about the code directly, you can access the [Javadoc](https://docs.ohsome.org/java/ohsome-api/), which gets updated daily. diff --git a/pom.xml b/pom.xml index 000698e2..973ee2a0 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,8 @@ 0.17.0 3.2.0 5.6 - 1.0.0-beta-1 + + 1.0.0-SNAPSHOT 1.18.16 src/main/lombok 2.5.14 diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java index 5b506488..95df128d 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java @@ -54,6 +54,7 @@ import org.heigit.ohsome.oshdb.filter.FilterParser; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.geometry.Geo; @@ -180,10 +181,10 @@ public static

Response aggregateGroupByBoundary TagTranslator tt = DbConnData.tagTranslator; Integer[] valuesInt = new Integer[groupByValues.length]; ArrayList> zeroFill = new ArrayList<>(); - int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt(); + int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1); if (groupByValues.length != 0) { for (int j = 0; j < groupByValues.length; j++) { - valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue(); + valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j); zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j])); } } @@ -217,7 +218,7 @@ public static

Response aggregateGroupByBoundary String tagIdentifier; // check for non-remainder objects (which do have the defined key and value) if (entry.getKey().getSecondIndex().getKey() != -1 && tagValue != -1) { - tagIdentifier = tt.getOSMTagOf(keysInt, tagValue).toString(); + tagIdentifier = tt.lookupTag(keysInt, tagValue).toString(); } else { tagIdentifier = "remainder"; } @@ -287,10 +288,10 @@ public static Response aggregateGroupByTag(RequestResource requestResource, TagTranslator tt = DbConnData.tagTranslator; Integer[] valuesInt = new Integer[groupByValues.length]; ArrayList> zeroFill = new ArrayList<>(); - int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt(); + int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1); if (groupByValues.length != 0) { for (int j = 0; j < groupByValues.length; j++) { - valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue(); + valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j); zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j])); } } @@ -307,7 +308,7 @@ public static Response aggregateGroupByTag(RequestResource requestResource, entry.getValue(), requestParameters.isDensity(), df, geom); // check for non-remainder objects (which do have the defined key and value) if (entry.getKey().getKey() != -1 && entry.getKey().getValue() != -1) { - groupByName = tt.getOSMTagOf(keysInt, entry.getKey().getValue()).toString(); + groupByName = tt.lookupTag(keysInt, entry.getKey().getValue()).toString(); } else { groupByName = "remainder"; } @@ -430,7 +431,7 @@ public static Response aggregateGroupByKey(RequestResource requestResource, TagTranslator tt = DbConnData.tagTranslator; Integer[] keysInt = new Integer[groupByKeys.length]; for (int i = 0; i < groupByKeys.length; i++) { - keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).toInt(); + keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).map(OSHDBTagKey::toInt).orElse(-i); } MapAggregator, OSMEntitySnapshot> preResult = mapRed.flatMap(f -> { @@ -460,7 +461,7 @@ public static Response aggregateGroupByKey(RequestResource requestResource, entry.getValue(), requestParameters.isDensity(), df, null); // check for non-remainder objects (which do have the defined key) if (entry.getKey() != -1) { - groupByName = tt.getOSMTagKeyOf(entry.getKey().intValue()).toString(); + groupByName = tt.lookupTag(entry.getKey(), 0).getKey(); } else { groupByName = "remainder"; } @@ -530,17 +531,18 @@ public static Response aggregateBasicFiltersRatio(RequestResource requestResourc Integer[] keysInt2 = new Integer[keys2.length]; Integer[] valuesInt2 = new Integer[values2.length]; for (int i = 0; i < requestParameters.getKeys().length; i++) { - keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).toInt(); + keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).map(OSHDBTagKey::toInt) + .orElse(-i); if (requestParameters.getValues() != null && i < requestParameters.getValues().length) { valuesInt1[i] = tt.getOSHDBTagOf(requestParameters.getKeys()[i], requestParameters.getValues()[i]) - .getValue(); + .map(OSHDBTag::getValue).orElse(-i); } } for (int i = 0; i < keys2.length; i++) { - keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).toInt(); + keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).map(OSHDBTagKey::toInt).orElse(-i); if (i < values2.length) { - valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).getValue(); + valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).map(OSHDBTag::getValue).orElse(-i); } } EnumSet osmTypes1 = @@ -780,17 +782,18 @@ public static

Response aggregateBasicFiltersRat Integer[] valuesInt2 = new Integer[values2.length]; TagTranslator tt = DbConnData.tagTranslator; for (int i = 0; i < requestParameters.getKeys().length; i++) { - keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).toInt(); + keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).map(OSHDBTagKey::toInt) + .orElse(-i); if (requestParameters.getValues() != null && i < requestParameters.getValues().length) { valuesInt1[i] = tt.getOSHDBTagOf(requestParameters.getKeys()[i], requestParameters.getValues()[i]) - .getValue(); + .map(OSHDBTag::getValue).orElse(-i); } } for (int i = 0; i < keys2.length; i++) { - keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).toInt(); + keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).map(OSHDBTagKey::toInt).orElse(-i); if (i < values2.length) { - valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).getValue(); + valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).map(OSHDBTag::getValue).orElse(-i); } } EnumSet osmTypes1 = (EnumSet) processingData.getOsmTypes(); diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java index 39438224..471b4b02 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java @@ -10,12 +10,10 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Streams; import com.opencsv.CSVWriter; -import com.zaxxer.hikari.HikariDataSource; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.Serializable; import java.nio.charset.StandardCharsets; -import java.sql.SQLException; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.ArrayList; @@ -46,7 +44,6 @@ import org.heigit.ohsome.ohsomeapi.controller.dataextraction.elements.ElementsGeometry; import org.heigit.ohsome.ohsomeapi.exception.BadRequestException; import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException; -import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages; import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor; import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData; import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType; @@ -78,15 +75,14 @@ import org.heigit.ohsome.oshdb.api.mapreducer.Mappable; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.celliterator.ContributionType; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; import org.heigit.ohsome.oshdb.util.function.SerializablePredicate; import org.heigit.ohsome.oshdb.util.function.SerializableSupplier; import org.heigit.ohsome.oshdb.util.geometry.Geo; import org.heigit.ohsome.oshdb.util.geometry.OSHDBGeometryBuilder; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; -import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; @@ -140,7 +136,7 @@ public static boolean snapshotMatches(OSMEntitySnapshot snapshot, Set o for (int i = 0; i < keysInt.length; i++) { boolean matchesTag; if (i < valuesInt.length) { - matchesTag = entity.getTags().hasTagValue(keysInt[i], valuesInt[i]); + matchesTag = entity.getTags().hasTag(keysInt[i], valuesInt[i]); } else { matchesTag = entity.getTags().hasTagKey(keysInt[i]); } @@ -856,20 +852,7 @@ private void writeStreamResponse(ThreadLocal outputJsonGen, final ServletOutputStream outputStream) throws ExecutionException, InterruptedException, IOException { ThreadLocal tts; - HikariDataSource keytablesConnectionPool; - if (DbConnData.keytablesDbPoolConfig != null) { - keytablesConnectionPool = new HikariDataSource(DbConnData.keytablesDbPoolConfig); - tts = ThreadLocal.withInitial(() -> { - try { - return new TagTranslator(keytablesConnectionPool.getConnection()); - } catch (OSHDBKeytablesNotFoundException | SQLException e) { - throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); - } - }); - } else { - keytablesConnectionPool = null; - tts = ThreadLocal.withInitial(() -> DbConnData.tagTranslator); - } + tts = ThreadLocal.withInitial(() -> DbConnData.tagTranslator); ReentrantLock lock = new ReentrantLock(); AtomicBoolean errored = new AtomicBoolean(false); ForkJoinPool threadPool = new ForkJoinPool(ProcessingData.getNumberOfDataExtractionThreads()); @@ -879,11 +862,10 @@ private void writeStreamResponse(ThreadLocal outputJsonGen, Map props = data.getProperties(); OSHDBTag[] tags = (OSHDBTag[]) props.remove("@tags"); if (tags != null) { - for (OSHDBTag tag : tags) { - OSMTag osmTag = tts.get().getOSMTagOf(tag); + tts.get().lookupTag(Set.of(tags)).forEach((oshdbTag, osmTag) -> { String key = osmTag.getKey(); props.put(key.startsWith("@") ? "@" + key : key, osmTag.getValue()); - } + }); } // 1. convert features to geojson try { @@ -921,9 +903,6 @@ private void writeStreamResponse(ThreadLocal outputJsonGen, })).get(); } finally { threadPool.shutdown(); - if (keytablesConnectionPool != null) { - keytablesConnectionPool.close(); - } outputStream.flush(); } } @@ -987,7 +966,7 @@ static Set keysToKeysInt(String[] keys, TagTranslator tt) { if (keys.length != 0) { keysInt = new HashSet<>(keys.length); for (String key : keys) { - keysInt.add(tt.getOSHDBTagKeyOf(key).toInt()); + tt.getOSHDBTagKeyOf(key).map(OSHDBTagKey::toInt).ifPresent(keysInt::add); } } else { keysInt = Collections.emptySet(); diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java index 0c294cd2..6e995c8c 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java @@ -30,6 +30,7 @@ import org.heigit.ohsome.oshdb.api.generic.OSHDBCombinedIndex; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; import org.heigit.ohsome.oshdb.util.function.SerializableFunction; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; @@ -119,10 +120,11 @@ public static Response countGroupByTag(HttpServletRequest servletRequest, TagTranslator tt = DbConnData.tagTranslator; Integer[] valuesInt = new Integer[groupByValues.length]; ArrayList> zeroFill = new ArrayList<>(); - int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt(); + int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1); if (groupByValues.length != 0) { for (int j = 0; j < groupByValues.length; j++) { - valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue(); + valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue) + .orElse(-j); zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j])); } } @@ -172,7 +174,7 @@ public static Response countGroupByTag(HttpServletRequest servletRequest, } else if (entry.getKey().getKey() == -1 && entry.getKey().getValue() == -1) { groupByName = "remainder"; } else { - groupByName = tt.getOSMTagOf(keysInt, entry.getKey().getValue()).toString(); + groupByName = tt.lookupTag(keysInt, entry.getKey().getValue()).toString(); } resultSet[count] = new GroupByResult(groupByName, results); count++; @@ -214,7 +216,7 @@ public static Response countGroupByKey(HttpServletRequest servletRequest, TagTranslator tt = DbConnData.tagTranslator; Integer[] keysInt = new Integer[groupByKeys.length]; for (int i = 0; i < groupByKeys.length; i++) { - keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).toInt(); + keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).map(OSHDBTagKey::toInt).orElse(-i); } SortedMap, Integer> result = null; result = mapRed @@ -256,7 +258,7 @@ public static Response countGroupByKey(HttpServletRequest servletRequest, } else if (entry.getKey() == -1) { groupByName = "remainder"; } else { - groupByName = tt.getOSMTagKeyOf(entry.getKey().intValue()).toString(); + groupByName = tt.lookupTag(entry.getKey(), 0).getKey(); } resultSet[count] = new GroupByResult(groupByName, results); count++; diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessingUtils.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessingUtils.java index 25fae145..08070545 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessingUtils.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessingUtils.java @@ -359,11 +359,12 @@ public boolean isSimpleFeatureType(String type) { public MapReducer filterOnPlanarRelations(MapReducer mapRed) { // further filtering to not look at all relations TagTranslator tt = DbConnData.tagTranslator; - OSHDBTag typeMultipolygon = tt.getOSHDBTagOf("type", "multipolygon"); - OSHDBTag typeBoundary = tt.getOSHDBTagOf("type", "boundary"); + OSHDBTag typeMultipolygon = tt.getOSHDBTagOf("type", "multipolygon") + .orElse(new OSHDBTag(-1, -1)); + OSHDBTag typeBoundary = tt.getOSHDBTagOf("type", "boundary").orElse(new OSHDBTag(-1, -1)); return mapRed.filter(Filter.byOSMEntity(entity -> !entity.getType().equals(OSMType.RELATION) - || entity.getTags().hasTagValue(typeMultipolygon.getKey(), typeMultipolygon.getValue()) - || entity.getTags().hasTagValue(typeBoundary.getKey(), typeBoundary.getValue()))); + || entity.getTags().hasTag(typeMultipolygon.getKey(), typeMultipolygon.getValue()) + || entity.getTags().hasTag(typeBoundary.getKey(), typeBoundary.getValue()))); } /** diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java index 3cbd6a82..9a9734dc 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java @@ -196,17 +196,9 @@ public MapReducer processParameters(ComputeMode } DbConnData.db.timeout(timeout); if (isSnapshot) { - if (DbConnData.keytables == null) { - mapRed = OSMEntitySnapshotView.on(DbConnData.db); - } else { - mapRed = OSMEntitySnapshotView.on(DbConnData.db).keytables(DbConnData.keytables); - } + mapRed = OSMEntitySnapshotView.on(DbConnData.db); } else { - if (DbConnData.keytables == null) { - mapRed = OSMContributionView.on(DbConnData.db); - } else { - mapRed = OSMContributionView.on(DbConnData.db).keytables(DbConnData.keytables); - } + mapRed = OSMContributionView.on(DbConnData.db); } if (boundary.isRectangle()) { mapRed = diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/DbConnData.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/DbConnData.java index 97d5e6ed..547d3de9 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/DbConnData.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/oshdb/DbConnData.java @@ -1,6 +1,6 @@ package org.heigit.ohsome.ohsomeapi.oshdb; -import com.zaxxer.hikari.HikariConfig; +import javax.sql.DataSource; import org.heigit.ohsome.oshdb.api.db.OSHDBDatabase; import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; @@ -11,8 +11,8 @@ public class DbConnData { public static OSHDBDatabase db = null; public static OSHDBJdbc keytables = null; public static TagTranslator tagTranslator = null; - public static RemoteTagTranslator mapTagTranslator = null; - public static HikariConfig keytablesDbPoolConfig = null; + public static DataSource dbSource = null; + public static DataSource keytablesDbSource = null; private DbConnData() { throw new IllegalStateException("Utility class"); diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index b8714db6..bb2e054c 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -1,20 +1,19 @@ package org.heigit.ohsome.ohsomeapi.utils; import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; import java.io.IOException; -import java.sql.DriverManager; -import java.sql.SQLException; +import java.nio.file.Path; +import org.h2.jdbcx.JdbcConnectionPool; import org.heigit.ohsome.ohsomeapi.Application; -import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException; -import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages; import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData; import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData; -import org.heigit.ohsome.ohsomeapi.oshdb.RemoteTagTranslator; +import org.heigit.ohsome.oshdb.api.db.H2Support; import org.heigit.ohsome.oshdb.api.db.OSHDBH2; import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException; -import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; +import org.heigit.ohsome.oshdb.util.tagtranslator.CachedTagTranslator; import org.springframework.boot.ApplicationArguments; public class ConfigureApplication { @@ -24,8 +23,7 @@ private enum DatabaseType { } private boolean multithreading = true; - private boolean caching = false; - private String dbPrefix = null; + private String dbPrefix = ""; private long timeoutInMilliseconds = Application.DEFAULT_TIMEOUT_IN_MILLISECONDS; private int numberOfClusterNodes = Application.DEFAULT_NUMBER_OF_CLUSTER_NODES; private int numberOfDataExtractionThreads = Application.DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS; @@ -77,11 +75,6 @@ private ConfigureApplication(ApplicationArguments args) { multithreading = false; } break; - case "database.caching": - if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("true")) { - caching = true; - } - break; case "database.prefix": dbPrefix = args.getOptionValues(paramName).get(0); break; @@ -102,58 +95,49 @@ private ConfigureApplication(ApplicationArguments args) { // refactor remainder back to Application.java public static void parseArguments(ApplicationArguments args) - throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException { + throws OSHDBKeytablesNotFoundException, IOException { var config = new ConfigureApplication(args); switch (config.keytablesType) { case H2: - DbConnData.db = new OSHDBH2(config.keytablesUrl); + DbConnData.keytablesDbSource = JdbcConnectionPool.create( + H2Support.pathToUrl(Path.of(config.keytablesUrl)), "sa", ""); break; case JDBC: - DbConnData.db = new OSHDBJdbc(config.keytablesClassName, config.keytablesUrl, - config.keytablesUser, config.keytablesPassword); - DbConnData.mapTagTranslator = new RemoteTagTranslator(() -> { - try { - Class.forName(config.keytablesClassName); - return new TagTranslator( - DriverManager.getConnection(config.keytablesUrl, config.keytablesUser, - config.keytablesPassword)); - } catch (ClassNotFoundException e) { - throw new RuntimeException("A class with this specific name could not be found"); - } catch (OSHDBKeytablesNotFoundException | SQLException e) { - throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); - } - }); - HikariConfig hikariConfig = new HikariConfig(); - hikariConfig.setJdbcUrl(config.keytablesUrl); - hikariConfig.setUsername(config.keytablesUser); - hikariConfig.setPassword(config.keytablesPassword); - hikariConfig.setMaximumPoolSize(config.numberOfDataExtractionThreads); - DbConnData.keytablesDbPoolConfig = hikariConfig; + var keytablesHc = new HikariConfig(); + keytablesHc.setJdbcUrl(config.keytablesUrl); + keytablesHc.setUsername(config.keytablesUser); + keytablesHc.setPassword(config.keytablesPassword); + keytablesHc.setMaximumPoolSize(config.numberOfDataExtractionThreads); + DbConnData.keytablesDbSource = new HikariDataSource(keytablesHc); break; default: - if (config.databaseType != DatabaseType.IGNITE) { - break; - } - throw new IllegalStateException("Unexpected value: " + config.keytablesType); + break; } switch (config.databaseType) { case H2: DbConnData.db = new OSHDBH2(config.databaseUrl); break; case JDBC: - DbConnData.db = new OSHDBJdbc(config.databaseClassName, config.databaseUrl, - config.databaseUser, config.databasePassword); + var hc = new HikariConfig(); + hc.setJdbcUrl(config.databaseUrl); + hc.setUsername(config.databaseUser); + hc.setPassword(config.databasePassword); + DbConnData.dbSource = new HikariDataSource(hc); + DbConnData.db = new OSHDBJdbc(DbConnData.dbSource, config.dbPrefix, + DbConnData.keytablesDbSource == null ? DbConnData.dbSource + : DbConnData.keytablesDbSource); break; case IGNITE: - DbConnData.db = new OSHDBIgnite(config.databaseUrl); + if (DbConnData.keytablesDbSource == null) { + throw new IllegalArgumentException("Keytables parameter missing"); + } + DbConnData.db = new OSHDBIgnite(config.databaseUrl, config.dbPrefix, + DbConnData.keytablesDbSource); break; default: - throw new IllegalStateException("Unexpected value: " + config.databaseType); - } - if (DbConnData.db == null) { - throw new RuntimeException( - "You have to define one of the following three database parameters: '--database.db', " - + "'--database.ignite', or '--database.jdbc'."); + throw new IllegalArgumentException( + "You have to define one of the following three database parameters: '--database.db', " + + "'--database.ignite', or '--database.jdbc'."); } ProcessingData.setTimeout(config.timeoutInMilliseconds / 1000.0); DbConnData.db.timeoutInMilliseconds(config.timeoutInMilliseconds); @@ -162,35 +146,20 @@ public static void parseArguments(ApplicationArguments args) if (DbConnData.db instanceof OSHDBJdbc) { DbConnData.db = ((OSHDBJdbc) DbConnData.db).multithreading(config.multithreading); } - if (DbConnData.db instanceof OSHDBH2) { - DbConnData.db = ((OSHDBH2) DbConnData.db).inMemory(config.caching); - } - if (DbConnData.keytables != null) { - DbConnData.tagTranslator = new TagTranslator(DbConnData.keytables.getConnection()); - } else { - if (!(DbConnData.db instanceof OSHDBJdbc)) { - throw new DatabaseAccessException("Missing keytables."); - } - DbConnData.tagTranslator = new TagTranslator(((OSHDBJdbc) DbConnData.db).getConnection()); - } + + // initialize TagTranslator + long maxBytesValues = Long.MAX_VALUE; + int maxNumRoles = Integer.MAX_VALUE; + DbConnData.tagTranslator = new CachedTagTranslator(DbConnData.db.getTagTranslator(), + maxBytesValues, maxNumRoles); + + // extract metadata RequestUtils.extractOSHDBMetadata(); - if (DbConnData.mapTagTranslator == null) { - DbConnData.mapTagTranslator = new RemoteTagTranslator(DbConnData.tagTranslator); - } + if (DbConnData.db instanceof OSHDBIgnite) { - RemoteTagTranslator mtt = DbConnData.mapTagTranslator; ((OSHDBIgnite) DbConnData.db).onClose(() -> { - try { - if (mtt.wasEvaluated()) { - mtt.get().getConnection().close(); - } - } catch (SQLException e) { - throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS); - } + // TODO add connections to close }); } - if (config.dbPrefix != null) { - DbConnData.db = DbConnData.db.prefix(config.dbPrefix); - } } } \ No newline at end of file From 981818d2e1e1ad216f30a8b713a806394d426791 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 21:09:26 +0100 Subject: [PATCH 13/20] replace JdbcConnectionPool with H2Support method --- .../heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index bb2e054c..fdcb516a 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -99,8 +99,7 @@ public static void parseArguments(ApplicationArguments args) var config = new ConfigureApplication(args); switch (config.keytablesType) { case H2: - DbConnData.keytablesDbSource = JdbcConnectionPool.create( - H2Support.pathToUrl(Path.of(config.keytablesUrl)), "sa", ""); + DbConnData.keytablesDbSource = H2Support.createJdbcPoolFromPath(config.keytablesUrl); break; case JDBC: var keytablesHc = new HikariConfig(); From fbcc8211dcfcc8b58445574d758fbb4cdf90759d Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Fri, 11 Nov 2022 21:09:45 +0100 Subject: [PATCH 14/20] add todos for CachedTagTranslator parameters --- .../org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index fdcb516a..73891901 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -147,6 +147,8 @@ public static void parseArguments(ApplicationArguments args) } // initialize TagTranslator + // TODO replace with reasonable values + // TODO make configurable long maxBytesValues = Long.MAX_VALUE; int maxNumRoles = Integer.MAX_VALUE; DbConnData.tagTranslator = new CachedTagTranslator(DbConnData.db.getTagTranslator(), From 8e767a0333f14fdbe564ead8f53bb1ef8053aff4 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Tue, 15 Nov 2022 14:20:20 +0100 Subject: [PATCH 15/20] update OSHDB version to 1.0.0-RC1 --- pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 973ee2a0..897f8d0b 100644 --- a/pom.xml +++ b/pom.xml @@ -35,8 +35,7 @@ 0.17.0 3.2.0 5.6 - - 1.0.0-SNAPSHOT + 1.0.0-RC1 1.18.16 src/main/lombok 2.5.14 From 7e91022d3401a7c246a09390fdb4f6c75327760d Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Tue, 15 Nov 2022 14:24:20 +0100 Subject: [PATCH 16/20] ConfigureApplication: remove unused variables --- .../heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index 73891901..de8374b5 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -3,8 +3,6 @@ import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import java.io.IOException; -import java.nio.file.Path; -import org.h2.jdbcx.JdbcConnectionPool; import org.heigit.ohsome.ohsomeapi.Application; import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData; import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData; @@ -27,12 +25,10 @@ private enum DatabaseType { private long timeoutInMilliseconds = Application.DEFAULT_TIMEOUT_IN_MILLISECONDS; private int numberOfClusterNodes = Application.DEFAULT_NUMBER_OF_CLUSTER_NODES; private int numberOfDataExtractionThreads = Application.DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS; - private String databaseClassName; private String databaseUser; private String databasePassword; private DatabaseType databaseType = DatabaseType.NONE; private String databaseUrl; - private String keytablesClassName; private String keytablesUser; private String keytablesPassword; private DatabaseType keytablesType = DatabaseType.NONE; @@ -49,7 +45,6 @@ private ConfigureApplication(ApplicationArguments args) { case "database.jdbc": databaseType = DatabaseType.JDBC; String[] jdbcParam = args.getOptionValues(paramName).get(0).split(";"); - databaseClassName = jdbcParam[0]; databaseUrl = jdbcParam[1]; databaseUser = jdbcParam[2]; databasePassword = jdbcParam[3]; @@ -65,7 +60,6 @@ private ConfigureApplication(ApplicationArguments args) { case "database.keytables.jdbc": keytablesType = DatabaseType.JDBC; String[] keytablesJdbcParam = args.getOptionValues(paramName).get(0).split(";"); - keytablesClassName = keytablesJdbcParam[0]; keytablesUrl = keytablesJdbcParam[1]; keytablesUser = keytablesJdbcParam[2]; keytablesPassword = keytablesJdbcParam[3]; From c3dfb57fea9e3cfb9b7042b110f958aaf772250b Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Tue, 15 Nov 2022 14:26:49 +0100 Subject: [PATCH 17/20] ConfigureApplication: add JavaDoc --- .../ohsome/ohsomeapi/utils/ConfigureApplication.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index de8374b5..37b3d6c6 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -14,6 +14,9 @@ import org.heigit.ohsome.oshdb.util.tagtranslator.CachedTagTranslator; import org.springframework.boot.ApplicationArguments; +/** + * Utility Class for parsing and configuration the ohsome API application. + */ public class ConfigureApplication { private enum DatabaseType { @@ -87,7 +90,10 @@ private ConfigureApplication(ApplicationArguments args) { } } - // refactor remainder back to Application.java + /** + * Method run by the Application class to parse incoming command line arguments + * @param args ApplicationArguments from spring to be parsed. + */ public static void parseArguments(ApplicationArguments args) throws OSHDBKeytablesNotFoundException, IOException { var config = new ConfigureApplication(args); From 6c96824ae78ea0087384a607b1e498c9812ca79a Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Tue, 15 Nov 2022 14:49:39 +0100 Subject: [PATCH 18/20] ConfigureApplication: make tag translator parameters configurable --- .../ohsomeapi/utils/ConfigureApplication.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index 37b3d6c6..f1ad6999 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -36,6 +36,9 @@ private enum DatabaseType { private String keytablesPassword; private DatabaseType keytablesType = DatabaseType.NONE; private String keytablesUrl; + // TODO replace the following two with reasonable values + private long ttMaxBytesValue = Long.MAX_VALUE; + private int ttMaxNumRoles = Integer.MAX_VALUE; private ConfigureApplication(ApplicationArguments args) { for (String paramName : args.getOptionNames()) { @@ -84,6 +87,12 @@ private ConfigureApplication(ApplicationArguments args) { case "cluster.dataextraction.threadcount": numberOfDataExtractionThreads = Integer.parseInt(args.getOptionValues(paramName).get(0)); break; + case "tt.maxbytesvalue": + ttMaxBytesValue = Long.parseLong(args.getOptionValues(paramName).get(0)); + break; + case "tt.maxnumroles": + ttMaxNumRoles = Integer.parseInt(args.getOptionValues(paramName).get(0)); + break; default: break; } @@ -147,12 +156,8 @@ public static void parseArguments(ApplicationArguments args) } // initialize TagTranslator - // TODO replace with reasonable values - // TODO make configurable - long maxBytesValues = Long.MAX_VALUE; - int maxNumRoles = Integer.MAX_VALUE; DbConnData.tagTranslator = new CachedTagTranslator(DbConnData.db.getTagTranslator(), - maxBytesValues, maxNumRoles); + config.ttMaxBytesValue, config.ttMaxNumRoles); // extract metadata RequestUtils.extractOSHDBMetadata(); From fe5b446e195666e6b9a3d1f63cdfa75fc59a84de Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Tue, 15 Nov 2022 14:52:24 +0100 Subject: [PATCH 19/20] Add CHANGELOG --- CHANGELOG.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa009e27..5ed8c8c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,13 @@ Changelog ## 1.8.0-SNAPSHOT (current master) ### Breaking Changes -* remove caching command line parameter (TODO) -* update to OSHDB 1.0.0-SNAPSHOT (TODO) +* remove caching command line parameter ([#281]) +* update to OSHDB 1.0.0-SNAPSHOT ([#281]) + +### New Features +* tag translator parameter are now configurable via CLI parameter: `tt.maxbytesvalue`, `tt.maxnumroles` ([#281]) + +[#281]: https://github.com/GIScience/ohsome-api/pull/281 ## 1.7.0 From baee9f0795da3984edb55418e5a45cdbe5dc9fa6 Mon Sep 17 00:00:00 2001 From: Johannes Visintini Date: Tue, 15 Nov 2022 14:54:06 +0100 Subject: [PATCH 20/20] Set reasonable value for ttMaxBytesValue --- .../heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java index f1ad6999..62cd0015 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/utils/ConfigureApplication.java @@ -36,8 +36,7 @@ private enum DatabaseType { private String keytablesPassword; private DatabaseType keytablesType = DatabaseType.NONE; private String keytablesUrl; - // TODO replace the following two with reasonable values - private long ttMaxBytesValue = Long.MAX_VALUE; + private long ttMaxBytesValue = 512L * 1024L * 1024L; private int ttMaxNumRoles = Integer.MAX_VALUE; private ConfigureApplication(ApplicationArguments args) {