Skip to content

Commit

Permalink
Merge pull request #281 from GIScience/oshdb-1.0.0-SNAPSHOT
Browse files Browse the repository at this point in the history 
update to OSHDB 1.0.0-RC1
  • Loading branch information
@joker234
joker234 authored Nov 15, 2022
2 parents 7e358a5 + baee9f0 commit 515b729
Show file tree
Hide file tree
Showing 12 changed files with 227 additions and 197 deletions.
9 changes: 9 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,15 @@ Changelog

## 1.8.0-SNAPSHOT (current master)

### Breaking Changes
* remove caching command line parameter ([#281])
* update to OSHDB 1.0.0-SNAPSHOT ([#281])

### New Features
* tag translator parameter are now configurable via CLI parameter: `tt.maxbytesvalue`, `tt.maxnumroles` ([#281])

[#281]: https://github.com/GIScience/ohsome-api/pull/281


## 1.7.0

Expand Down
1 change: 0 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,6 @@ Now you should have a running local API, which is ready for receiving requests u
*Note:*
* additionally you can add optional run-parameters:
* to disable multithreading: `--database.multithreading=false`
* to enable in-memory-caching: `--database.caching=true` (caution.. enabling this option requires quite some memory, but makes processing much faster)
* if you want to run the maven project in your IDE, you need to set the paths to your data in the run configurations
* in Eclipse: *Run As --> Run Configurations --> (x)= Arguments --> Program arguments: 'enter the parameters here'*
* if you want to get information about the code directly, you can access the [Javadoc](https://docs.ohsome.org/java/ohsome-api/), which gets updated daily.
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
<jts2geojson.version>0.17.0</jts2geojson.version>
<mavenjar.version>3.2.0</mavenjar.version>
<opencsv.version>5.6</opencsv.version>
<oshdb.version>1.0.0-beta-1</oshdb.version>
<oshdb.version>1.0.0-RC1</oshdb.version>
<projectlombok.version>1.18.16</projectlombok.version>
<sonar.sources>src/main/lombok</sonar.sources>
<springboot.version>2.5.14</springboot.version>
Expand Down
132 changes: 2 additions & 130 deletions src/main/lombok/org/heigit/ohsome/ohsomeapi/Application.java
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,11 @@
package org.heigit.ohsome.ohsomeapi;

import com.zaxxer.hikari.HikariConfig;
import java.io.IOException;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;
import org.heigit.ohsome.ohsomeapi.exception.DatabaseAccessException;
import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages;
import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData;
import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData;
import org.heigit.ohsome.ohsomeapi.oshdb.RemoteTagTranslator;
import org.heigit.ohsome.ohsomeapi.utils.RequestUtils;
import org.heigit.ohsome.oshdb.api.db.OSHDBH2;
import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite;
import org.heigit.ohsome.oshdb.api.db.OSHDBJdbc;
import org.heigit.ohsome.ohsomeapi.utils.ConfigureApplication;
import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException;
import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.DefaultApplicationArguments;
Expand Down Expand Up @@ -84,125 +74,7 @@ public static void main(String[] args) {
*/
public static void preRun(ApplicationArguments args)
throws ClassNotFoundException, SQLException, OSHDBKeytablesNotFoundException, IOException {
final String dbProperty = "database.db";
boolean multithreading = true;
boolean caching = false;
String dbPrefix = null;
long timeoutInMilliseconds = DEFAULT_TIMEOUT_IN_MILLISECONDS;
int numberOfClusterNodes = DEFAULT_NUMBER_OF_CLUSTER_NODES;
int numberOfDataExtractionThreads = DEFAULT_NUMBER_OF_DATA_EXTRACTION_THREADS;
// only used when tests are executed directly in Eclipse
if (System.getProperty(dbProperty) != null) {
DbConnData.db = new OSHDBH2(System.getProperty(dbProperty));
}
for (String paramName : args.getOptionNames()) {
switch (paramName) {
case dbProperty:
DbConnData.db = new OSHDBH2(args.getOptionValues(paramName).get(0));
break;
case "database.jdbc":
String[] jdbcParam = args.getOptionValues(paramName).get(0).split(";");
DbConnData.db = new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]);
break;
case "database.ignite":
if (DbConnData.db != null) {
break;
}
DbConnData.db = new OSHDBIgnite(args.getOptionValues(paramName).get(0));
break;
case "database.keytables":
DbConnData.keytables = new OSHDBH2(args.getOptionValues(paramName).get(0));
break;
case "database.keytables.jdbc":
jdbcParam = args.getOptionValues(paramName).get(0).split(";");
DbConnData.keytables =
new OSHDBJdbc(jdbcParam[0], jdbcParam[1], jdbcParam[2], jdbcParam[3]);
DbConnData.mapTagTranslator = new RemoteTagTranslator(() -> {
try {
Class.forName(jdbcParam[0]);
return new TagTranslator(
DriverManager.getConnection(jdbcParam[1], jdbcParam[2], jdbcParam[3]));
} catch (ClassNotFoundException e) {
throw new RuntimeException("A class with this specific name could not be found");
} catch (OSHDBKeytablesNotFoundException | SQLException e) {
throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS);
}
});
HikariConfig hikariConfig = new HikariConfig();
hikariConfig.setJdbcUrl(jdbcParam[1]);
hikariConfig.setUsername(jdbcParam[2]);
hikariConfig.setPassword(jdbcParam[3]);
hikariConfig.setMaximumPoolSize(numberOfDataExtractionThreads);
DbConnData.keytablesDbPoolConfig = hikariConfig;
break;
case "database.multithreading":
if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("false")) {
multithreading = false;
}
break;
case "database.caching":
if (args.getOptionValues(paramName).get(0).equalsIgnoreCase("true")) {
caching = true;
}
break;
case "database.prefix":
dbPrefix = args.getOptionValues(paramName).get(0);
break;
case "database.timeout":
timeoutInMilliseconds = Long.parseLong(args.getOptionValues(paramName).get(0));
break;
case "cluster.servernodes.count":
numberOfClusterNodes = Integer.parseInt(args.getOptionValues(paramName).get(0));
break;
case "cluster.dataextraction.threadcount":
numberOfDataExtractionThreads = Integer.parseInt(args.getOptionValues(paramName).get(0));
break;
default:
break;
}
}
if (DbConnData.db == null) {
throw new RuntimeException(
"You have to define one of the following three database parameters: '--database.db', "
+ "'--database.ignite', or '--database.jdbc'.");
}
ProcessingData.setTimeout(timeoutInMilliseconds / 1000.0);
DbConnData.db.timeoutInMilliseconds(timeoutInMilliseconds);
ProcessingData.setNumberOfClusterNodes(numberOfClusterNodes);
ProcessingData.setNumberOfDataExtractionThreads(numberOfDataExtractionThreads);
if (DbConnData.db instanceof OSHDBJdbc) {
DbConnData.db = ((OSHDBJdbc) DbConnData.db).multithreading(multithreading);
}
if (DbConnData.db instanceof OSHDBH2) {
DbConnData.db = ((OSHDBH2) DbConnData.db).inMemory(caching);
}
if (DbConnData.keytables != null) {
DbConnData.tagTranslator = new TagTranslator(DbConnData.keytables.getConnection());
} else {
if (!(DbConnData.db instanceof OSHDBJdbc)) {
throw new DatabaseAccessException("Missing keytables.");
}
DbConnData.tagTranslator = new TagTranslator(((OSHDBJdbc) DbConnData.db).getConnection());
}
RequestUtils.extractOSHDBMetadata();
if (DbConnData.mapTagTranslator == null) {
DbConnData.mapTagTranslator = new RemoteTagTranslator(DbConnData.tagTranslator);
}
if (DbConnData.db instanceof OSHDBIgnite) {
RemoteTagTranslator mtt = DbConnData.mapTagTranslator;
((OSHDBIgnite) DbConnData.db).onClose(() -> {
try {
if (mtt.wasEvaluated()) {
mtt.get().getConnection().close();
}
} catch (SQLException e) {
throw new DatabaseAccessException(ExceptionMessages.DATABASE_ACCESS);
}
});
}
if (dbPrefix != null) {
DbConnData.db = DbConnData.db.prefix(dbPrefix);
}
ConfigureApplication.parseArguments(args);
}

@Override
Expand Down
35 changes: 19 additions & 16 deletions src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import org.heigit.ohsome.oshdb.filter.FilterParser;
import org.heigit.ohsome.oshdb.osm.OSMEntity;
import org.heigit.ohsome.oshdb.osm.OSMType;
import org.heigit.ohsome.oshdb.util.OSHDBTagKey;
import org.heigit.ohsome.oshdb.util.celliterator.ContributionType;
import org.heigit.ohsome.oshdb.util.function.SerializableFunction;
import org.heigit.ohsome.oshdb.util.geometry.Geo;
Expand Down Expand Up @@ -180,10 +181,10 @@ public static <P extends Geometry & Polygonal> Response aggregateGroupByBoundary
TagTranslator tt = DbConnData.tagTranslator;
Integer[] valuesInt = new Integer[groupByValues.length];
ArrayList<Pair<Integer, Integer>> zeroFill = new ArrayList<>();
int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt();
int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1);
if (groupByValues.length != 0) {
for (int j = 0; j < groupByValues.length; j++) {
valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue();
valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j);
zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j]));
}
}
Expand Down Expand Up @@ -217,7 +218,7 @@ public static <P extends Geometry & Polygonal> Response aggregateGroupByBoundary
String tagIdentifier;
// check for non-remainder objects (which do have the defined key and value)
if (entry.getKey().getSecondIndex().getKey() != -1 && tagValue != -1) {
tagIdentifier = tt.getOSMTagOf(keysInt, tagValue).toString();
tagIdentifier = tt.lookupTag(keysInt, tagValue).toString();
} else {
tagIdentifier = "remainder";
}
Expand Down Expand Up @@ -287,10 +288,10 @@ public static Response aggregateGroupByTag(RequestResource requestResource,
TagTranslator tt = DbConnData.tagTranslator;
Integer[] valuesInt = new Integer[groupByValues.length];
ArrayList<Pair<Integer, Integer>> zeroFill = new ArrayList<>();
int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).toInt();
int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1);
if (groupByValues.length != 0) {
for (int j = 0; j < groupByValues.length; j++) {
valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).getValue();
valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j);
zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j]));
}
}
Expand All @@ -307,7 +308,7 @@ public static Response aggregateGroupByTag(RequestResource requestResource,
entry.getValue(), requestParameters.isDensity(), df, geom);
// check for non-remainder objects (which do have the defined key and value)
if (entry.getKey().getKey() != -1 && entry.getKey().getValue() != -1) {
groupByName = tt.getOSMTagOf(keysInt, entry.getKey().getValue()).toString();
groupByName = tt.lookupTag(keysInt, entry.getKey().getValue()).toString();
} else {
groupByName = "remainder";
}
Expand Down Expand Up @@ -430,7 +431,7 @@ public static Response aggregateGroupByKey(RequestResource requestResource,
TagTranslator tt = DbConnData.tagTranslator;
Integer[] keysInt = new Integer[groupByKeys.length];
for (int i = 0; i < groupByKeys.length; i++) {
keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).toInt();
keysInt[i] = tt.getOSHDBTagKeyOf(groupByKeys[i]).map(OSHDBTagKey::toInt).orElse(-i);
}
MapAggregator<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, OSMEntitySnapshot> preResult =
mapRed.flatMap(f -> {
Expand Down Expand Up @@ -460,7 +461,7 @@ public static Response aggregateGroupByKey(RequestResource requestResource,
entry.getValue(), requestParameters.isDensity(), df, null);
// check for non-remainder objects (which do have the defined key)
if (entry.getKey() != -1) {
groupByName = tt.getOSMTagKeyOf(entry.getKey().intValue()).toString();
groupByName = tt.lookupTag(entry.getKey(), 0).getKey();
} else {
groupByName = "remainder";
}
Expand Down Expand Up @@ -530,17 +531,18 @@ public static Response aggregateBasicFiltersRatio(RequestResource requestResourc
Integer[] keysInt2 = new Integer[keys2.length];
Integer[] valuesInt2 = new Integer[values2.length];
for (int i = 0; i < requestParameters.getKeys().length; i++) {
keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).toInt();
keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).map(OSHDBTagKey::toInt)
.orElse(-i);
if (requestParameters.getValues() != null && i < requestParameters.getValues().length) {
valuesInt1[i] =
tt.getOSHDBTagOf(requestParameters.getKeys()[i], requestParameters.getValues()[i])
.getValue();
.map(OSHDBTag::getValue).orElse(-i);
}
}
for (int i = 0; i < keys2.length; i++) {
keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).toInt();
keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).map(OSHDBTagKey::toInt).orElse(-i);
if (i < values2.length) {
valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).getValue();
valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).map(OSHDBTag::getValue).orElse(-i);
}
}
EnumSet<OSMType> osmTypes1 =
Expand Down Expand Up @@ -780,17 +782,18 @@ public static <P extends Geometry & Polygonal> Response aggregateBasicFiltersRat
Integer[] valuesInt2 = new Integer[values2.length];
TagTranslator tt = DbConnData.tagTranslator;
for (int i = 0; i < requestParameters.getKeys().length; i++) {
keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).toInt();
keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).map(OSHDBTagKey::toInt)
.orElse(-i);
if (requestParameters.getValues() != null && i < requestParameters.getValues().length) {
valuesInt1[i] =
tt.getOSHDBTagOf(requestParameters.getKeys()[i], requestParameters.getValues()[i])
.getValue();
.map(OSHDBTag::getValue).orElse(-i);
}
}
for (int i = 0; i < keys2.length; i++) {
keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).toInt();
keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).map(OSHDBTagKey::toInt).orElse(-i);
if (i < values2.length) {
valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).getValue();
valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).map(OSHDBTag::getValue).orElse(-i);
}
}
EnumSet<OSMType> osmTypes1 = (EnumSet<OSMType>) processingData.getOsmTypes();
Expand Down
Loading

0 comments on commit 515b729

Please sign in to comment.