From f14e509bd1a795630755608748dae55a4337653d Mon Sep 17 00:00:00 2001 From: Martin Raifer Date: Wed, 16 Nov 2022 18:36:27 +0100 Subject: [PATCH] drop support for deprecated types/keys/values filter parameters --- docs/endpoints.rst | 28 +- docs/filter.rst | 3 - .../controller/ParameterDescriptions.java | 3 - .../exception/ExceptionMessages.java | 4 - .../executor/AggregateRequestExecutor.java | 11 +- .../executor/ContributionsExecutor.java | 8 +- .../executor/DataExtractionTransformer.java | 39 +- .../executor/DataRequestExecutor.java | 22 +- .../executor/ElementsRequestExecutor.java | 391 +----------------- .../ohsomeapi/executor/ExecutionUtils.java | 19 +- .../ohsomeapi/executor/RequestParameters.java | 3 - .../executor/UsersRequestExecutor.java | 16 +- .../inputprocessing/InputProcessor.java | 258 +----------- .../inputprocessing/ProcessingData.java | 5 - .../inputprocessing/ResourceParameters.java | 15 +- .../inputprocessing/StringSimilarity.java | 1 - .../controller/GetControllerTest.java | 24 +- .../executor/ContributionsExecutorTest.java | 5 +- .../inputprocessing/GeometryBuilderTest.java | 12 +- 19 files changed, 75 insertions(+), 792 deletions(-) diff --git a/docs/endpoints.rst b/docs/endpoints.rst index 11e94c98..ce752724 100644 --- a/docs/endpoints.rst +++ b/docs/endpoints.rst @@ -4,9 +4,6 @@ API Endpoints .. note:: For **POST requests** the fields are given analogous to **GET requests**. When you just have a smaller set of spatial parameters, a GET request fits perfectly. POST mostly makes sense when you start to use GeoJSON as input geometries. - The usage of the parameters **types**, **keys** and **values** is not recommended as they are deprecated. Please use the - filter_ parameter for your requests. - Elements Aggregation -------------------- @@ -22,9 +19,6 @@ Elements Aggregation :query format: 'json' or 'csv'; default: 'json' :query showMetadata: add additional metadata information to the response: 'true', 'false', 'yes', 'no'; default: 'false' :query timeout: custom timeout to limit the processing time in seconds; default: dependent on server settings, retrievable via the /metadata request - :query types: Deprecated! Use **filter** parameter instead! Old parameter which allowed to specify OSM type(s) ‘node’ and/or ‘way’ and/or ‘relation’ OR simple feature type(s) ‘point’ and/or ‘line’ and/or 'polygon’ and/or 'other'; default: all three OSM types - :query keys: Deprecated! Use **filter** parameter instead! Old parameter which allowed to specify OSM key(s) given as a list and combined with the 'AND' operator; default: empty - :query values: Deprecated! Use **filter** parameter instead! Old parameter which allowed to specify OSM value(s) given as a list and combined with the 'AND' operator; values(n) MUST fit to keys(n); default: empty **Example request**: @@ -217,9 +211,6 @@ What is the density of restaurants with wheelchair access in Heidelberg? :query : see above_ :query filter2: see filter_ - :query keys2: Deprecated! see **filter2** - :query types2: Deprecated! use **filter2** - :query values2: Deprecated! see **filter2** .. note:: The result of a **ratio request** may contain the value **"NaN"**, when the ratio calculation involves a division of zero by zero. @@ -819,9 +810,6 @@ Compare length of different types of streets for two or more regions. :query : see above_ :query filter2: see filter_ - :query keys2: Deprecated! see **filter2** - :query types2: Deprecated! use **filter2** - :query values2: Deprecated! see **filter2** Users Aggregation ----------------- @@ -1008,7 +996,7 @@ Contributions Aggregation .. http:post :: /contributions/count - Get the count of the contributions provided to the OSM data. This endpoint does not support the deprecated ``types``, ``keys``, ``values`` parameters. List of endpoints: + Get the count of the contributions provided to the OSM data. List of endpoints: * **/count** * **/count/density** @@ -1126,7 +1114,7 @@ Number of contributions to the building 'Stadthalle Heidelberg' between 2010 and .. http:post :: /contributions/count/density - Get the density of the count of contributions in the total query area in counts per square-kilometers. This endpoint does not support the deprecated ``types``, ``keys``, ``values`` parameters. + Get the density of the count of contributions in the total query area in counts per square-kilometers. **Example request**: @@ -1230,7 +1218,7 @@ Density of contributions to shops within the oldtown area of Heidelberg between .. http:post :: /contributions/latest/count - Get the count of the latest contributions provided to the OSM data. This endpoint does not support the deprecated ``types``, ``keys``, ``values`` parameters. + Get the count of the latest contributions provided to the OSM data. **Example request**: @@ -1334,7 +1322,7 @@ Number of the latest contributions to residential buildings with a geometry chan .. http:post :: /contributions/latest/count/density - Get the density of the count of the latest contributions in the total query area in counts per square-kilometers. This endpoint does not support the deprecated ``types``, ``keys``, ``values`` parameters. + Get the density of the count of the latest contributions in the total query area in counts per square-kilometers. **Example request**: @@ -1438,7 +1426,7 @@ Density of the latest contributions with a geometry change to shops within the o .. http:post :: /contributions/count/groupBy/boundary - Get the count of the contributions provided to the OSM data grouped by the specified boundaries of the query. This endpoint does not support the deprecated ``types``, ``keys``, ``values`` parameters. + Get the count of the contributions provided to the OSM data grouped by the specified boundaries of the query. **Example request**: @@ -1574,7 +1562,7 @@ Number of contributions to shops in different suburbs of Heidelberg (Altstadt an .. http:post :: /contributions/count/density/groupBy/boundary - Get the density of the count of contributions in the total query area in counts per square-kilometers grouped by the specified boundaries of the query. This endpoint does not support the deprecated ``types``, ``keys``, ``values`` parameters. + Get the density of the count of contributions in the total query area in counts per square-kilometers grouped by the specified boundaries of the query. **Example request**: @@ -1975,7 +1963,7 @@ Contributions Extraction .. http:post :: /contributions/(geometryType) - Get the contributions provided to the OSM data. This endpoint does not support the deprecated ``types``, ``keys``, ``values`` parameters. + Get the contributions provided to the OSM data. This endpoint supports the geometry types ``bbox``, ``centroid`` and ``geometry``. :query : see above_ (except **format**) @@ -2327,7 +2315,7 @@ Get the changes of pharmacies with opening hours in a certain area of Heidelberg .. http:post :: /contributions/latest/(geometryType) - Get the the latest state of the contributions provided to the OSM data. This endpoint does not support the deprecated ``types``, ``keys``, ``values`` parameters. + Get the the latest state of the contributions provided to the OSM data. This endpoint supports the geometry types ``bbox``, ``centroid`` and ``geometry``. :query : see above_ (except **format**) diff --git a/docs/filter.rst b/docs/filter.rst index d08f4afb..09546997 100644 --- a/docs/filter.rst +++ b/docs/filter.rst @@ -5,9 +5,6 @@ The filter parameter combines the following: definition of the OSM type, the geometry (simple feature) type, as well as the OSM tag. The filter syntax is defined in textual form. A filter expression can be composed out of several actual filters, which are combined with boolean operators and parentheses. -.. note:: Please note that you **cannot combine** - the filter parameter with the **deprecated types, keys and values** parameters. - | Selectors diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/controller/ParameterDescriptions.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/controller/ParameterDescriptions.java index 6a5f18cd..314446b7 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/controller/ParameterDescriptions.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/controller/ParameterDescriptions.java @@ -15,9 +15,6 @@ public class ParameterDescriptions { + "FeatureCollection. The first point has to be the same as the last point and " + "MultiPolygons are only supported in GeoJSON; no default value " + "(one boundary parameter must be defined)"; - public static final String TYPES = - "OSM type(s) 'node' and/or 'way' and/or 'relation' OR simple feature type(s) 'point' and/or " - + "'line' and/or 'polygon'; default: all three OSM types"; public static final String KEYS = "OSM key(s) e.g.: 'highway', 'building'; default: no key"; public static final String GROUP_BY_KEY = "OSM key e.g.: 'highway', 'building'; no default " + "value (one groupByKey parameter must be defined)"; diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/exception/ExceptionMessages.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/exception/ExceptionMessages.java index 0a7563a8..949623ff 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/exception/ExceptionMessages.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/exception/ExceptionMessages.java @@ -25,16 +25,12 @@ public class ExceptionMessages { + "bcircles, or bpolys) does not fit its format, or you defined more than one boundary " + "parameter."; public static final String DATABASE_ACCESS = "Keytables not found or access to database failed"; - public static final String FILTER_PARAM = "The keys, values and types parameters must be empty, " - + "when you set the filter parameter."; public static final String FILTER_SYNTAX = "Invalid filter syntax. Please look at the additional " + "info and examples about the filter parameter at https://docs.ohsome.org/ohsome-api."; public static final String GROUP_BY_KEY_PARAM = "You need to give one groupByKey parameter, if you want to use groupBy/tag."; public static final String GROUP_BY_KEYS_PARAM = "You need to give one groupByKeys parameter, if you want to use groupBy/key."; - public static final String KEYS_VALUES_RATIO_INVALID = "There cannot be more input values in the " - + "values|values2 than in the keys|keys2 parameter, as values_n must fit to keys_n."; public static final String NO_DEFINED_PARAMS = "The query did not specify any parameter. " + "Please remember: " + ExceptionMessages.NO_BOUNDARY; public static final String NO_BOUNDARY = diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/AggregateRequestExecutor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/AggregateRequestExecutor.java index 939694b6..382071b8 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/AggregateRequestExecutor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/AggregateRequestExecutor.java @@ -79,8 +79,7 @@ public AggregateRequestExecutor(RequestResource requestResource, */ public Response aggregate() throws Exception { final SortedMap result; - MapReducer mapRed = null; - mapRed = inputProcessor.processParameters(); + MapReducer mapRed = inputProcessor.processParameters(); var mapRedGeom = mapRed.map(OSMEntitySnapshot::getGeometry); switch (requestResource) { case COUNT: @@ -136,7 +135,7 @@ public Response aggregateGroupByBoundary() throws Exception { MapReducer mapRed = inputProcessor.processParameters(); InputProcessingUtils utils = inputProcessor.getUtils(); var result = computeCountLengthPerimeterAreaGbB(requestResource, - processingData.getBoundaryType(), mapRed, inputProcessor); + processingData.getBoundaryType(), mapRed); SortedMap> groupByResult; groupByResult = ExecutionUtils.nest(result); GroupByResult[] resultSet = new GroupByResult[groupByResult.size()]; @@ -336,8 +335,7 @@ private ElementsResult[] fillElementsResult(SortedMap SortedMap, ? extends Number> computeCountLengthPerimeterAreaGbB(RequestResource requestResource, - BoundaryType boundaryType, MapReducer mapRed, - InputProcessor inputProcessor) throws Exception { + BoundaryType boundaryType, MapReducer mapRed) throws Exception { if (boundaryType == BoundaryType.NOBOUNDARY) { throw new BadRequestException(ExceptionMessages.NO_BOUNDARY); } @@ -347,9 +345,6 @@ private ElementsResult[] fillElementsResult(SortedMap idx, idx -> (P) arrGeoms.get(idx))); MapAggregator, OSMEntitySnapshot> mapAgg = mapRed.aggregateByTimestamp().aggregateByGeometry(geoms); - if (processingData.isContainingSimpleFeatureTypes()) { - mapAgg = inputProcessor.filterOnSimpleFeatures(mapAgg); - } Optional filter = processingData.getFilterExpression(); if (filter.isPresent()) { mapAgg = mapAgg.filter(filter.get()); diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ContributionsExecutor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ContributionsExecutor.java index c37d5f4c..a9709620 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ContributionsExecutor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ContributionsExecutor.java @@ -78,7 +78,6 @@ public ContributionsExecutor(HttpServletRequest servletRequest, */ public Response count(boolean isUsersRequest, boolean isContributionsLatestCount) throws UnsupportedOperationException, Exception { - MapReducer mapRed; final SortedMap result; if (isContributionsLatestCount) { // the setFullHistory flag needs to be set, because @@ -87,7 +86,7 @@ public Response count(boolean isUsersRequest, boolean isContributionsLatestCount inputProcessor.getProcessingData() .setFullHistory(true); } - mapRed = inputProcessor.processParameters(); + MapReducer mapRed = inputProcessor.processParameters(); if (isUsersRequest) { result = usersCount(mapRed); } else { @@ -191,7 +190,7 @@ private SortedMap contributionsCount(MapReducer & Serializable> Response countGroupByBoundary(boolean isUsersRequest) throws Exception { inputProcessor.getProcessingData().setGroupByBoundary(true); - var mapRed = inputProcessor.processParameters(); + MapReducer mapRed = inputProcessor.processParameters(); final var requestParameters = processingData.getRequestParameters(); List arrGeoms = processingData.getBoundaryList(); var arrGeomIds = inputProcessor.getUtils() @@ -211,9 +210,6 @@ V extends Comparable & Serializable> Response countGroupByBoundary(boolean is if (filter.isPresent()) { mapAgg = mapAgg.filter(filter.get()); } - if (isUsersRequest && processingData.isContainingSimpleFeatureTypes()) { - mapAgg = inputProcessor.filterOnSimpleFeatures(mapAgg); - } SortedMap, Integer> result; if (isUsersRequest) { result = mapAgg.filter(ExecutionUtils.contributionsFilter(servletRequest diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataExtractionTransformer.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataExtractionTransformer.java index 04fa160b..561c2d3b 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataExtractionTransformer.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataExtractionTransformer.java @@ -6,11 +6,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TreeMap; import org.heigit.ohsome.ohsomeapi.controller.dataextraction.elements.ElementsGeometry; -import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessingUtils; -import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType; import org.heigit.ohsome.ohsomeapi.utils.TimestampFormatter; import org.heigit.ohsome.oshdb.filter.FilterExpression; import org.heigit.ohsome.oshdb.osm.OSMEntity; @@ -39,14 +36,7 @@ public class DataExtractionTransformer implements Serializable { private final boolean includeOSMMetadata; private final boolean includeContributionTypes; private final ElementsGeometry outputGeometry; - private final InputProcessingUtils inputUtils; private final ExecutionUtils exeUtils; - @Deprecated - private final Set keysInt; - @Deprecated - private final Set simpleFeatureTypes; - @Deprecated - private final boolean isContainingSimpleFeatureTypes; /** * Creates a new data extraction transformer, adhering to the given parameters. @@ -66,37 +56,26 @@ public class DataExtractionTransformer implements Serializable { * changeset id, timestamp, version number) * @param includeContributionTypes set true if the result should include the contribution type * for `/elements/contributions` resources - * @param inputUtils input processing utility object * @param exeUtils the execution utils object - * @param keysInt (for the deprecated `keys` filter parameter) set the list of always to be - * returned OSM tags in the GeoJSON's features' properties * @param outputGeometry specifies what should be returned as the GeoJSON feature's geometry: * either the full geometry, its bbox or its centroid. - * @param simpleFeatureTypes if the query uses the (deprecated) types parameter, and it contains - * simple feature "geometry" types, specify the set of to be returned geometry types here - * @param isContainingSimpleFeatureTypes set true if the query uses the (deprecated) types */ public DataExtractionTransformer(String startTimestamp, String endTimestamp, FilterExpression filter, boolean isContributionsEndpoint, boolean isContributionsLatestEndpoint, boolean clipGeometries, boolean includeTags, - boolean includeOSMMetadata, boolean includeContributionTypes, InputProcessingUtils inputUtils, - ExecutionUtils exeUtils, Set keysInt, ElementsGeometry outputGeometry, - Set simpleFeatureTypes, boolean isContainingSimpleFeatureTypes) { + boolean includeOSMMetadata, boolean includeContributionTypes, + ExecutionUtils exeUtils, ElementsGeometry outputGeometry) { this.isContributionsLatestEndpoint = isContributionsLatestEndpoint; this.isContributionsEndpoint = isContributionsEndpoint; this.exeUtils = exeUtils; this.clipGeometries = clipGeometries; this.startTimestamp = startTimestamp; - this.inputUtils = inputUtils; - this.simpleFeatureTypes = simpleFeatureTypes; this.filter = filter; - this.keysInt = keysInt; this.includeTags = includeTags; this.includeOSMMetadata = includeOSMMetadata; this.includeContributionTypes = includeContributionTypes; this.outputGeometry = outputGeometry; this.endTimestamp = endTimestamp; - this.isContainingSimpleFeatureTypes = isContainingSimpleFeatureTypes; } /** @@ -156,7 +135,7 @@ public List buildChangedFeatures(List contributions) { properties.put(TIMESTAMP_PROPERTY, validTo); properties.put(CONTRIBUTION_CHANGESET_ID_PROPERTY, contribution.getChangesetId()); } - output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, + output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, includeTags, includeOSMMetadata, includeContributionTypes, isContributionsEndpoint, outputGeometry, contribution.getContributionTypes())); } @@ -192,7 +171,7 @@ public List buildChangedFeatures(List contributions) { if (!currentGeom.isEmpty()) { boolean addToOutput = addEntityToOutput(currentEntity, currentGeom); if (addToOutput) { - output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, + output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, includeTags, includeOSMMetadata, includeContributionTypes, isContributionsEndpoint, outputGeometry, lastContribution.getContributionTypes())); } @@ -204,7 +183,7 @@ public List buildChangedFeatures(List contributions) { properties.put(TIMESTAMP_PROPERTY, TimestampFormatter.getInstance().isoDateTime(lastContribution.getTimestamp())); properties.put(CONTRIBUTION_CHANGESET_ID_PROPERTY, lastContribution.getChangesetId()); - output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, keysInt, false, + output.add(exeUtils.createOSMFeature(currentEntity, currentGeom, properties, false, includeOSMMetadata, includeContributionTypes, isContributionsEndpoint, outputGeometry, lastContribution.getContributionTypes())); } @@ -238,7 +217,7 @@ public List buildUnchangedFeatures(OSMEntitySnapshot snapshot) { boolean addToOutput = addEntityToOutput(entity, geom); if (addToOutput) { return Collections.singletonList(exeUtils.createOSMFeature(entity, geom, properties, - keysInt, includeTags, includeOSMMetadata, includeContributionTypes, + includeTags, includeOSMMetadata, includeContributionTypes, isContributionsEndpoint, outputGeometry, EnumSet.noneOf(ContributionType.class))); } else { return Collections.emptyList(); @@ -247,10 +226,6 @@ public List buildUnchangedFeatures(OSMEntitySnapshot snapshot) { /** Checks whether the given entity should be added to the output (true) or not (false). */ private boolean addEntityToOutput(OSMEntity currentEntity, Geometry currentGeom) { - if (isContainingSimpleFeatureTypes) { - return inputUtils.checkGeometryOnSimpleFeatures(currentGeom, simpleFeatureTypes); - } else { - return filter == null || filter.applyOSMGeometry(currentEntity, currentGeom); - } + return filter == null || filter.applyOSMGeometry(currentEntity, currentGeom); } } diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java index 992b8655..6de62e08 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/DataRequestExecutor.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.stream.Stream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -13,10 +12,8 @@ import org.heigit.ohsome.ohsomeapi.controller.dataextraction.elements.ElementsGeometry; import org.heigit.ohsome.ohsomeapi.exception.BadRequestException; import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages; -import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessingUtils; import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor; import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData; -import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType; import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData; import org.heigit.ohsome.ohsomeapi.output.ExtractionResponse; import org.heigit.ohsome.ohsomeapi.output.Metadata; @@ -26,7 +23,6 @@ import org.heigit.ohsome.oshdb.filter.FilterExpression; import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; -import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.heigit.ohsome.oshdb.util.time.IsoDateTimeParser; import org.wololo.geojson.Feature; @@ -63,8 +59,8 @@ public void extract() throws Exception { inputProcessor.getProcessingData().setFullHistory(true); InputProcessor snapshotInputProcessor = new InputProcessor(servletRequest, true, false); snapshotInputProcessor.getProcessingData().setFullHistory(true); - MapReducer mapRedSnapshot = null; - MapReducer mapRedContribution = null; + MapReducer mapRedSnapshot; + MapReducer mapRedContribution; if (DbConnData.db instanceof OSHDBIgnite) { // on ignite: Use AffinityCall backend, which is the only one properly supporting streaming // of result data, without buffering the whole result in memory before returning the result. @@ -75,20 +71,15 @@ public void extract() throws Exception { mapRedSnapshot = snapshotInputProcessor.processParameters(); mapRedContribution = inputProcessor.processParameters(); } - RequestParameters requestParameters = processingData.getRequestParameters(); String[] time = inputProcessor.splitParamOnComma( inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("time"))); if (time.length != 2) { throw new BadRequestException( ExceptionMessages.TIME_FORMAT_CONTRS_EXTRACTION_AND_FULL_HISTORY); } - TagTranslator tt = DbConnData.tagTranslator; - String[] keys = requestParameters.getKeys(); - final Set keysInt = ExecutionUtils.keysToKeysInt(keys, tt); final ExecutionUtils exeUtils = new ExecutionUtils(processingData); inputProcessor.processPropertiesParam(); inputProcessor.processIsUnclippedParam(); - InputProcessingUtils utils = inputProcessor.getUtils(); final boolean includeTags = inputProcessor.includeTags(); final boolean includeOSMMetadata = inputProcessor.includeOSMMetadata(); final boolean includeContributionTypes = inputProcessor.includeContributionTypes(); @@ -97,7 +88,7 @@ public void extract() throws Exception { requestResource.equals(RequestResource.CONTRIBUTIONSLATEST); final boolean isContributionsEndpoint = isContributionsLatestEndpoint || requestResource.equals(RequestResource.CONTRIBUTIONS); - final Set simpleFeatureTypes = processingData.getSimpleFeatureTypes(); + RequestParameters requestParameters = processingData.getRequestParameters(); String startTimestamp = IsoDateTimeParser.parseIsoDateTime(requestParameters.getTime()[0]) .format(DateTimeFormatter.ISO_DATE_TIME); String endTimestamp = IsoDateTimeParser.parseIsoDateTime(requestParameters.getTime()[1]) @@ -109,13 +100,10 @@ public void extract() throws Exception { mapRedSnapshots = mapRedSnapshots.filter(filter.get()); mapRedContributions = mapRedContributions.filter(filter.get()); } - final boolean isContainingSimpleFeatureTypes = processingData.isContainingSimpleFeatureTypes(); DataExtractionTransformer dataExtractionTransformer = new DataExtractionTransformer( startTimestamp, endTimestamp, filter.orElse(null), isContributionsEndpoint, - isContributionsLatestEndpoint, - clipGeometries, includeTags, includeOSMMetadata, includeContributionTypes, utils, exeUtils, - keysInt, elementsGeometry, simpleFeatureTypes, - isContainingSimpleFeatureTypes); + isContributionsLatestEndpoint, clipGeometries, includeTags, includeOSMMetadata, + includeContributionTypes, exeUtils, elementsGeometry); MapReducer contributionPreResult = mapRedContributions .flatMap(dataExtractionTransformer::buildChangedFeatures) .filter(Objects::nonNull); diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java index 95df128d..4cabc387 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java @@ -10,7 +10,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.stream.Collectors; @@ -29,7 +28,6 @@ import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessingUtils; import org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor; import org.heigit.ohsome.ohsomeapi.inputprocessing.ProcessingData; -import org.heigit.ohsome.ohsomeapi.inputprocessing.SimpleFeatureType; import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData; import org.heigit.ohsome.ohsomeapi.oshdb.ExtractMetadata; import org.heigit.ohsome.ohsomeapi.output.Attribution; @@ -40,7 +38,6 @@ import org.heigit.ohsome.ohsomeapi.output.elements.ElementsResult; import org.heigit.ohsome.ohsomeapi.output.groupby.GroupByResponse; import org.heigit.ohsome.ohsomeapi.output.groupby.GroupByResult; -import org.heigit.ohsome.ohsomeapi.utils.FilterUtil; import org.heigit.ohsome.ohsomeapi.utils.GroupByBoundaryGeoJsonGenerator; import org.heigit.ohsome.ohsomeapi.utils.TimestampFormatter; import org.heigit.ohsome.oshdb.OSHDBTag; @@ -95,7 +92,7 @@ private ElementsRequestExecutor() { public static void extract(RequestResource requestResource, ElementsGeometry elemGeom, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws Exception { InputProcessor inputProcessor = new InputProcessor(servletRequest, true, false); - MapReducer mapRed = null; + MapReducer mapRed; inputProcessor.processPropertiesParam(); inputProcessor.processIsUnclippedParam(); final boolean includeTags = inputProcessor.includeTags(); @@ -110,10 +107,6 @@ public static void extract(RequestResource requestResource, ElementsGeometry ele mapRed = inputProcessor.processParameters(); } ProcessingData processingData = inputProcessor.getProcessingData(); - RequestParameters requestParameters = processingData.getRequestParameters(); - TagTranslator tt = DbConnData.tagTranslator; - String[] keys = requestParameters.getKeys(); - final Set keysInt = ExecutionUtils.keysToKeysInt(keys, tt); final MapReducer preResult; final ExecutionUtils exeUtils = new ExecutionUtils(processingData); preResult = mapRed.map(snapshot -> { @@ -128,7 +121,7 @@ public static void extract(RequestResource requestResource, ElementsGeometry ele if (!clipGeometries) { geom = snapshot.getGeometryUnclipped(); } - return exeUtils.createOSMFeature(snapshot.getEntity(), geom, properties, keysInt, includeTags, + return exeUtils.createOSMFeature(snapshot.getEntity(), geom, properties, includeTags, includeOSMMetadata, false, false, elemGeom, EnumSet.noneOf(ContributionType.class)); }).filter(Objects::nonNull); @@ -165,7 +158,6 @@ public static

Response aggregateGroupByBoundary RequestResource requestResource, HttpServletRequest servletRequest, HttpServletResponse servletResponse, boolean isSnapshot, boolean isDensity) throws Exception { final long startTime = System.currentTimeMillis(); - MapReducer mapRed = null; InputProcessor inputProcessor = new InputProcessor(servletRequest, isSnapshot, isDensity); inputProcessor.getProcessingData().setGroupByBoundary(true); String[] groupByKey = inputProcessor.splitParamOnComma( @@ -173,7 +165,7 @@ public static

Response aggregateGroupByBoundary if (groupByKey.length != 1) { throw new BadRequestException(ExceptionMessages.GROUP_BY_KEY_PARAM); } - mapRed = inputProcessor.processParameters(); + MapReducer mapRed = inputProcessor.processParameters(); ProcessingData processingData = inputProcessor.getProcessingData(); RequestParameters requestParameters = processingData.getRequestParameters(); String[] groupByValues = inputProcessor.splitParamOnComma( @@ -184,7 +176,8 @@ public static

Response aggregateGroupByBoundary int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1); if (groupByValues.length != 0) { for (int j = 0; j < groupByValues.length; j++) { - valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j); + valuesInt[j] = + tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j); zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j])); } } @@ -193,9 +186,6 @@ public static

Response aggregateGroupByBoundary Map geoms = IntStream.range(0, arrGeoms.size()).boxed() .collect(Collectors.toMap(idx -> idx, idx -> (P) arrGeoms.get(idx))); MapAggregator mapAgg = mapRed.aggregateByGeometry(geoms); - if (processingData.isContainingSimpleFeatureTypes()) { - mapAgg = inputProcessor.filterOnSimpleFeatures(mapAgg); - } Optional filter = processingData.getFilterExpression(); if (filter.isPresent()) { mapAgg = mapAgg.filter(filter.get()); @@ -273,14 +263,13 @@ public static Response aggregateGroupByTag(RequestResource requestResource, HttpServletRequest servletRequest, HttpServletResponse servletResponse, boolean isSnapshot, boolean isDensity) throws Exception { final long startTime = System.currentTimeMillis(); - MapReducer mapRed = null; InputProcessor inputProcessor = new InputProcessor(servletRequest, isSnapshot, isDensity); String[] groupByKey = inputProcessor.splitParamOnComma( inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("groupByKey"))); if (groupByKey.length != 1) { throw new BadRequestException(ExceptionMessages.GROUP_BY_KEY_PARAM); } - mapRed = inputProcessor.processParameters(); + MapReducer mapRed = inputProcessor.processParameters(); ProcessingData processingData = inputProcessor.getProcessingData(); RequestParameters requestParameters = processingData.getRequestParameters(); String[] groupByValues = inputProcessor.splitParamOnComma( @@ -291,7 +280,8 @@ public static Response aggregateGroupByTag(RequestResource requestResource, int keysInt = tt.getOSHDBTagKeyOf(groupByKey[0]).map(OSHDBTagKey::toInt).orElse(-1); if (groupByValues.length != 0) { for (int j = 0; j < groupByValues.length; j++) { - valuesInt[j] = tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j); + valuesInt[j] = + tt.getOSHDBTagOf(groupByKey[0], groupByValues[j]).map(OSHDBTag::getValue).orElse(-j); zeroFill.add(new ImmutablePair<>(keysInt, valuesInt[j])); } } @@ -357,15 +347,14 @@ public static Response aggregateGroupByType(RequestResource requestResource, HttpServletRequest servletRequest, HttpServletResponse servletResponse, boolean isSnapshot, boolean isDensity) throws Exception { final long startTime = System.currentTimeMillis(); - MapReducer mapRed = null; InputProcessor inputProcessor = new InputProcessor(servletRequest, isSnapshot, isDensity); - mapRed = inputProcessor.processParameters(); + MapReducer mapRed = inputProcessor.processParameters(); ProcessingData processingData = inputProcessor.getProcessingData(); RequestParameters requestParameters = processingData.getRequestParameters(); MapAggregator, OSMEntitySnapshot> preResult; preResult = mapRed.aggregateByTimestamp().aggregateBy( (SerializableFunction) f -> f.getEntity().getType(), - processingData.getOsmTypes()); + EnumSet.allOf(OSMType.class)); var result = ExecutionUtils.computeResult(requestResource, preResult); var groupByResult = ExecutionUtils.nest(result); GroupByResult[] resultSet = new GroupByResult[groupByResult.size()]; @@ -418,14 +407,13 @@ public static Response aggregateGroupByKey(RequestResource requestResource, HttpServletRequest servletRequest, HttpServletResponse servletResponse, boolean isSnapshot, boolean isDensity) throws Exception { final long startTime = System.currentTimeMillis(); - MapReducer mapRed = null; InputProcessor inputProcessor = new InputProcessor(servletRequest, isSnapshot, isDensity); String[] groupByKeys = inputProcessor.splitParamOnComma( inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("groupByKeys"))); if (groupByKeys == null || groupByKeys.length == 0) { throw new BadRequestException(ExceptionMessages.GROUP_BY_KEYS_PARAM); } - mapRed = inputProcessor.processParameters(); + MapReducer mapRed = inputProcessor.processParameters(); ProcessingData processingData = inputProcessor.getProcessingData(); RequestParameters requestParameters = processingData.getRequestParameters(); TagTranslator tt = DbConnData.tagTranslator; @@ -488,152 +476,6 @@ public static Response aggregateGroupByKey(RequestResource requestResource, } - - /** - * Performs a count|length|perimeter|area|ratio calculation. - * - * @param requestResource {@link org.heigit.ohsome.ohsomeapi.executor.RequestResource - * RequestResource} definition of the request resource - * @param servletRequest {@link javax.servlet.http.HttpServletRequest HttpServletRequest} incoming - * request object - * @param servletResponse {@link javax.servlet.http.HttpServletResponse HttpServletResponse]} - * outgoing response object - * @return {@link org.heigit.ohsome.ohsomeapi.output.Response Response} - * @throws Exception thrown by {@link org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor - * #processParameters() processParameters} and - * {@link org.heigit.ohsome.ohsomeapi.executor.ExecutionUtils - * #computeResult(RequestResource, MapAggregator) computeResult} - * @deprecated Will be removed in next major version update. - */ - @Deprecated(forRemoval = true) - public static Response aggregateBasicFiltersRatio(RequestResource requestResource, - HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws Exception { - final long startTime = System.currentTimeMillis(); - // these 2 parameters always have these values for /ratio requests - final boolean isSnapshot = true; - final boolean isDensity = false; - InputProcessor inputProcessor = new InputProcessor(servletRequest, isSnapshot, isDensity); - inputProcessor.getProcessingData().setRatio(true); - final MapReducer intermediateMapRed = inputProcessor.processParameters(); - final ProcessingData processingData = inputProcessor.getProcessingData(); - final RequestParameters requestParameters = processingData.getRequestParameters(); - TagTranslator tt = DbConnData.tagTranslator; - String[] keys2 = inputProcessor.splitParamOnComma( - inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("keys2"))); - String[] values2 = inputProcessor.splitParamOnComma( - inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("values2"))); - inputProcessor.checkKeysValues(keys2, values2); - Pair keys2Vals2 = inputProcessor.processKeys2Vals2(keys2, values2); - keys2 = keys2Vals2.getKey(); - values2 = keys2Vals2.getValue(); - Integer[] keysInt1 = new Integer[requestParameters.getKeys().length]; - Integer[] valuesInt1 = new Integer[requestParameters.getValues().length]; - Integer[] keysInt2 = new Integer[keys2.length]; - Integer[] valuesInt2 = new Integer[values2.length]; - for (int i = 0; i < requestParameters.getKeys().length; i++) { - keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).map(OSHDBTagKey::toInt) - .orElse(-i); - if (requestParameters.getValues() != null && i < requestParameters.getValues().length) { - valuesInt1[i] = - tt.getOSHDBTagOf(requestParameters.getKeys()[i], requestParameters.getValues()[i]) - .map(OSHDBTag::getValue).orElse(-i); - } - } - for (int i = 0; i < keys2.length; i++) { - keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).map(OSHDBTagKey::toInt).orElse(-i); - if (i < values2.length) { - valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).map(OSHDBTag::getValue).orElse(-i); - } - } - EnumSet osmTypes1 = - (EnumSet) inputProcessor.getProcessingData().getOsmTypes(); - String[] types1 = inputProcessor.splitParamOnComma( - inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("types"))); - String[] types2 = inputProcessor.splitParamOnComma( - inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("types2"))); - final EnumSet simpleFeatureTypes1 = - inputProcessor.defineSimpleFeatureTypes(types1); - inputProcessor.defineTypes(types2, intermediateMapRed); - EnumSet osmTypes2 = - (EnumSet) inputProcessor.getProcessingData().getOsmTypes(); - final EnumSet simpleFeatureTypes2 = - inputProcessor.defineSimpleFeatureTypes(types2); - EnumSet osmTypes = osmTypes1.clone(); - osmTypes.addAll(osmTypes2); - String[] osmTypesString = - osmTypes.stream().map(OSMType::toString).map(String::toLowerCase).toArray(String[]::new); - MapReducer mapRed = null; - if (!inputProcessor.compareKeysValues(requestParameters.getKeys(), keys2, - requestParameters.getValues(), values2)) { - RequestParameters requestParams = - new RequestParameters(servletRequest.getMethod(), isSnapshot, isDensity, - servletRequest.getParameter("bboxes"), servletRequest.getParameter("bcircles"), - servletRequest.getParameter("bpolys"), osmTypesString, new String[] {}, - new String[] {}, servletRequest.getParameterValues("time"), - servletRequest.getParameter("format"), servletRequest.getParameter("showMetadata"), - ProcessingData.getTimeout(), servletRequest.getParameter("filter")); - ProcessingData secondProcessingData = - new ProcessingData(requestParams, servletRequest.getRequestURL().toString()); - InputProcessor secondInputProcessor = - new InputProcessor(servletRequest, isSnapshot, isDensity); - secondInputProcessor.setProcessingData(secondProcessingData); - mapRed = secondInputProcessor.processParameters(); - } else { - mapRed = inputProcessor.processParameters(); - } - mapRed = mapRed.filter(FilterUtil.filter(osmTypes)); - mapRed = ExecutionUtils.snapshotFilter(mapRed, osmTypes1, osmTypes2, simpleFeatureTypes1, - simpleFeatureTypes2, keysInt1, keysInt2, valuesInt1, valuesInt2); - var preResult = mapRed.aggregateByTimestamp().aggregateBy(snapshot -> { - boolean matches1 = ExecutionUtils.snapshotMatches(snapshot, osmTypes1, - simpleFeatureTypes1, keysInt1, valuesInt1); - boolean matches2 = ExecutionUtils.snapshotMatches(snapshot, osmTypes2, - simpleFeatureTypes2, keysInt2, valuesInt2); - if (matches1 && matches2) { - return MatchType.MATCHESBOTH; - } else if (matches1) { - return MatchType.MATCHES1; - } else if (matches2) { - return MatchType.MATCHES2; - } else { - // this should never be reached - assert false : "MatchType matches none."; - return MatchType.MATCHESNONE; - } - }, EnumSet.allOf(MatchType.class)); - var result = ExecutionUtils.computeResult(requestResource, preResult); - int resultSize = result.size(); - Double[] value1 = new Double[resultSize / 4]; - Double[] value2 = new Double[resultSize / 4]; - String[] timeArray = new String[resultSize / 4]; - int value1Count = 0; - int value2Count = 0; - int matchesBothCount = 0; - // time and value extraction - for (var entry : result.entrySet()) { - if (entry.getKey().getSecondIndex() == MatchType.MATCHES2) { - timeArray[value2Count] = - TimestampFormatter.getInstance().isoDateTime(entry.getKey().getFirstIndex()); - value2[value2Count] = Double.parseDouble(df.format(entry.getValue().doubleValue())); - value2Count++; - } - if (entry.getKey().getSecondIndex() == MatchType.MATCHES1) { - value1[value1Count] = Double.parseDouble(df.format(entry.getValue().doubleValue())); - value1Count++; - } - if (entry.getKey().getSecondIndex() == MatchType.MATCHESBOTH) { - value1[matchesBothCount] = value1[matchesBothCount] - + Double.parseDouble(df.format(entry.getValue().doubleValue())); - value2[matchesBothCount] = value2[matchesBothCount] - + Double.parseDouble(df.format(entry.getValue().doubleValue())); - matchesBothCount++; - } - } - ExecutionUtils exeUtils = new ExecutionUtils(processingData); - return exeUtils.createRatioResponse(timeArray, value1, value2, startTime, requestResource, - inputProcessor.getRequestUrlIfGetRequest(servletRequest), servletResponse); - } - /** * Performs a count|length|perimeter|area|ratio calculation using the filter and filter2 * parameters. @@ -652,11 +494,6 @@ public static Response aggregateBasicFiltersRatio(RequestResource requestResourc */ public static Response aggregateRatio(RequestResource requestResource, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws Exception { - if (null == servletRequest.getParameter("filter") - && (null != servletRequest.getParameter("types") - || null != servletRequest.getParameter("keys"))) { - return aggregateBasicFiltersRatio(requestResource, servletRequest, servletResponse); - } final long startTime = System.currentTimeMillis(); // these 2 parameters always have these values for /ratio requests final boolean isSnapshot = true; @@ -675,7 +512,6 @@ public static Response aggregateRatio(RequestResource requestResource, RequestParameters requestParamsCombined = new RequestParameters(servletRequest.getMethod(), isSnapshot, isDensity, servletRequest.getParameter("bboxes"), servletRequest.getParameter("bcircles"), servletRequest.getParameter("bpolys"), - new String[] {}, new String[] {}, new String[] {}, servletRequest.getParameterValues("time"), servletRequest.getParameter("format"), servletRequest.getParameter("showMetadata"), ProcessingData.getTimeout(), combinedFilter); ProcessingData processingDataCombined = @@ -735,200 +571,6 @@ public static Response aggregateRatio(RequestResource requestResource, inputProcessor.getRequestUrlIfGetRequest(servletRequest), servletResponse); } - /** - * Performs a count|length|perimeter|area-ratio calculation grouped by the boundary. - * - * @param requestResource {@link org.heigit.ohsome.ohsomeapi.executor.RequestResource - * RequestResource} definition of the request resource - * @param servletRequest {@link javax.servlet.http.HttpServletRequest HttpServletRequest} incoming - * request object - * @param servletResponse {@link javax.servlet.http.HttpServletResponse HttpServletResponse]} - * outgoing response object - * @return {@link org.heigit.ohsome.ohsomeapi.output.Response Response} - * @throws BadRequestException if a boundary parameter (bboxes, bcircles, bpolys) is not defined - * @throws Exception thrown by {@link org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor - * #processParameters() processParameters}, - * {@link org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator#count() count}, or - * {@link org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator#sum() sum} - * @deprecated Will be removed in next major version update. - */ - @Deprecated(forRemoval = true) - public static

Response aggregateBasicFiltersRatioGroupByBoundary( - RequestResource requestResource, HttpServletRequest servletRequest, - HttpServletResponse servletResponse) throws Exception { - final long startTime = System.currentTimeMillis(); - final boolean isSnapshot = true; - final boolean isDensity = false; - InputProcessor inputProcessor = new InputProcessor(servletRequest, isSnapshot, isDensity); - inputProcessor.getProcessingData().setGroupByBoundary(true); - inputProcessor.getProcessingData().setRatio(true); - final MapReducer intermediateMapRed = inputProcessor.processParameters(); - final ProcessingData processingData = inputProcessor.getProcessingData(); - final RequestParameters requestParameters = processingData.getRequestParameters(); - if (processingData.getBoundaryType() == BoundaryType.NOBOUNDARY) { - throw new BadRequestException(ExceptionMessages.NO_BOUNDARY); - } - String[] keys2 = inputProcessor.splitParamOnComma( - inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("keys2"))); - String[] values2 = inputProcessor.splitParamOnComma( - inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("values2"))); - inputProcessor.checkKeysValues(keys2, values2); - Pair keys2Vals2 = inputProcessor.processKeys2Vals2(keys2, values2); - keys2 = keys2Vals2.getKey(); - values2 = keys2Vals2.getValue(); - Integer[] keysInt1 = new Integer[requestParameters.getKeys().length]; - Integer[] valuesInt1 = new Integer[requestParameters.getValues().length]; - Integer[] keysInt2 = new Integer[keys2.length]; - Integer[] valuesInt2 = new Integer[values2.length]; - TagTranslator tt = DbConnData.tagTranslator; - for (int i = 0; i < requestParameters.getKeys().length; i++) { - keysInt1[i] = tt.getOSHDBTagKeyOf(requestParameters.getKeys()[i]).map(OSHDBTagKey::toInt) - .orElse(-i); - if (requestParameters.getValues() != null && i < requestParameters.getValues().length) { - valuesInt1[i] = - tt.getOSHDBTagOf(requestParameters.getKeys()[i], requestParameters.getValues()[i]) - .map(OSHDBTag::getValue).orElse(-i); - } - } - for (int i = 0; i < keys2.length; i++) { - keysInt2[i] = tt.getOSHDBTagKeyOf(keys2[i]).map(OSHDBTagKey::toInt).orElse(-i); - if (i < values2.length) { - valuesInt2[i] = tt.getOSHDBTagOf(keys2[i], values2[i]).map(OSHDBTag::getValue).orElse(-i); - } - } - EnumSet osmTypes1 = (EnumSet) processingData.getOsmTypes(); - String[] types1 = inputProcessor.splitParamOnComma( - inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("types"))); - String[] types2 = inputProcessor.splitParamOnComma( - inputProcessor.createEmptyArrayIfNull(servletRequest.getParameterValues("types2"))); - final EnumSet simpleFeatureTypes1 = - inputProcessor.defineSimpleFeatureTypes(types1); - inputProcessor.defineTypes(types2, intermediateMapRed); - EnumSet osmTypes2 = - (EnumSet) inputProcessor.getProcessingData().getOsmTypes(); - EnumSet osmTypes = osmTypes1.clone(); - final EnumSet simpleFeatureTypes2 = - inputProcessor.defineSimpleFeatureTypes(types2); - osmTypes.addAll(osmTypes2); - String[] osmTypesString = - osmTypes.stream().map(OSMType::toString).map(String::toLowerCase).toArray(String[]::new); - MapReducer mapRed = null; - if (!inputProcessor.compareKeysValues(requestParameters.getKeys(), keys2, - requestParameters.getValues(), values2)) { - RequestParameters requestParams = - new RequestParameters(servletRequest.getMethod(), isSnapshot, isDensity, - servletRequest.getParameter("bboxes"), servletRequest.getParameter("bcircles"), - servletRequest.getParameter("bpolys"), osmTypesString, new String[] {}, - new String[] {}, servletRequest.getParameterValues("time"), - servletRequest.getParameter("format"), servletRequest.getParameter("showMetadata"), - ProcessingData.getTimeout(), servletRequest.getParameter("filter")); - ProcessingData secondProcessingData = - new ProcessingData(requestParams, servletRequest.getRequestURL().toString()); - InputProcessor secondInputProcessor = - new InputProcessor(servletRequest, isSnapshot, isDensity); - secondInputProcessor.setProcessingData(secondProcessingData); - mapRed = secondInputProcessor.processParameters(); - } else { - mapRed = inputProcessor.processParameters(); - } - mapRed = mapRed.filter(FilterUtil.filter(osmTypes)); - ArrayList arrGeoms = new ArrayList<>(processingData.getBoundaryList()); - // intentionally as check for P on Polygonal is already performed - @SuppressWarnings({"unchecked"}) - Map geoms = - arrGeoms.stream().collect(Collectors.toMap(arrGeoms::indexOf, geom -> (P) geom)); - var mapRed2 = mapRed.aggregateByTimestamp().aggregateByGeometry(geoms); - mapRed2 = ExecutionUtils.snapshotFilter(mapRed2, osmTypes1, osmTypes2, simpleFeatureTypes1, - simpleFeatureTypes2, keysInt1, keysInt2, valuesInt1, valuesInt2); - var preResult = - mapRed2.aggregateBy((SerializableFunction) snapshot -> { - boolean matches1 = ExecutionUtils.snapshotMatches(snapshot, - osmTypes1, simpleFeatureTypes1, keysInt1, valuesInt1); - boolean matches2 = ExecutionUtils.snapshotMatches(snapshot, - osmTypes2, simpleFeatureTypes2, keysInt2, valuesInt2); - if (matches1 && matches2) { - return MatchType.MATCHESBOTH; - } else if (matches1) { - return MatchType.MATCHES1; - } else if (matches2) { - return MatchType.MATCHES2; - } else { - assert false : "MatchType matches none."; - } - return MatchType.MATCHESNONE; - }, EnumSet.allOf(MatchType.class)).map(OSMEntitySnapshot::getGeometry); - SortedMap, MatchType>, ? extends - Number> result = null; - switch (requestResource) { - case COUNT: - result = preResult.count(); - break; - case LENGTH: - result = - preResult.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.lengthOf(geom))); - break; - case PERIMETER: - result = preResult.sum(geom -> { - if (!(geom instanceof Polygonal)) { - return 0.0; - } - return ExecutionUtils.cacheInUserData(geom, () -> Geo.lengthOf(geom.getBoundary())); - }); - break; - case AREA: - result = - preResult.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.areaOf(geom))); - break; - default: - break; - } - InputProcessingUtils utils = inputProcessor.getUtils(); - var groupByResult = ExecutionUtils.nest(result); - Object[] boundaryIds = utils.getBoundaryIds(); - Double[] resultValues1 = null; - Double[] resultValues2 = null; - String[] timeArray = null; - boolean timeArrayFilled = false; - for (var entry : groupByResult.entrySet()) { - var resultSet = entry.getValue().entrySet(); - if (!timeArrayFilled) { - timeArray = new String[resultSet.size()]; - } - if (entry.getKey() == MatchType.MATCHES2) { - resultValues2 = ExecutionUtils.fillElementsRatioGroupByBoundaryResultValues(resultSet, df); - } else if (entry.getKey() == MatchType.MATCHES1) { - resultValues1 = ExecutionUtils.fillElementsRatioGroupByBoundaryResultValues(resultSet, df); - } else if (entry.getKey() == MatchType.MATCHESBOTH) { - int matchesBothCount = 0; - int timeArrayCount = 0; - for (var innerEntry : resultSet) { - assert resultValues1 != null; - assert resultValues2 != null; - resultValues1[matchesBothCount] = resultValues1[matchesBothCount] - + Double.parseDouble(df.format(innerEntry.getValue().doubleValue())); - resultValues2[matchesBothCount] = resultValues2[matchesBothCount] - + Double.parseDouble(df.format(innerEntry.getValue().doubleValue())); - if (!timeArrayFilled) { - String time = innerEntry.getKey().getFirstIndex().toString(); - if (matchesBothCount == 0 || !timeArray[timeArrayCount - 1].equals(time)) { - timeArray[timeArrayCount] = innerEntry.getKey().getFirstIndex().toString(); - timeArrayCount++; - } - } - matchesBothCount++; - } - timeArray = Arrays.stream(timeArray).filter(Objects::nonNull).toArray(String[]::new); - timeArrayFilled = true; - } else { - // on MatchType.MATCHESNONE aggregated values are not needed / do not exist - } - } - ExecutionUtils exeUtils = new ExecutionUtils(processingData); - return exeUtils.createRatioGroupByBoundaryResponse(boundaryIds, timeArray, resultValues1, - resultValues2, startTime, requestResource, - inputProcessor.getRequestUrlIfGetRequest(servletRequest), servletResponse); - } - /** * Performs a count|length|perimeter|area-ratio calculation grouped by the boundary using the * filter and filter2 parameters. @@ -949,12 +591,6 @@ public static

Response aggregateBasicFiltersRat public static

Response aggregateRatioGroupByBoundary( RequestResource requestResource, HttpServletRequest servletRequest, HttpServletResponse servletResponse) throws Exception { - if (null == servletRequest.getParameter("filter") - && (null != servletRequest.getParameter("types") - || null != servletRequest.getParameter("keys"))) { - return aggregateBasicFiltersRatioGroupByBoundary(requestResource, servletRequest, - servletResponse); - } final long startTime = System.currentTimeMillis(); // these 2 parameters always have these values for /ratio requests final boolean isSnapshot = true; @@ -978,7 +614,6 @@ public static

Response aggregateRatioGroupByBou RequestParameters requestParamsCombined = new RequestParameters(servletRequest.getMethod(), isSnapshot, isDensity, servletRequest.getParameter("bboxes"), servletRequest.getParameter("bcircles"), servletRequest.getParameter("bpolys"), - new String[] {}, new String[] {}, new String[] {}, servletRequest.getParameterValues("time"), servletRequest.getParameter("format"), servletRequest.getParameter("showMetadata"), ProcessingData.getTimeout(), combinedFilter); ProcessingData processingDataCombined = @@ -1075,9 +710,7 @@ public static

Response aggregateRatioGroupByBou } timeArray = Arrays.stream(timeArray).filter(Objects::nonNull).toArray(String[]::new); timeArrayFilled = true; - } else { - // on MatchType.MATCHESNONE aggregated values are not needed / do not exist - } + } // else: on MatchType.MATCHESNONE aggregated values are not needed / do not exist } ExecutionUtils exeUtils = new ExecutionUtils(processingData); return exeUtils.createRatioGroupByBoundaryResponse(boundaryIds, timeArray, resultValues1, diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java index 471b4b02..df8bf4ab 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java @@ -8,7 +8,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.collect.Iterables; -import com.google.common.collect.Streams; import com.opencsv.CSVWriter; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -71,7 +70,6 @@ import org.heigit.ohsome.oshdb.OSHDBTimestamp; import org.heigit.ohsome.oshdb.api.generic.OSHDBCombinedIndex; import org.heigit.ohsome.oshdb.api.mapreducer.MapAggregator; -import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; import org.heigit.ohsome.oshdb.api.mapreducer.Mappable; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMType; @@ -101,16 +99,6 @@ public class ExecutionUtils implements Serializable { private final DecimalFormat ratioDf = defineDecimalFormat("#.######"); private final GeometryPrecisionReducer gpr = createGeometryPrecisionReducer(); - /** Applies a filter on the given MapReducer object using the given parameters. */ - public static MapReducer snapshotFilter(MapReducer mapRed, - Set osmTypes1, Set osmTypes2, Set simpleFeatureTypes1, - Set simpleFeatureTypes2, Integer[] keysInt1, Integer[] keysInt2, - Integer[] valuesInt1, Integer[] valuesInt2) { - return mapRed.filter( - snapshot -> snapshotMatches(snapshot, osmTypes1, simpleFeatureTypes1, keysInt1, valuesInt1) - || snapshotMatches(snapshot, osmTypes2, simpleFeatureTypes2, keysInt2, valuesInt2)); - } - /** * Applies a filter on the given MapReducer object using the given parameters. Used in * /ratio/groupBy/boundary requests. @@ -386,8 +374,8 @@ public static List createCsvTopComments(String url, String text, Strin /** Creates the Feature objects in the OSM data response. */ public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry geometry, - Map properties, Set keysInt, boolean includeTags, - boolean includeOSMMetadata, boolean includeContributionTypes, boolean isContributionsEndpoint, + Map properties, boolean includeTags, boolean includeOSMMetadata, + boolean includeContributionTypes, boolean isContributionsEndpoint, ElementsGeometry elemGeom, EnumSet contributionTypes) { if (geometry.isEmpty() && !contributionTypes.contains(ContributionType.DELETION)) { // skip invalid geometries (e.g. ways with 0 nodes) @@ -395,9 +383,6 @@ public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry ge } if (includeTags) { properties.put("@tags", Iterables.toArray(entity.getTags(), OSHDBTag.class)); - } else if (!keysInt.isEmpty()) { - properties.put("@tags", Streams.stream(entity.getTags()) - .filter(t -> keysInt.contains(t.getKey())).toArray(OSHDBTag[]::new)); } if (includeOSMMetadata) { properties.put("@version", entity.getVersion()); diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/RequestParameters.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/RequestParameters.java index 14f09769..910062c8 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/RequestParameters.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/RequestParameters.java @@ -17,9 +17,6 @@ public class RequestParameters implements Serializable { private String bboxes; private String bcircles; private String bpolys; - private String[] types; - private String[] keys; - private String[] values; private String[] time; private String format; private String showMetadata; diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java index 6e995c8c..50ef2128 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/UsersRequestExecutor.java @@ -3,6 +3,7 @@ import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; @@ -56,10 +57,9 @@ private UsersRequestExecutor() { public static Response countGroupByType(HttpServletRequest servletRequest, HttpServletResponse servletResponse, boolean isDensity) throws Exception { long startTime = System.currentTimeMillis(); - SortedMap, Integer> result = null; - MapReducer mapRed = null; + SortedMap, Integer> result; InputProcessor inputProcessor = new InputProcessor(servletRequest, false, isDensity); - mapRed = inputProcessor.processParameters(); + MapReducer mapRed = inputProcessor.processParameters(); ProcessingData processingData = inputProcessor.getProcessingData(); RequestParameters requestParameters = processingData.getRequestParameters(); result = mapRed.filter(ExecutionUtils.contributionsFilter(servletRequest.getParameter( @@ -67,7 +67,7 @@ public static Response countGroupByType(HttpServletRequest servletRequest, .aggregateByTimestamp() .aggregateBy( (SerializableFunction) f -> f.getEntityAfter().getType(), - processingData.getOsmTypes()) + EnumSet.allOf(OSMType.class)) .map(OSMContribution::getContributorUserId) .countUniq(); SortedMap> groupByResult; @@ -111,8 +111,7 @@ public static Response countGroupByTag(HttpServletRequest servletRequest, if (groupByKey.length != 1) { throw new BadRequestException(ExceptionMessages.GROUP_BY_KEY_PARAM); } - MapReducer mapRed = null; - mapRed = inputProcessor.processParameters(); + MapReducer mapRed = inputProcessor.processParameters(); ProcessingData processingData = inputProcessor.getProcessingData(); RequestParameters requestParameters = processingData.getRequestParameters(); String[] groupByValues = inputProcessor.splitParamOnComma(inputProcessor.createEmptyArrayIfNull( @@ -209,8 +208,7 @@ public static Response countGroupByKey(HttpServletRequest servletRequest, if (groupByKeys == null || groupByKeys.length == 0) { throw new BadRequestException(ExceptionMessages.GROUP_BY_KEYS_PARAM); } - MapReducer mapRed = null; - mapRed = inputProcessor.processParameters(); + MapReducer mapRed = inputProcessor.processParameters(); ProcessingData processingData = inputProcessor.getProcessingData(); RequestParameters requestParameters = processingData.getRequestParameters(); TagTranslator tt = DbConnData.tagTranslator; @@ -248,7 +246,7 @@ public static Response countGroupByKey(HttpServletRequest servletRequest, SortedMap> groupByResult; groupByResult = ExecutionUtils.nest(result); GroupByResult[] resultSet = new GroupByResult[groupByResult.size()]; - String groupByName = ""; + String groupByName; int count = 0; for (Entry> entry : groupByResult.entrySet()) { ContributionsResult[] results = ExecutionUtils.fillContributionsResult(entry.getValue(), diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java index 9a9734dc..4166d61d 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/InputProcessor.java @@ -4,18 +4,11 @@ import java.io.IOException; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; -import java.util.EnumSet; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; import org.geojson.GeoJsonObject; import org.heigit.ohsome.ohsomeapi.exception.BadRequestException; import org.heigit.ohsome.ohsomeapi.exception.ExceptionMessages; @@ -23,21 +16,16 @@ import org.heigit.ohsome.ohsomeapi.executor.RequestParameters; import org.heigit.ohsome.ohsomeapi.oshdb.DbConnData; import org.heigit.ohsome.ohsomeapi.oshdb.ExtractMetadata; -import org.heigit.ohsome.ohsomeapi.utils.FilterUtil; import org.heigit.ohsome.ohsomeapi.utils.RequestUtils; import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite; import org.heigit.ohsome.oshdb.api.db.OSHDBIgnite.ComputeMode; import org.heigit.ohsome.oshdb.api.mapreducer.MapReducer; -import org.heigit.ohsome.oshdb.api.mapreducer.Mappable; import org.heigit.ohsome.oshdb.api.mapreducer.OSMContributionView; import org.heigit.ohsome.oshdb.api.mapreducer.OSMEntitySnapshotView; import org.heigit.ohsome.oshdb.filter.FilterExpression; import org.heigit.ohsome.oshdb.filter.FilterParser; -import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.geometry.OSHDBGeometryBuilder; import org.heigit.ohsome.oshdb.util.mappable.OSHDBMapReducible; -import org.heigit.ohsome.oshdb.util.mappable.OSMContribution; -import org.heigit.ohsome.oshdb.util.mappable.OSMEntitySnapshot; import org.heigit.ohsome.oshdb.util.time.IsoDateTimeParser; import org.heigit.ohsome.oshdb.util.time.OSHDBTimestamps; import org.locationtech.jts.geom.Geometry; @@ -88,8 +76,7 @@ public InputProcessor(HttpServletRequest servletRequest, boolean isSnapshot, boo processingData = new ProcessingData(new RequestParameters(servletRequest.getMethod(), isSnapshot, isDensity, servletRequest.getParameter("bboxes"), servletRequest.getParameter("bcircles"), - servletRequest.getParameter("bpolys"), servletRequest.getParameterValues("types"), - servletRequest.getParameterValues("keys"), servletRequest.getParameterValues("values"), + servletRequest.getParameter("bpolys"), servletRequest.getParameterValues("time"), servletRequest.getParameter("format"), servletRequest.getParameter("showMetadata"), ProcessingData.getTimeout(), servletRequest.getParameter("filter")), servletRequest.getRequestURL().toString()); @@ -125,12 +112,6 @@ public MapReducer processParameters(ComputeMode String bboxes = createEmptyStringIfNull(processingData.getRequestParameters().getBboxes()); String bcircles = createEmptyStringIfNull(processingData.getRequestParameters().getBcircles()); String bpolys = createEmptyStringIfNull(processingData.getRequestParameters().getBpolys()); - String[] types = - splitParamOnComma(createEmptyArrayIfNull(processingData.getRequestParameters().getTypes())); - String[] keys = - splitParamOnComma(createEmptyArrayIfNull(processingData.getRequestParameters().getKeys())); - String[] values = splitParamOnComma( - createEmptyArrayIfNull(processingData.getRequestParameters().getValues())); String[] time = splitParamOnComma(createEmptyArrayIfNull(processingData.getRequestParameters().getTime())); String format = createEmptyStringIfNull(processingData.getRequestParameters().getFormat()); @@ -141,9 +122,9 @@ public MapReducer processParameters(ComputeMode // overwriting RequestParameters object with splitted/non-null parameters processingData .setRequestParameters(new RequestParameters(requestMethod, isSnapshot, isDensity, bboxes, - bcircles, bpolys, types, keys, values, time, format, showMetadata, timeout, filter)); + bcircles, bpolys, time, format, showMetadata, timeout, filter)); processingData.setFormat(format); - MapReducer mapRed = null; + MapReducer mapRed; processingData.setBoundaryType(setBoundaryType(bboxes, bcircles, bpolys)); geomBuilder = new GeometryBuilder(processingData); utils = new InputProcessingUtils(); @@ -226,95 +207,19 @@ public MapReducer processParameters(ComputeMode } processingData.setGeoJsonGeoms(geoJsonGeoms); } - mapRed = defineTypes(types, mapRed); - // the OSM type will be set in the ratio implementation within the ElementsRequestExecutor.java - if (!processingData.isRatio()) { - mapRed = mapRed.filter(FilterUtil.filter(processingData.getOsmTypes())); - } - if (processingData.isContainingSimpleFeatureTypes() - // skip in ratio or groupByBoundary requests -> needs to be done later in the processing - && !processingData.isRatio() && !processingData.isGroupByBoundary() - && !processingData.isFullHistory()) { - mapRed = filterOnSimpleFeatures(mapRed); - } mapRed = extractTime(mapRed, time, isSnapshot); - if (!"".equals(filter)) { - if (keys.length != 0 || values.length != 0 || types.length != 0) { - throw new BadRequestException(ExceptionMessages.FILTER_PARAM); - } else { - // call translator and add filters to mapRed - FilterParser fp = new FilterParser(DbConnData.tagTranslator); - FilterExpression filterExpr = utils.parseFilter(fp, filter); - processingData.setFilterExpression(filterExpr); - if (!(processingData.isRatio() - || processingData.isGroupByBoundary() - || processingData.isFullHistory())) { - // skip in ratio or groupByBoundary requests -> needs to be done later in the processing - mapRed = mapRed.filter(filterExpr); - } - } - } else { - mapRed = extractKeysValues(mapRed, keys, values); + // call translator and add filters to mapRed + FilterParser fp = new FilterParser(DbConnData.tagTranslator); + FilterExpression filterExpr = utils.parseFilter(fp, filter); + processingData.setFilterExpression(filterExpr); + if (!(processingData.isRatio() + || processingData.isGroupByBoundary() + || processingData.isFullHistory())) { + // skip in ratio or groupByBoundary requests -> needs to be done later in the processing + mapRed = mapRed.filter(filterExpr); } - return (MapReducer) mapRed; - } - /** - * Defines the type(s) out of the given String[]. - * - * @param types String array containing one, two, or all 3 OSM types (node, way, - * relation), or simple feature types (point, line, polygon, other). If the array is empty, - * all three OSM types are used. - */ - public MapReducer defineTypes(String[] types, - MapReducer mapRed) { - types = createEmptyArrayIfNull(types); - checkTypes(types); - processingData.setOsmTypes(EnumSet.noneOf(OSMType.class)); - if (types.length == 0 || types.length == 1 && types[0].isEmpty()) { - processingData.setOsmTypes(EnumSet.of(OSMType.NODE, OSMType.WAY, OSMType.RELATION)); - } else { - if (!processingData.isContainingSimpleFeatureTypes()) { - for (String type : types) { - if ("node".equalsIgnoreCase(type)) { - processingData.getOsmTypes().add(OSMType.NODE); - } else if ("way".equalsIgnoreCase(type)) { - processingData.getOsmTypes().add(OSMType.WAY); - } else { - processingData.getOsmTypes().add(OSMType.RELATION); - } - } - } else { - processingData.setSimpleFeatureTypes(defineSimpleFeatureTypes(types)); - if (!processingData.isRatio() && (processingData.getOsmTypes().contains(OSMType.WAY) - || processingData.getOsmTypes().contains(OSMType.RELATION))) { - mapRed = utils.filterOnPlanarRelations(mapRed); - } - } - } - return mapRed; - } - - /** Defines the simple feature types and corresponding OSM types out of the given String array. */ - public EnumSet defineSimpleFeatureTypes(String[] types) { - EnumSet simpleFeatures = EnumSet.noneOf(SimpleFeatureType.class); - for (String type : types) { - if ("point".equalsIgnoreCase(type)) { - simpleFeatures.add(SimpleFeatureType.POINT); - processingData.getOsmTypes().add(OSMType.NODE); - } else if ("line".equalsIgnoreCase(type)) { - simpleFeatures.add(SimpleFeatureType.LINE); - processingData.getOsmTypes().add(OSMType.WAY); - } else if ("polygon".equalsIgnoreCase(type)) { - simpleFeatures.add(SimpleFeatureType.POLYGON); - processingData.getOsmTypes().add(OSMType.WAY); - processingData.getOsmTypes().add(OSMType.RELATION); - } else if ("other".equalsIgnoreCase(type)) { - simpleFeatures.add(SimpleFeatureType.OTHER); - processingData.getOsmTypes().add(OSMType.RELATION); - } - } - return simpleFeatures; + return (MapReducer) mapRed; } /** @@ -357,34 +262,6 @@ public String createEmptyStringIfNull(String toCheck) { return toCheck; } - /** - * Checks the given keys and values String[] on their length. - * - * @throws BadRequestException if values_n doesn't fit to keys_n. There cannot be more input - * values in the values|values2 than in the keys|keys2 parameter. - */ - public void checkKeysValues(String[] keys, String[] values) { - if (values != null && keys.length < values.length) { - throw new BadRequestException(ExceptionMessages.KEYS_VALUES_RATIO_INVALID); - } - } - - /** - * Compares the keys and values arrays with each other. Returns true only if keys=keys2 and - * values=values2. - */ - public boolean compareKeysValues(String[] keys, String[] keys2, String[] values, - String[] values2) { - return Arrays.equals(keys, keys2) && Arrays.equals(values, values2); - } - - /** Used in /ratio requests. */ - public Pair processKeys2Vals2(String[] keys2, String[] values2) { - keys2 = createEmptyArrayIfNull(keys2); - values2 = createEmptyArrayIfNull(values2); - return new ImmutablePair<>(keys2, values2); - } - /** * Processes the properties parameter used in data-extraction resources and sets the respective * boolean values includeTags, includeOSMMetadata, unclippedGeometries, and @@ -396,14 +273,10 @@ public void processPropertiesParam() { String[] properties = splitParamOnComma(createEmptyArrayIfNull(requestParameters.get("properties"))); for (String property : properties) { - @Deprecated - boolean oldUnclippedParameter = "unclipped".equalsIgnoreCase(property); if ("tags".equalsIgnoreCase(property)) { this.includeTags = true; } else if ("metadata".equalsIgnoreCase(property)) { this.includeOSMMetadata = true; - } else if (oldUnclippedParameter) { - this.clipGeometry = false; } else if (RequestUtils.isContributionsExtraction(requestUrl)) { if ("contributionTypes".equalsIgnoreCase(property)) { this.includeContributionTypes = true; @@ -436,47 +309,6 @@ public String getRequestUrlIfGetRequest(HttpServletRequest servletRequest) { return null; } - /** - * Applies respective Puntal|Lineal|Polygonal filter(s) on features of the given MapReducer. - * - * @return MapReducer with filtered geometries - * @throws RuntimeException if {@link org.heigit.ohsome.ohsomeapi.inputprocessing.InputProcessor - * #filterOnSimpleFeatures(Mappable) filterOnSimpleFeatures} was called on mapped entries - */ - // suppressed, as filter always returns the same mappable type T - @SuppressWarnings("unchecked") - public > T filterOnSimpleFeatures(T mapRed) { - Set simpleFeatureTypes = processingData.getSimpleFeatureTypes(); - return (T) mapRed.filter(data -> { - if (data instanceof OSMEntitySnapshot) { - Geometry snapshotGeom; - if (clipGeometry) { - snapshotGeom = ((OSMEntitySnapshot) data).getGeometry(); - } else { - snapshotGeom = ((OSMEntitySnapshot) data).getGeometryUnclipped(); - } - return utils.checkGeometryOnSimpleFeatures(snapshotGeom, simpleFeatureTypes); - } else if (data instanceof OSMContribution) { - Geometry contribGeomBefore; - Geometry contribGeomAfter; - if (clipGeometry) { - contribGeomBefore = ((OSMContribution) data).getGeometryBefore(); - contribGeomAfter = ((OSMContribution) data).getGeometryAfter(); - } else { - contribGeomBefore = ((OSMContribution) data).getGeometryUnclippedBefore(); - contribGeomAfter = ((OSMContribution) data).getGeometryUnclippedAfter(); - } - return contribGeomBefore != null - && utils.checkGeometryOnSimpleFeatures(contribGeomBefore, simpleFeatureTypes) - || contribGeomAfter != null - && utils.checkGeometryOnSimpleFeatures(contribGeomAfter, simpleFeatureTypes); - } else { - assert false : "filterOnSimpleFeatures() called on mapped entries"; - throw new RuntimeException("filterOnSimpleFeatures() called on mapped entries"); - } - }); - } - /** * Checks the given filter parameter if it's null or blank. Currently used for filter2 parameter * of /ratio processing. @@ -491,42 +323,6 @@ public void checkFilter(String filter) { } } - /** - * Checks the given keys and values parameters on their length and includes them in the - * {@link org.heigit.ohsome.oshdb.api.mapreducer.MapReducer#osmTag(String) osmTag(key)}, - * or {@link org.heigit.ohsome.oshdb.api.mapreducer.MapReducer#osmTag(String, String) - * osmTag(key, value)} method. - * - * @param mapRed current {@link org.heigit.ohsome.oshdb.api.mapreducer.MapReducer MapReducer} - * @return {@link org.heigit.ohsome.oshdb.api.mapreducer.MapReducer MapReducer} object - * including the filters derived from the given parameters. - */ - private MapReducer extractKeysValues( - MapReducer mapRed, String[] keys, String[] values) { - checkKeysValues(keys, values); - if (keys.length != values.length) { - String[] tempVal = new String[keys.length]; - System.arraycopy(values, 0, tempVal, 0, values.length); - for (int i = values.length; i < keys.length; i++) { - tempVal[i] = ""; - } - values = tempVal; - } - // prerequisites: both arrays (keys and values) must be of the same length - // and key-value pairs need to be at the same index in both arrays - var filters = new ArrayList(); - - for (int i = 0; i < keys.length; i++) { - if ("".equals(values[i])) { - filters.add(keys[i] + "=*"); - } else { - filters.add(keys[i] + "=" + values[i]); - } - } - var filter = filters.stream().collect(Collectors.joining(" and ")); - return mapRed.filter(filter); - } - /** * Extracts the information from the given time array and fills the toTimestamps[] with content * (in case of isSnapshot=false). @@ -576,34 +372,6 @@ private MapReducer extractTime( return mapRed; } - /** - * Checks the given type(s) String[] on its length and content. - * - * @throws BadRequestException if the given types parameter is invalid. - */ - private void checkTypes(String[] types) { - if (types.length > 4) { - throw new BadRequestException( - "Parameter 'types' (and 'types2') cannot have more than 4 entries."); - } else if (types.length == 0 || types.length == 1 && types[0].isEmpty()) { - // do nothing - } else { - processingData.setContainingSimpleFeatureTypes(!"node".equalsIgnoreCase(types[0]) - && !"way".equalsIgnoreCase(types[0]) && !"relation".equalsIgnoreCase(types[0])); - for (String type : types) { - if (utils.isSimpleFeatureType(type)) { - if (!processingData.isContainingSimpleFeatureTypes()) { - throw new BadRequestException(ExceptionMessages.TYPES_PARAM); - } - } else { - if (processingData.isContainingSimpleFeatureTypes()) { - throw new BadRequestException(ExceptionMessages.TYPES_PARAM); - } - } - } - } - } - /** * Checks the content of the given format parameter. * diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/ProcessingData.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/ProcessingData.java index 5762474f..41c4054c 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/ProcessingData.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/ProcessingData.java @@ -1,7 +1,6 @@ package org.heigit.ohsome.ohsomeapi.inputprocessing; import java.io.Serializable; -import java.util.EnumSet; import java.util.List; import java.util.Optional; import lombok.Getter; @@ -9,7 +8,6 @@ import org.geojson.GeoJsonObject; import org.heigit.ohsome.ohsomeapi.executor.RequestParameters; import org.heigit.ohsome.oshdb.filter.FilterExpression; -import org.heigit.ohsome.oshdb.osm.OSMType; import org.locationtech.jts.geom.Geometry; /** Holds the relevant objects for processing the request and creating the response.*/ @@ -30,14 +28,11 @@ public class ProcessingData implements Serializable { private BoundaryType boundaryType; private String[] boundaryValues; private String boundaryValuesGeoJson; - private EnumSet osmTypes; private boolean showMetadata; private String format; private Geometry requestGeom; private List boundaryList; private GeoJsonObject[] geoJsonGeoms; - private boolean isContainingSimpleFeatureTypes; - private EnumSet simpleFeatureTypes; @Getter @Setter private static int numberOfClusterNodes; diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/ResourceParameters.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/ResourceParameters.java index 7fa10bda..3dfe72d2 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/ResourceParameters.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/ResourceParameters.java @@ -25,7 +25,7 @@ private ResourceParameters() { public static List getResourceSpecificParams(HttpServletRequest servletRequest) { String uri = servletRequest.getRequestURI(); List possibleParams = new LinkedList<>(Arrays.asList("bboxes", "bcircles", "bpolys", - "types", "keys", "values", "timeout", "time", "showMetadata", "filter")); + "timeout", "time", "showMetadata", "filter")); if (uri.contains("/count") || uri.contains("/length") || uri.contains("/area") || uri.contains("/perimeter")) { possibleParams.add("format"); @@ -37,21 +37,12 @@ public static List getResourceSpecificParams(HttpServletRequest servletR possibleParams.add("groupByKeys"); } else if (uri.contains("/ratio")) { possibleParams.add("filter2"); - possibleParams.add("keys2"); - possibleParams.add("types2"); - possibleParams.add("values2"); } else if (uri.contains("/bbox") || uri.contains("/centroid") || uri.contains("/geometry")) { possibleParams.add("properties"); possibleParams.add("clipGeometry"); } - if (uri.contains("/contributions")) { - // removing deprecated params from newer endpoint - possibleParams.remove("types"); - possibleParams.remove("keys"); - possibleParams.remove("values"); - if (uri.contains("/count")) { - possibleParams.add("contributionType"); - } + if (uri.contains("/contributions") && uri.contains("/count")) { + possibleParams.add("contributionType"); } if (uri.contains("/users")) { possibleParams.add("contributionType"); diff --git a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/StringSimilarity.java b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/StringSimilarity.java index 5bb91f13..9c523c0a 100644 --- a/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/StringSimilarity.java +++ b/src/main/lombok/org/heigit/ohsome/ohsomeapi/inputprocessing/StringSimilarity.java @@ -2,7 +2,6 @@ import java.math.BigDecimal; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Locale; import org.apache.commons.text.similarity.FuzzyScore; diff --git a/src/test/java/org/heigit/ohsome/ohsomeapi/controller/GetControllerTest.java b/src/test/java/org/heigit/ohsome/ohsomeapi/controller/GetControllerTest.java index d7623e99..f1c8ee61 100644 --- a/src/test/java/org/heigit/ohsome/ohsomeapi/controller/GetControllerTest.java +++ b/src/test/java/org/heigit/ohsome/ohsomeapi/controller/GetControllerTest.java @@ -1034,8 +1034,8 @@ public void getElementsLengthDensityGroupByBoundaryGroupByTagCsvTest() throws IO // remainder , value 1 , ... , value N) final double expectedValue = 3190.17; String responseBody = getResponseBody("/elements/length/density/groupBy/boundary" - + "/groupBy/tag?bboxes=b1:8.68086,49.39948,8.69401" - + ",49.40609|b2:8.68081,49.39943,8.69408,49.40605&types=way&time=2017-10-08&keys=highway&" + + "/groupBy/tag?bboxes=b1:8.68086,49.39948,8.69401,49.40609|" + + "b2:8.68081,49.39943,8.69408,49.40605&time=2017-10-08&filter=type:way and highway=*&" + "groupByKey=highway&format=csv&groupByValues=residential,primary"); List records = Helper.getCsvRecords(responseBody); assertEquals(1, Helper.getCsvRecords(responseBody).size()); @@ -1255,26 +1255,6 @@ public void ratioEmptyFilter2Test() { assertEquals(400, response.getBody().get("status").asInt()); } - @Test - public void getElementsCountWrongFilterTypesCombinationTest() { - TestRestTemplate restTemplate = new TestRestTemplate(); - ResponseEntity response = - restTemplate.getForEntity(server + port + "/elements/count?bboxes=8.67452,49.40961," - + "8.70392,49.41823&time=2015-01-01&filter=building=*&types=way", JsonNode.class); - assertEquals(400, response.getBody().get("status").asInt()); - } - - @Test - public void getElementsCountWrongFilterKeysCombinationTest() { - TestRestTemplate restTemplate = new TestRestTemplate(); - ResponseEntity response = - restTemplate.getForEntity( - server + port + "/elements/count?bboxes=8.67452,49.40961," - + "8.70392,49.41823&time=2015-01-01&filter=building=*&keys=building", - JsonNode.class); - assertEquals(400, response.getBody().get("status").asInt()); - } - @Test public void getFilterTest() { final double expectedValue = 585.48; diff --git a/src/test/java/org/heigit/ohsome/ohsomeapi/executor/ContributionsExecutorTest.java b/src/test/java/org/heigit/ohsome/ohsomeapi/executor/ContributionsExecutorTest.java index 7cbf4d59..aa68362b 100644 --- a/src/test/java/org/heigit/ohsome/ohsomeapi/executor/ContributionsExecutorTest.java +++ b/src/test/java/org/heigit/ohsome/ohsomeapi/executor/ContributionsExecutorTest.java @@ -1,9 +1,9 @@ package org.heigit.ohsome.ohsomeapi.executor; - import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.EnumSet; import java.util.Set; import java.util.function.Predicate; @@ -45,7 +45,8 @@ public void contributionsFilter() { @Test public void contributionsFilterInvalid() { - assertThrows(BadRequestException.class, () -> ExecutionUtils.contributionsFilter("doesnotexist")); + assertThrows(BadRequestException.class, () -> + ExecutionUtils.contributionsFilter("doesnotexist")); } diff --git a/src/test/java/org/heigit/ohsome/ohsomeapi/inputprocessing/GeometryBuilderTest.java b/src/test/java/org/heigit/ohsome/ohsomeapi/inputprocessing/GeometryBuilderTest.java index 602dd21e..2f12ea59 100644 --- a/src/test/java/org/heigit/ohsome/ohsomeapi/inputprocessing/GeometryBuilderTest.java +++ b/src/test/java/org/heigit/ohsome/ohsomeapi/inputprocessing/GeometryBuilderTest.java @@ -124,7 +124,8 @@ public void createPolygonFromGeoJsonWithWrongGeomType() { + "{\"id\":\"Neuenheim\"},\"geometry\":{\"type\":\"Point\",\"coordinates\":[[[8.68465," + "49.41769]]]}}]}"; InputProcessor inputProcessor = new InputProcessor(processingData); - assertThrows(BadRequestException.class, () -> geomBuilder.createGeometryFromGeoJson(geoJson, inputProcessor)); + assertThrows(BadRequestException.class, () -> + geomBuilder.createGeometryFromGeoJson(geoJson, inputProcessor)); } @Test @@ -136,14 +137,16 @@ public void createPolygonFromGeoJsonWithWrongFormat() { + "[8.2933214,49.4329125],[8.2936734,49.4330121],[8.2940745,49.4331354]," + "[8.2950478,49.4317345],[8.2944706,49.4313443]]]]}]}"; InputProcessor inputProcessor = new InputProcessor(processingData); - assertThrows(BadRequestException.class, () -> geomBuilder.createGeometryFromGeoJson(geoJson, inputProcessor)); + assertThrows(BadRequestException.class, () -> + geomBuilder.createGeometryFromGeoJson(geoJson, inputProcessor)); } @Test public void createGeometryFromInvalidInputGeoJson() { String geoJson = "{\"type\": \"FeatureCollection\"}"; InputProcessor inputProcessor = new InputProcessor(processingData); - assertThrows(BadRequestException.class, () -> geomBuilder.createGeometryFromGeoJson(geoJson, inputProcessor)); + assertThrows(BadRequestException.class, () -> + geomBuilder.createGeometryFromGeoJson(geoJson, inputProcessor)); } @Test @@ -194,6 +197,7 @@ public void createGeometryFromMetadataGeoJson() { @Test public void createGeometryFromWrongMetadataGeoJson() { String geoJson = "{\"type\":\"Polygon\",\"coordinates\":[Invalid-Input]}"; - assertThrows(RuntimeException.class, () ->geomBuilder.createGeometryFromMetadataGeoJson(geoJson)); + assertThrows(RuntimeException.class, () -> + geomBuilder.createGeometryFromMetadataGeoJson(geoJson)); } }