From 2a165b3b4620009d12213cd45c2b743e455c0f4c Mon Sep 17 00:00:00 2001 From: Armin Date: Wed, 10 Jan 2024 13:30:05 +0100 Subject: [PATCH 1/2] [BIK-878] Add support for sensor export --- .../java/de/cyface/model/Measurement.java | 147 ++++++++++----- .../kotlin/de/cyface/model/ExportOptions.kt | 171 ++++++++++++++++++ .../java/de/cyface/model/MeasurementTest.java | 35 +++- 3 files changed, 304 insertions(+), 49 deletions(-) create mode 100644 libs/model/src/main/kotlin/de/cyface/model/ExportOptions.kt diff --git a/libs/model/src/main/java/de/cyface/model/Measurement.java b/libs/model/src/main/java/de/cyface/model/Measurement.java index 1243661..2caa0b4 100644 --- a/libs/model/src/main/java/de/cyface/model/Measurement.java +++ b/libs/model/src/main/java/de/cyface/model/Measurement.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2023 Cyface GmbH + * Copyright 2019-2024 Cyface GmbH * * This file is part of the Serialization. * @@ -51,7 +51,7 @@ * * @author Armin Schnabel * @author Klemens Muthmann - * @version 2.1.1 + * @version 3.0.0 * @since 1.0.0 */ public class Measurement implements Serializable { @@ -190,30 +190,43 @@ public void setTracks(final List tracks) { /** * Exports this measurement as a CSV file. * - * @param withCsvHeader {@code True} if a CSV header should be added before the data + * @param options The options which describe which data should be exported. * @param handler A handler that gets one line of CSV output per call */ @SuppressWarnings("unused") // API used by backend/executables/cyface-to-csv - public void asCsv(final boolean withCsvHeader, final Consumer handler) { - asCsv(withCsvHeader, null, handler); + public void asCsv(final ExportOptions options, final Consumer handler) { + asCsv(options, null, handler); } /** * Exports this measurement as a CSV file. * - * @param withCsvHeader {@code True} if a CSV header should be added before the data + * @param options The options which describe which data should be exported. * @param username The name of the user who uploaded the data or {@code null} to not export this field * @param handler A handler that gets one line of CSV output per call */ - public void asCsv(final boolean withCsvHeader, final String username, final Consumer handler) { - if (withCsvHeader) { - if (username == null) { - csvHeaderWithUsername(handler); - } else { - csvHeader(handler); - } + public void asCsv(final ExportOptions options, final String username, final Consumer handler) { + Validate.isTrue(!options.getIncludeUsername() || username != null); + + if (options.getIncludeHeader()) { + csvHeader(options, handler); + } + + switch (options.getType()) { + case LOCATION: + locationDataAsCsv(options, username, handler); + break; + case ACCELERATION: + case ROTATION: + case DIRECTION: + sensorDataAsCsv(options, username, handler); + break; + default: + throw new IllegalArgumentException(String.format("Unsupported type: %s", options.getType())); } + } + private void locationDataAsCsv(ExportOptions options, String username, Consumer handler) { Modality lastModality = Modality.UNKNOWN; // Iterate through tracks @@ -244,15 +257,34 @@ public void asCsv(final boolean withCsvHeader, final String username, final Cons modalityTypeTravelTime = 0L; } - handler.accept(csvRow(username, getMetaData(), locationRecord, trackId, + handler.accept(csvRow(options, username, getMetaData(), locationRecord, trackId, modalityTypeDistance, totalDistance, modalityTypeTravelTime, totalTravelTime)); + handler.accept("\r\n"); lastLocation = locationRecord; } } } + private void sensorDataAsCsv(ExportOptions options, String username, Consumer handler) { + // Iterate through tracks + for (var trackId = 0; trackId < tracks.size(); trackId++) { + final var track = tracks.get(trackId); + + // Iterate through sensor points + final var points = options.getType().equals(DataType.ACCELERATION) ? track.getAccelerations() + : options.getType().equals(DataType.ROTATION) ? track.getRotations() + : options.getType().equals(DataType.DIRECTION) ? track.getDirections() + : null; + Validate.notNull(points, "Unsupported type: " + options.getType()); + for (final var point : points) { + handler.accept(csvSensorRow(options, username, getMetaData(), point, trackId)); + handler.accept("\r\n"); + } + } + } + /** * Exports this measurement as GeoJSON feature. * @@ -428,44 +460,47 @@ private int getIndexOfTrackContaining(final long timestamp) throws TimestampNotF /** * Creates a CSV header for this measurement. * + * @param options The options describing which data is exported * @param handler The handler that is notified of the new CSV row. */ - public static void csvHeader(final Consumer handler) { - csvHeader(true, handler); - } - - /** - * Creates a CSV header for this measurement. - * - * @param handler The handler that is notified of the new CSV row. - */ - public static void csvHeaderWithUsername(final Consumer handler) { - csvHeader(false, handler); - } - - /** - * Creates a CSV header for this measurement. - * - * @param includeUsername {@code True} if the username should be included - * @param handler The handler that is notified of the new CSV row. - */ - private static void csvHeader(final boolean includeUsername, final Consumer handler) { + public static void csvHeader(final ExportOptions options, final Consumer handler) { final var elements = new ArrayList(); - elements.add("userId"); - if (includeUsername) { + if (options.getIncludeUserId()) { + elements.add("userId"); + } + if (options.getIncludeUsername()) { elements.add("username"); } - elements.addAll(List.of("deviceId", "measurementId", "trackId", "timestamp [ms]", "latitude", "longitude", - "speed [m/s]", "accuracy [m]", "modalityType", "modalityTypeDistance [m]", "distance [m]", - "modalityTypeTravelTime [ms]", "travelTime [ms]")); + elements.addAll(List.of("deviceId", "measurementId", "trackId", "timestamp [ms]")); + switch (options.getType()) { + case LOCATION: + elements.addAll(List.of("latitude", "longitude", + "speed [m/s]", "accuracy [m]", "modalityType", "modalityTypeDistance [m]", "distance [m]", + "modalityTypeTravelTime [ms]", "travelTime [ms]")); + break; + case ACCELERATION: + elements.addAll(List.of("x [m/s^2]", "y [m/s^2]", "z [m/s^2]")); + break; + case ROTATION: + elements.addAll(List.of("x [rad/s]", "y [rad/s]", "z [rad/s]")); + break; + case DIRECTION: + elements.addAll(List.of("x [uT]", "y [uT]", "z [uT]")); + break; + default: + throw new IllegalArgumentException(String.format("Unsupported type: %s", options.getType())); + } + final var csvHeaderRow = String.join(",", elements); handler.accept(csvHeaderRow); + handler.accept("\r\n"); } /** * Converts one location entry annotated with metadata to a CSV row. * + * @param options The options which describe which data should be exported. * @param username the name of the user who uploaded the data or {@code null} to not annotate a username * @param metaData the {@code Measurement} of the {@param location} * @param locationRecord the {@code GeoLocationRecord} to be processed @@ -476,7 +511,8 @@ private static void csvHeader(final boolean includeUsername, final Consumer(); - elements.add(userId.toString()); - if (username != null) { + if (options.getIncludeUserId()) { + elements.add(userId.toString()); + } + if (options.getIncludeUsername()) { + Validate.notNull(username); elements.add(username); } elements.addAll(List.of(deviceId, measurementId, String.valueOf(trackId), @@ -500,6 +539,30 @@ private String csvRow(final String username, final MetaData metaData, final RawR return String.join(",", elements); } + private String csvSensorRow(ExportOptions options, final String username, final MetaData metaData, + final Point3DImpl pointRecord, + final int trackId) { + + final var userId = metaData.getUserId(); + final var deviceId = metaData.getIdentifier().getDeviceIdentifier(); + final var measurementId = String.valueOf(metaData.getIdentifier().getMeasurementIdentifier()); + + final var elements = new ArrayList(); + if (options.getIncludeUserId()) { + elements.add(userId.toString()); + } + if (options.getIncludeUsername()) { + Validate.notNull(username); + elements.add(username); + } + elements.addAll(List.of(deviceId, measurementId, String.valueOf(trackId), + String.valueOf(pointRecord.getTimestamp()), + String.valueOf(pointRecord.getX()), + String.valueOf(pointRecord.getY()), + String.valueOf(pointRecord.getZ()))); + return String.join(",", elements); + } + /** * Converts a single track to geoJson "coordinates". * diff --git a/libs/model/src/main/kotlin/de/cyface/model/ExportOptions.kt b/libs/model/src/main/kotlin/de/cyface/model/ExportOptions.kt new file mode 100644 index 0000000..1316cbd --- /dev/null +++ b/libs/model/src/main/kotlin/de/cyface/model/ExportOptions.kt @@ -0,0 +1,171 @@ +/* + * Copyright 2024 Cyface GmbH + * + * This file is part of the Serialization. + * + * The Serialization is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * The Serialization is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the Serialization. If not, see . + */ + +package de.cyface.model + +/** + * The options which describe which data should be exported. + * + * @author Armin Schnabel + * @version 1.0.0 + * @since 3.3.0 + * @property format The format in which the data should be exported. + * @property type The type of data which should be exported. + * @property includeHeader `true` if the export should start with a header line, if supported by the [format]. + * @property includeUserId `true` if the user id should be included in the export. + * @property includeUsername `true` if the username should be included in the export. + */ +class ExportOptions { + var format: DataFormat = DataFormat.UNDEFINED + var type: DataType = DataType.UNDEFINED + var includeHeader: Boolean = false + var includeUserId: Boolean = false + var includeUsername: Boolean = false + + fun format(format: DataFormat): ExportOptions { + this.format = format + return this + } + + fun type(type: DataType): ExportOptions { + this.type = type + return this + } + + fun includeHeader(includeHeader: Boolean): ExportOptions { + require(!includeHeader || format == DataFormat.CSV) { "Format without header support: $format" } + this.includeHeader = includeHeader + return this + } + + fun includeUserId(includeUserId: Boolean): ExportOptions { + this.includeUserId = includeUserId + return this + } + + fun includeUsername(includeUsername: Boolean): ExportOptions { + this.includeUsername = includeUsername + return this + } +} + +/** + * Supported export data format. + * + * @author Armin Schnabel + * @version 1.0.0 + * @since 3.3.0 + * @param parameterValue The parameter value which represents the enum option. + */ +enum class DataFormat( + @Suppress("MemberVisibilityCanBePrivate") val parameterValue: String +) { + /** + * Default value when no format was specified. + */ + UNDEFINED("undefined"), + + /** + * Comma separated values. + */ + CSV("csv"), + + /** + * JSON format. + */ + @Suppress("unused") // Used + JSON("json"); + + companion object { + private val BY_PARAMETER_VALUE: MutableMap = HashMap() + + init { + values().forEach { format -> + BY_PARAMETER_VALUE[format.parameterValue] = format + } + } + + /** + * Returns the [DataFormat] from it's `#parameterValue` value. + * + * @param parameterValue The `String` value of the parameterValue. + * @return The `DataFormat` for the parameterValue. + */ + @Suppress("unused") // Part of the API + fun valueOfParameterValue(parameterValue: String): DataFormat? { + return BY_PARAMETER_VALUE[parameterValue] + } + } +} + +/** + * Supported data types which are used during data collection. + * + * @author Armin Schnabel + * @version 1.0.0 + * @since 3.3.0 + * @param parameterValue The parameter value which represents the enum option. + */ +enum class DataType(@Suppress("MemberVisibilityCanBePrivate") val parameterValue: String) { + /** + * Default value when no type was specified. + */ + UNDEFINED("undefined"), + + /** + * Data collected by the accelerometer. + */ + ACCELERATION("acceleration"), + + /** + * Data collected by the gyroscope. + */ + ROTATION("rotation"), + + /** + * Data collected by the magnetometer. + */ + DIRECTION("direction"), + + /** + * Location data, e.g. collected from a GNSS signal. + */ + LOCATION("location"); + + companion object { + private val BY_PARAMETER_VALUE: MutableMap = HashMap() + + init { + DataType.values().forEach { entry -> + BY_PARAMETER_VALUE[entry.parameterValue] = entry + } + } + + /** + * Returns the [DataType] from it's `#parameterValue` value. + * + * @param parameterValue The `String` value of the parameterValue. + * @return The `DataType` for the parameterValue. + */ + @Suppress("unused") // Part of the API + fun valueOfParameterValue(parameterValue: String): DataType? { + return BY_PARAMETER_VALUE[parameterValue] + } + } +} \ No newline at end of file diff --git a/libs/model/src/test/java/de/cyface/model/MeasurementTest.java b/libs/model/src/test/java/de/cyface/model/MeasurementTest.java index 59c6405..02794fd 100644 --- a/libs/model/src/test/java/de/cyface/model/MeasurementTest.java +++ b/libs/model/src/test/java/de/cyface/model/MeasurementTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2023 Cyface GmbH + * Copyright 2020-2024 Cyface GmbH * * This file is part of the Serialization. * @@ -47,7 +47,7 @@ * * @author Klemens Muthmann * @author Armin Schnabel - * @version 1.0.3 + * @version 1.0.4 */ public class MeasurementTest { @@ -77,11 +77,20 @@ void testWriteCsvHeaderRow() { // Arrange final var expectedHeader = "userId,username,deviceId,measurementId,trackId,timestamp [ms],latitude,longitude," + "speed [m/s],accuracy [m],modalityType,modalityTypeDistance [m],distance [m],modalityTypeTravelTime" - + " [ms],travelTime [ms]"; + + " [ms],travelTime [ms]\r\n"; // Act + final var csvOutput = new StringBuilder(); + final var options = new ExportOptions() + .format(DataFormat.CSV) + .type(DataType.LOCATION) + .includeHeader(true) + .includeUserId(true) + .includeUsername(true); + Measurement.csvHeader(options, csvOutput::append); + // Assert - Measurement.csvHeader(row -> assertThat(row, is(equalTo(expectedHeader)))); + assertThat(csvOutput.toString(), is(equalTo(expectedHeader))); } /** @@ -119,7 +128,13 @@ void testWriteLocationAsCsvRows_withoutModalityChanges() { // Act final var csvOutput = new StringBuilder(); - measurement.asCsv(true, TEST_USER_USERNAME, line -> csvOutput.append(line).append("\r\n")); + final var options = new ExportOptions() + .format(DataFormat.CSV) + .type(DataType.LOCATION) + .includeHeader(true) + .includeUserId(true) + .includeUsername(true); + measurement.asCsv(options, TEST_USER_USERNAME, csvOutput::append); // Assert assertThat(csvOutput.toString(), is(equalTo(expectedOutput))); @@ -152,7 +167,6 @@ void testWriteLocationAsCsvRows_withModalityTypeChanges() { point3DS, point3DS, point3DS)); final var measurement = new Measurement(metaData, tracks); - final var csvOutput = new StringBuilder(); final var expectedOutput = "userId,username,deviceId,measurementId,trackId,timestamp [ms],latitude,longitude," + "speed [m/s],accuracy [m],modalityType,modalityTypeDistance [m],distance [m]," + "modalityTypeTravelTime [ms],travelTime [ms]\r\n" @@ -174,7 +188,14 @@ void testWriteLocationAsCsvRows_withModalityTypeChanges() { + ",13.110048189675535,26.236156837054857,1000,1500\r\n"; // Act - measurement.asCsv(true, TEST_USER_USERNAME, line -> csvOutput.append(line).append("\r\n")); + final var csvOutput = new StringBuilder(); + final var options = new ExportOptions() + .format(DataFormat.CSV) + .type(DataType.LOCATION) + .includeHeader(true) + .includeUserId(true) + .includeUsername(true); + measurement.asCsv(options, TEST_USER_USERNAME, csvOutput::append); // Assert assertThat(csvOutput.toString(), is(equalTo(expectedOutput))); From abad6e4f8c6649e676404e42eb4aae6537013c2d Mon Sep 17 00:00:00 2001 From: Armin Date: Fri, 12 Jan 2024 13:27:19 +0100 Subject: [PATCH 2/2] Implement PR feedback (cleanup) --- libs/model/src/main/kotlin/de/cyface/model/ExportOptions.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/model/src/main/kotlin/de/cyface/model/ExportOptions.kt b/libs/model/src/main/kotlin/de/cyface/model/ExportOptions.kt index 1316cbd..f7ab6df 100644 --- a/libs/model/src/main/kotlin/de/cyface/model/ExportOptions.kt +++ b/libs/model/src/main/kotlin/de/cyface/model/ExportOptions.kt @@ -74,7 +74,7 @@ class ExportOptions { * @param parameterValue The parameter value which represents the enum option. */ enum class DataFormat( - @Suppress("MemberVisibilityCanBePrivate") val parameterValue: String + val parameterValue: String ) { /** * Default value when no format was specified. @@ -108,7 +108,7 @@ enum class DataFormat( * @return The `DataFormat` for the parameterValue. */ @Suppress("unused") // Part of the API - fun valueOfParameterValue(parameterValue: String): DataFormat? { + fun of(parameterValue: String): DataFormat? { return BY_PARAMETER_VALUE[parameterValue] } } @@ -164,7 +164,7 @@ enum class DataType(@Suppress("MemberVisibilityCanBePrivate") val parameterValue * @return The `DataType` for the parameterValue. */ @Suppress("unused") // Part of the API - fun valueOfParameterValue(parameterValue: String): DataType? { + fun of(parameterValue: String): DataType? { return BY_PARAMETER_VALUE[parameterValue] } }