From 94403686df7a8311ffca21f067165969b87e0d2c Mon Sep 17 00:00:00 2001
From: davitbzh <44586065+davitbzh@users.noreply.github.com>
Date: Tue, 4 Apr 2023 22:48:56 +0200
Subject: [PATCH] [FSTORE-789] multi modular hsfs java client (#975)
multi modular hsfs java client
---
.github/workflows/mkdocs-master.yml | 8 +-
.github/workflows/mkdocs-release.yml | 8 +-
java/hsfs/pom.xml | 28 ++++
.../com/logicalclocks/hsfs/DataFormat.java | 0
.../hsfs/DeltaStreamerJobConf.java | 0
.../hsfs/EntityEndpointType.java | 0
.../hsfs/ExternalDataFormat.java | 0
.../java/com/logicalclocks/hsfs/Feature.java | 0
.../logicalclocks/hsfs/FeatureGroupBase.java | 0
.../hsfs/FeatureGroupBaseForApi.java | 0
.../hsfs/FeatureGroupCommit.java | 0
.../logicalclocks/hsfs/FeatureStoreBase.java | 0
.../hsfs/FeatureStoreException.java | 0
.../com/logicalclocks/hsfs/FeatureType.java | 0
.../logicalclocks/hsfs/FeatureViewBase.java | 0
.../hsfs/HopsworksConnectionBase.java | 0
.../logicalclocks/hsfs/HudiOperationType.java | 0
.../logicalclocks/hsfs/JobConfiguration.java | 0
.../java/com/logicalclocks/hsfs/Project.java | 0
.../com/logicalclocks/hsfs/SecretStore.java | 0
.../logicalclocks/hsfs/SecurityProtocol.java | 0
.../java/com/logicalclocks/hsfs/Split.java | 0
.../SslEndpointIdentificationAlgorithm.java | 0
.../logicalclocks/hsfs/StatisticsConfig.java | 0
.../java/com/logicalclocks/hsfs/Storage.java | 0
.../logicalclocks/hsfs/StorageConnector.java | 14 +-
.../hsfs/StorageConnectorType.java | 0
.../logicalclocks/hsfs/TimeTravelFormat.java | 0
.../hsfs/TrainingDatasetBase.java | 0
.../hsfs/TrainingDatasetFeature.java | 0
.../hsfs/TrainingDatasetType.java | 0
.../hsfs/TransformationFunction.java | 0
.../hsfs/constructor/FeatureGroupAlias.java | 0
.../hsfs/constructor/Filter.java | 0
.../hsfs/constructor/FilterLogic.java | 0
.../hsfs/constructor/FsQueryBase.java | 0
.../logicalclocks/hsfs/constructor/Join.java | 0
.../hsfs/constructor/JoinType.java | 0
.../PreparedStatementParameter.java | 0
.../hsfs/constructor/QueryBase.java | 0
.../constructor/ServingPreparedStatement.java | 0
.../hsfs/constructor/SqlFilterCondition.java | 0
.../hsfs/constructor/SqlFilterLogic.java | 0
.../logicalclocks/hsfs/engine/CodeEngine.java | 0
.../hsfs/engine/FeatureGroupEngineBase.java | 2 +-
.../hsfs/engine/FeatureGroupUtils.java | 8 +-
.../hsfs/engine/FeatureViewEngineBase.java | 8 +-
.../hsfs/engine/VectorServer.java | 0
.../hsfs/metadata/AuthorizationHandler.java | 0
.../com/logicalclocks/hsfs/metadata/Code.java | 0
.../logicalclocks/hsfs/metadata/CodeApi.java | 0
.../hsfs/metadata/Credentials.java | 0
.../hsfs/metadata/FeatureGroupApi.java | 0
.../hsfs/metadata/FeatureStoreApi.java | 0
.../hsfs/metadata/FeatureViewApi.java | 0
.../hsfs/metadata/HopsworksClient.java | 0
.../metadata/HopsworksExternalClient.java | 0
.../metadata/HopsworksHostnameVerifier.java | 0
.../hsfs/metadata/HopsworksHttpClient.java | 0
.../metadata/HopsworksInternalClient.java | 0
.../hsfs/metadata/InternalException.java | 0
.../logicalclocks/hsfs/metadata/KafkaApi.java | 0
.../hsfs/metadata/KafkaClusterInfo.java | 0
.../hsfs/metadata/OnDemandOptions.java | 0
.../logicalclocks/hsfs/metadata/Option.java | 0
.../hsfs/metadata/PartitionDetails.java | 0
.../hsfs/metadata/ProjectApi.java | 0
.../hsfs/metadata/QueryConstructorApi.java | 2 +-
.../logicalclocks/hsfs/metadata/RestDto.java | 0
.../hsfs/metadata/SplitStatistics.java | 0
.../hsfs/metadata/Statistics.java | 0
.../hsfs/metadata/StatisticsApi.java | 13 +-
.../hsfs/metadata/StorageConnectorApi.java | 0
.../logicalclocks/hsfs/metadata/Subject.java | 0
.../com/logicalclocks/hsfs/metadata/Tags.java | 0
.../logicalclocks/hsfs/metadata/TagsApi.java | 12 +-
.../hsfs/metadata/TrainingDatasetApi.java | 0
.../TransformationFunctionAttached.java | 0
.../hsfs/metadata/UnauthorizedException.java | 0
.../com/logicalclocks/hsfs/metadata/User.java | 0
.../logicalclocks/hsfs/util/Constants.java | 0
.../hsfs}/TestHopsworksExternalClient.java | 3 +-
.../hsfs/metadata/TestHopsworksClient.java | 2 +-
.../hsfs/metadata/TestTagsApi.java | 0
java/pom.xml | 158 +++++-------------
java/spark/pom.xml | 158 ++++++++++++++++++
.../hsfs/spark/ExternalFeatureGroup.java | 6 +-
.../hsfs/spark/FeatureGroup.java | 6 +-
.../hsfs/spark/FeatureStore.java | 33 ++--
.../logicalclocks/hsfs/spark/FeatureView.java | 8 +-
.../hsfs/spark/HopsworksConnection.java | 2 +-
.../logicalclocks/hsfs/spark/MainClass.java | 4 +-
.../hsfs/spark/StreamFeatureGroup.java | 6 +-
.../hsfs/spark/TrainingDataset.java | 8 +-
.../hsfs/spark/TrainingDatasetBundle.java | 2 +-
.../hsfs/spark/constructor/FsQuery.java | 2 +-
.../hsfs/spark/constructor/Query.java | 4 +-
.../hsfs/spark/engine/FeatureGroupEngine.java | 6 +-
.../hsfs/spark/engine/FeatureViewEngine.java | 2 +-
.../hsfs/spark/engine/SparkEngine.java | 6 +-
.../hsfs/spark/engine/StatisticsEngine.java | 0
.../spark/engine/TrainingDatasetEngine.java | 2 +-
.../spark/engine/TrainingDatasetUtils.java | 0
.../hudi/DeltaStreamerAvroDeserializer.java | 0
.../engine/hudi/DeltaStreamerConfig.java | 0
.../engine/hudi/DeltaStreamerKafkaSource.java | 0
.../hudi/DeltaStreamerSchemaProvider.java | 0
.../engine/hudi/DeltaStreamerTransformer.java | 0
.../hsfs/spark/engine/hudi/HudiEngine.java | 12 +-
.../spark/util/StorageConnectorUtils.java | 11 +-
.../hsfs/spark/TestExternalFeatureGroup.java | 0
.../logicalclocks/hsfs/spark/TestFeature.java | 0
.../hsfs/spark/TestFeatureGroup.java | 0
.../hsfs/spark}/TestStorageConnector.java | 14 +-
.../spark/engine/TestFeatureViewEngine.java | 0
.../hsfs/spark/engine/TestHudiEngine.java | 0
.../hsfs/spark/engine/TestSparkEngine.java | 0
.../test/resources/hadoop/bin/winutils.exe | Bin 0 -> 112640 bytes
.../src/test/resources/system.properties | 1 +
119 files changed, 338 insertions(+), 221 deletions(-)
create mode 100644 java/hsfs/pom.xml
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/DataFormat.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/Feature.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/FeatureType.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/FeatureViewBase.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/HopsworksConnectionBase.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/HudiOperationType.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/JobConfiguration.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/Project.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/SecretStore.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/SecurityProtocol.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/Split.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/SslEndpointIdentificationAlgorithm.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/StatisticsConfig.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/Storage.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/StorageConnector.java (96%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/TimeTravelFormat.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/TrainingDatasetBase.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/TrainingDatasetFeature.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/TransformationFunction.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/FeatureGroupAlias.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/Filter.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/FilterLogic.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/FsQueryBase.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/Join.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/JoinType.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/PreparedStatementParameter.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/QueryBase.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/ServingPreparedStatement.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterCondition.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterLogic.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/engine/CodeEngine.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/engine/VectorServer.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/Code.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/CodeApi.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/FeatureViewApi.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/KafkaApi.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/KafkaClusterInfo.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/OnDemandOptions.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/Option.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/PartitionDetails.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/SplitStatistics.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/Statistics.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java (93%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/Subject.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java (96%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/TransformationFunctionAttached.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/metadata/User.java (100%)
rename java/{ => hsfs}/src/main/java/com/logicalclocks/hsfs/util/Constants.java (100%)
rename java/{src/test/java/com/logicalclocks => hsfs/src/test/java/com/logicalclocks/hsfs}/TestHopsworksExternalClient.java (97%)
rename java/{ => hsfs}/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java (97%)
rename java/{ => hsfs}/src/test/java/com/logicalclocks/hsfs/metadata/TestTagsApi.java (100%)
create mode 100644 java/spark/pom.xml
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java (98%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java (99%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java (99%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/TrainingDatasetBundle.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/constructor/FsQuery.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/constructor/Query.java (94%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/FeatureGroupEngine.java (99%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/FeatureViewEngine.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/SparkEngine.java (99%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/StatisticsEngine.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetEngine.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetUtils.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerAvroDeserializer.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerConfig.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerKafkaSource.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerSchemaProvider.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerTransformer.java (100%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java (97%)
rename java/{ => spark}/src/main/java/com/logicalclocks/hsfs/spark/util/StorageConnectorUtils.java (97%)
rename java/{ => spark}/src/test/java/com/logicalclocks/hsfs/spark/TestExternalFeatureGroup.java (100%)
rename java/{ => spark}/src/test/java/com/logicalclocks/hsfs/spark/TestFeature.java (100%)
rename java/{ => spark}/src/test/java/com/logicalclocks/hsfs/spark/TestFeatureGroup.java (100%)
rename java/{src/test/java/com/logicalclocks => spark/src/test/java/com/logicalclocks/hsfs/spark}/TestStorageConnector.java (94%)
rename java/{ => spark}/src/test/java/com/logicalclocks/hsfs/spark/engine/TestFeatureViewEngine.java (100%)
rename java/{ => spark}/src/test/java/com/logicalclocks/hsfs/spark/engine/TestHudiEngine.java (100%)
rename java/{ => spark}/src/test/java/com/logicalclocks/hsfs/spark/engine/TestSparkEngine.java (100%)
create mode 100644 java/spark/src/test/resources/hadoop/bin/winutils.exe
create mode 100644 java/spark/src/test/resources/system.properties
diff --git a/.github/workflows/mkdocs-master.yml b/.github/workflows/mkdocs-master.yml
index 10ad5ad6f2..0ea51b9216 100644
--- a/.github/workflows/mkdocs-master.yml
+++ b/.github/workflows/mkdocs-master.yml
@@ -35,9 +35,15 @@ jobs:
restore-keys: |
${{ runner.os }}-maven-
+ - name: Set up JDK 8
+ uses: actions/setup-java@v3
+ with:
+ java-version: '8'
+ distribution: 'adopt'
+
- name: Build java doc documentation
working-directory: ./java
- run: mvn clean package javadoc:javadoc -DskipTests && cp -r target/apidocs ../docs/javadoc
+ run: mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc
- name: setup git
run: |
diff --git a/.github/workflows/mkdocs-release.yml b/.github/workflows/mkdocs-release.yml
index 1fdc28288c..f80e9ad2f4 100644
--- a/.github/workflows/mkdocs-release.yml
+++ b/.github/workflows/mkdocs-release.yml
@@ -42,9 +42,15 @@ jobs:
restore-keys: |
${{ runner.os }}-maven-
+ - name: Set up JDK 8
+ uses: actions/setup-java@v3
+ with:
+ java-version: '8'
+ distribution: 'adopt'
+
- name: Build java doc documentation
working-directory: ./java
- run: mvn clean package javadoc:javadoc -DskipTests && cp -r target/apidocs ../docs/javadoc
+ run: mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc
- name: setup git
run: |
diff --git a/java/hsfs/pom.xml b/java/hsfs/pom.xml
new file mode 100644
index 0000000000..319f4a6794
--- /dev/null
+++ b/java/hsfs/pom.xml
@@ -0,0 +1,28 @@
+
+
+
+ hsfs-sdk
+ com.logicalclocks
+ 3.2.0-SNAPSHOT
+
+ 4.0.0
+
+ hsfs
+
+
+
+ 2.2.11
+
+
+
+
+
+ javax.xml.bind
+ jaxb-api
+ ${javax.version}
+ provided
+
+
+
diff --git a/java/src/main/java/com/logicalclocks/hsfs/DataFormat.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/DataFormat.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/DataFormat.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/DataFormat.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/Feature.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/Feature.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/Feature.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/Feature.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureType.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/FeatureType.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureType.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureViewBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureViewBase.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/FeatureViewBase.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureViewBase.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnectionBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/HopsworksConnectionBase.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/HopsworksConnectionBase.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/HopsworksConnectionBase.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/HudiOperationType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/HudiOperationType.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/HudiOperationType.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/HudiOperationType.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/JobConfiguration.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/JobConfiguration.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/JobConfiguration.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/JobConfiguration.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/Project.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/Project.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/Project.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/Project.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/SecretStore.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/SecretStore.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/SecretStore.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/SecretStore.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/SecurityProtocol.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/SecurityProtocol.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/SecurityProtocol.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/SecurityProtocol.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/Split.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/Split.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/Split.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/Split.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/SslEndpointIdentificationAlgorithm.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/SslEndpointIdentificationAlgorithm.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/SslEndpointIdentificationAlgorithm.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/SslEndpointIdentificationAlgorithm.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/StatisticsConfig.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/StatisticsConfig.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/StatisticsConfig.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/StatisticsConfig.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/Storage.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/Storage.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/Storage.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/Storage.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnector.java
similarity index 96%
rename from java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnector.java
index 91c6b42154..9eeb0c82eb 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java
+++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnector.java
@@ -23,9 +23,8 @@
import com.google.common.base.Strings;
import com.logicalclocks.hsfs.metadata.Option;
import com.logicalclocks.hsfs.metadata.StorageConnectorApi;
-
import com.logicalclocks.hsfs.util.Constants;
-import com.logicalclocks.hsfs.spark.engine.SparkEngine;
+
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -33,10 +32,7 @@
import lombok.ToString;
import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.time.Instant;
-import java.util.Base64;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -491,17 +487,11 @@ public static class BigqueryConnector extends StorageConnector {
/**
* Set spark options specific to BigQuery.
* @return Map
- * @throws IOException IOException
*/
@Override
- public Map sparkOptions() throws IOException {
+ public Map sparkOptions() {
Map options = new HashMap<>();
- // Base64 encode the credentials file
- String localKeyPath = SparkEngine.getInstance().addFile(keyPath);
- byte[] fileContent = Files.readAllBytes(Paths.get(localKeyPath));
- options.put(Constants.BIGQ_CREDENTIALS, Base64.getEncoder().encodeToString(fileContent));
-
options.put(Constants.BIGQ_PARENT_PROJECT, parentProject);
if (!Strings.isNullOrEmpty(materializationDataset)) {
options.put(Constants.BIGQ_MATERIAL_DATASET, materializationDataset);
diff --git a/java/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/TimeTravelFormat.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/TimeTravelFormat.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/TimeTravelFormat.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/TimeTravelFormat.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetBase.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetBase.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetBase.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetFeature.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetFeature.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetFeature.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetFeature.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/TransformationFunction.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/TransformationFunction.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/TransformationFunction.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/TransformationFunction.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/FeatureGroupAlias.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FeatureGroupAlias.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/FeatureGroupAlias.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FeatureGroupAlias.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/Filter.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/Filter.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/Filter.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/Filter.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/FilterLogic.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FilterLogic.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/FilterLogic.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FilterLogic.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/FsQueryBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FsQueryBase.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/FsQueryBase.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FsQueryBase.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/Join.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/Join.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/Join.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/Join.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/JoinType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/JoinType.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/JoinType.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/JoinType.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/PreparedStatementParameter.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/PreparedStatementParameter.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/PreparedStatementParameter.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/PreparedStatementParameter.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/QueryBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/QueryBase.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/QueryBase.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/QueryBase.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/ServingPreparedStatement.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/ServingPreparedStatement.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/ServingPreparedStatement.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/ServingPreparedStatement.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterCondition.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterCondition.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterCondition.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterCondition.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterLogic.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterLogic.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterLogic.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterLogic.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/CodeEngine.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/CodeEngine.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/engine/CodeEngine.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/CodeEngine.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java
index a2b0ee73c4..bbfeb738e5 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java
+++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java
@@ -18,9 +18,9 @@
package com.logicalclocks.hsfs.engine;
import com.logicalclocks.hsfs.metadata.FeatureGroupApi;
-import com.logicalclocks.hsfs.FeatureGroupBase;
import com.logicalclocks.hsfs.metadata.HopsworksClient;
import com.logicalclocks.hsfs.metadata.TagsApi;
+import com.logicalclocks.hsfs.FeatureGroupBase;
import com.logicalclocks.hsfs.EntityEndpointType;
import com.logicalclocks.hsfs.Feature;
import com.logicalclocks.hsfs.FeatureStoreException;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java
index 1aa89eb856..aa5b7eaaa3 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java
+++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java
@@ -17,16 +17,16 @@
package com.logicalclocks.hsfs.engine;
-import com.logicalclocks.hsfs.Feature;
-import com.logicalclocks.hsfs.FeatureStoreBase;
import com.logicalclocks.hsfs.metadata.FeatureGroupApi;
-import com.logicalclocks.hsfs.FeatureGroupBase;
import com.logicalclocks.hsfs.metadata.HopsworksClient;
import com.logicalclocks.hsfs.metadata.KafkaApi;
+import com.logicalclocks.hsfs.metadata.Subject;
+import com.logicalclocks.hsfs.Feature;
+import com.logicalclocks.hsfs.FeatureStoreBase;
+import com.logicalclocks.hsfs.FeatureGroupBase;
import com.logicalclocks.hsfs.FeatureGroupCommit;
import com.logicalclocks.hsfs.FeatureStoreException;
-import com.logicalclocks.hsfs.metadata.Subject;
import lombok.SneakyThrows;
import org.apache.avro.Schema;
import org.apache.avro.SchemaBuilder;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java
index 175a9fb053..2780f6eeee 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java
+++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java
@@ -20,15 +20,15 @@
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.logicalclocks.hsfs.constructor.Join;
+import com.logicalclocks.hsfs.constructor.QueryBase;
+import com.logicalclocks.hsfs.metadata.FeatureViewApi;
+import com.logicalclocks.hsfs.metadata.TagsApi;
import com.logicalclocks.hsfs.Feature;
import com.logicalclocks.hsfs.FeatureGroupBaseForApi;
import com.logicalclocks.hsfs.Split;
import com.logicalclocks.hsfs.TrainingDatasetBase;
-import com.logicalclocks.hsfs.constructor.Join;
-import com.logicalclocks.hsfs.constructor.QueryBase;
import com.logicalclocks.hsfs.FeatureGroupBase;
-import com.logicalclocks.hsfs.metadata.FeatureViewApi;
-import com.logicalclocks.hsfs.metadata.TagsApi;
import com.logicalclocks.hsfs.EntityEndpointType;
import com.logicalclocks.hsfs.FeatureStoreBase;
import com.logicalclocks.hsfs.FeatureStoreException;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/VectorServer.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/VectorServer.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/engine/VectorServer.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/VectorServer.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Code.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Code.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Code.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Code.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/CodeApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/CodeApi.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/CodeApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/CodeApi.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureViewApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureViewApi.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureViewApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureViewApi.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/KafkaApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/KafkaApi.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/KafkaApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/KafkaApi.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/KafkaClusterInfo.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/KafkaClusterInfo.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/KafkaClusterInfo.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/KafkaClusterInfo.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/OnDemandOptions.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/OnDemandOptions.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/OnDemandOptions.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/OnDemandOptions.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Option.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Option.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Option.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Option.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/PartitionDetails.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/PartitionDetails.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/PartitionDetails.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/PartitionDetails.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java
index 2486ed72a3..95c4035836 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java
+++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java
@@ -18,10 +18,10 @@
package com.logicalclocks.hsfs.metadata;
import com.damnhandy.uri.template.UriTemplate;
-import com.logicalclocks.hsfs.FeatureGroupBase;
import com.logicalclocks.hsfs.constructor.FeatureGroupAlias;
import com.logicalclocks.hsfs.constructor.FsQueryBase;
import com.logicalclocks.hsfs.constructor.QueryBase;
+import com.logicalclocks.hsfs.FeatureGroupBase;
import com.logicalclocks.hsfs.FeatureStoreBase;
import com.logicalclocks.hsfs.FeatureStoreException;
import org.apache.http.HttpHeaders;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/SplitStatistics.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/SplitStatistics.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/SplitStatistics.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/SplitStatistics.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Statistics.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Statistics.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Statistics.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Statistics.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java
similarity index 93%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java
index 874c55fb75..a2725e4f7c 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java
+++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java
@@ -31,9 +31,6 @@
import java.io.IOException;
-import static com.logicalclocks.hsfs.metadata.HopsworksClient.PROJECT_PATH;
-import static com.logicalclocks.hsfs.metadata.HopsworksClient.getInstance;
-
public class StatisticsApi {
private static final String ENTITY_ROOT_PATH = "{/entityType}";
@@ -65,7 +62,7 @@ public Statistics post(TrainingDatasetBase trainingDatasetBase, Statistics stati
private Statistics post(Integer projectId, Integer featurestoreId, Integer entityId, Statistics statistics)
throws FeatureStoreException, IOException {
- String pathTemplate = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + STATISTICS_PATH;
+ String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + STATISTICS_PATH;
String uri = UriTemplate.fromTemplate(pathTemplate)
.set("projectId", projectId)
@@ -79,7 +76,7 @@ private Statistics post(Integer projectId, Integer featurestoreId, Integer entit
public Statistics post(FeatureViewBase featureViewBase,
Integer trainingDataVersion, Statistics statistics)
throws FeatureStoreException, IOException {
- String pathTemplate = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + FV_STATISTICS_PATH;
+ String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + FV_STATISTICS_PATH;
String uri = UriTemplate.fromTemplate(pathTemplate)
.set("projectId", featureViewBase.getFeatureStore().getProjectId())
@@ -92,7 +89,7 @@ public Statistics post(FeatureViewBase featureViewBase,
}
private Statistics post(String uri, Statistics statistics) throws FeatureStoreException, IOException {
- HopsworksClient hopsworksClient = getInstance();
+ HopsworksClient hopsworksClient = HopsworksClient.getInstance();
LOGGER.info("Sending metadata request: " + uri);
HttpPost postRequest = new HttpPost(uri);
postRequest.setEntity(hopsworksClient.buildStringEntity(statistics));
@@ -114,7 +111,7 @@ public Statistics get(TrainingDatasetBase trainingDatasetBase, String commitTime
private Statistics get(Integer projectId, Integer featurestoreId, Integer entityId, String commitTime)
throws FeatureStoreException, IOException {
HopsworksClient hopsworksClient = HopsworksClient.getInstance();
- String pathTemplate = PROJECT_PATH
+ String pathTemplate = HopsworksClient.PROJECT_PATH
+ FeatureStoreApi.FEATURE_STORE_PATH
+ STATISTICS_PATH;
@@ -152,7 +149,7 @@ public Statistics getLast(TrainingDatasetBase trainingDatasetBase) throws Featur
private Statistics getLast(Integer projectId, Integer featurestoreId, Integer entityId)
throws FeatureStoreException, IOException {
HopsworksClient hopsworksClient = HopsworksClient.getInstance();
- String pathTemplate = PROJECT_PATH
+ String pathTemplate = HopsworksClient.PROJECT_PATH
+ FeatureStoreApi.FEATURE_STORE_PATH
+ STATISTICS_PATH;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Subject.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Subject.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Subject.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Subject.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java
similarity index 96%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java
index 6c6b744c94..c6f5cb7470 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java
+++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java
@@ -38,16 +38,14 @@
import java.util.Map;
import java.util.Optional;
-import static com.logicalclocks.hsfs.metadata.HopsworksClient.PROJECT_PATH;
-
public class TagsApi {
public static final String ENTITY_ROOT_PATH = "{/entityType}";
public static final String ENTITY_ID_PATH = ENTITY_ROOT_PATH + "{/entityId}";
public static final String TAGS_PATH = ENTITY_ID_PATH + "/tags{/name}{?value}";
- public static final String FV_TAGS_PATH = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH
+ public static final String FV_TAGS_PATH = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH
+ "/featureview{/fvName}/version{/fvVersion}/tags{/name}";
- public static final String FV_TD_TAGS_PATH = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH
+ public static final String FV_TD_TAGS_PATH = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH
+ "/featureview{/fvName}/version{/fvVersion}"
+ "/trainingdatasets/version{/tdVersion}/tags{/name}";
@@ -62,7 +60,7 @@ public TagsApi(@NonNull EntityEndpointType entityType) {
private void add(Integer projectId, Integer featurestoreId, Integer entityId, String name, Object value)
throws FeatureStoreException, IOException {
- String pathTemplate = PROJECT_PATH
+ String pathTemplate = HopsworksClient.PROJECT_PATH
+ FeatureStoreApi.FEATURE_STORE_PATH
+ TAGS_PATH;
@@ -147,7 +145,7 @@ private UriTemplate getFvTdUriTemplate(FeatureViewBase featureViewBase, Integer
private Map get(Integer projectId, Integer featurestoreId, Integer entityId, Optional name)
throws FeatureStoreException, IOException {
- String pathTemplate = PROJECT_PATH
+ String pathTemplate = HopsworksClient.PROJECT_PATH
+ FeatureStoreApi.FEATURE_STORE_PATH
+ TAGS_PATH;
@@ -238,7 +236,7 @@ public Object parseTagValue(ObjectMapper objectMapper, Object value) throws IOEx
private void deleteTag(Integer projectId, Integer featurestoreId, Integer entityId, String name)
throws FeatureStoreException, IOException {
- String pathTemplate = PROJECT_PATH
+ String pathTemplate = HopsworksClient.PROJECT_PATH
+ FeatureStoreApi.FEATURE_STORE_PATH
+ TAGS_PATH;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/TransformationFunctionAttached.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TransformationFunctionAttached.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/TransformationFunctionAttached.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TransformationFunctionAttached.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/User.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/User.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/metadata/User.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/User.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/util/Constants.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/util/Constants.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/util/Constants.java
rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/util/Constants.java
diff --git a/java/src/test/java/com/logicalclocks/TestHopsworksExternalClient.java b/java/hsfs/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java
similarity index 97%
rename from java/src/test/java/com/logicalclocks/TestHopsworksExternalClient.java
rename to java/hsfs/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java
index d330a9abb4..379f1a0459 100644
--- a/java/src/test/java/com/logicalclocks/TestHopsworksExternalClient.java
+++ b/java/hsfs/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java
@@ -14,10 +14,9 @@
* See the License for the specific language governing permissions and limitations under the License.
*
*/
-package com.logicalclocks;
+package com.logicalclocks.hsfs;
import com.logicalclocks.hsfs.metadata.HopsworksExternalClient;
-import com.logicalclocks.hsfs.FeatureStoreException;
import com.logicalclocks.hsfs.metadata.Credentials;
import io.specto.hoverfly.junit.core.SimulationSource;
import io.specto.hoverfly.junit.dsl.HttpBodyConverter;
diff --git a/java/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java b/java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java
similarity index 97%
rename from java/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java
rename to java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java
index 266bffae77..6d27912fa2 100644
--- a/java/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java
+++ b/java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java
@@ -42,7 +42,7 @@ public void testStringEntitySerialization() throws IOException {
Assertions.assertEquals("Content-Type: application/json; charset=UTF-8",
stringEntity.getContentType().toString());
- String json = IOUtils.toString(stringEntity.getContent(), StandardCharsets.UTF_8);
+ String json = IOUtils.toString(stringEntity.getContent(), String.valueOf(StandardCharsets.UTF_8));
Assertions.assertEquals("{\"email\":\"test@test.com\",\"firstName\":\"test\",\"lastName\":\"de la Rúa Martínez\"}",
json);
}
diff --git a/java/src/test/java/com/logicalclocks/hsfs/metadata/TestTagsApi.java b/java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestTagsApi.java
similarity index 100%
rename from java/src/test/java/com/logicalclocks/hsfs/metadata/TestTagsApi.java
rename to java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestTagsApi.java
diff --git a/java/pom.xml b/java/pom.xml
index 3ec65ee63a..5b2083c88e 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -5,8 +5,13 @@
4.0.0
com.logicalclocks
- hsfs
+ hsfs-sdk
+ pom
3.2.0-SNAPSHOT
+
+ hsfs
+ spark
+
1.8
@@ -19,7 +24,6 @@
2.1.8
1.18.10
2.10.0
- 3.1.1.3
1.1.0.6-SNAPSHOT
0.10.0.3
2.10.40
@@ -31,6 +35,7 @@
5.9.1
2.22.0
4.3.1
+ 1.11.1
UTF-8
${project.basedir}/delombok
@@ -43,73 +48,6 @@
${lombok.version}
-
- com.logicalclocks
- deequ_${scala-short.version}
- ${deequ.version}
-
-
- org.apache.spark
- *
-
-
- org.scala-lang
- *
-
-
-
-
-
- org.apache.spark
- spark-core_${scala-short.version}
- ${spark.version}
- provided
-
-
- com.fasterxml.jackson.core
- jackson-core
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
-
-
-
- org.apache.spark
- spark-sql_${scala-short.version}
- ${spark.version}
- provided
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
- com.fasterxml.jackson.core
- jackson-core
-
-
-
-
-
- org.apache.spark
- spark-avro_${scala-short.version}
- ${spark.version}
- provided
-
-
- com.fasterxml.jackson.core
- jackson-core
-
-
- com.fasterxml.jackson.core
- jackson-databind
-
-
-
-
com.fasterxml.jackson.core
jackson-databind
@@ -208,41 +146,6 @@
provided
-
-
- io.hops.hudi
- hudi-spark3-bundle_${scala-short.version}
- ${hudi.version}
- provided
-
-
- org.apache.httpcomponents
- *
-
-
- com.fasterxml.jackson.core
- *
-
-
-
-
-
- io.hops.hudi
- hudi-utilities-bundle_${scala-short.version}
- ${hudi.version}
- provided
-
-
- org.apache.httpcomponents
- *
-
-
- com.fasterxml.jackson.core
- *
-
-
-
-
org.json
json
@@ -270,19 +173,33 @@
test
-
- org.apache.spark
- spark-hive_${scala-short.version}
- ${spark.version}
- test
-
-
org.mockito
mockito-core
${mockito.version}
test
+
+
+
+ org.apache.avro
+ avro
+ ${avro.version}
+
+
+
+
+ org.scala-lang
+ scala-library
+ ${scala.version}
+
+
+
+
+ org.apache.commons
+ commons-io
+ 1.3.2
+
@@ -389,26 +306,29 @@
-
org.apache.maven.plugins
maven-javadoc-plugin
- 3.1.1
+ 3.5.0
- 1.8
- 1.8
- ${delombok.output}
+
**/MainClass.java
+
+ **/beam/constructor/*
+ **/flink/constructor/*
-
- attach-javadocs
+ aggregate
- jar
+ aggregate
+ site
+
+
+
diff --git a/java/spark/pom.xml b/java/spark/pom.xml
new file mode 100644
index 0000000000..d6b385d07c
--- /dev/null
+++ b/java/spark/pom.xml
@@ -0,0 +1,158 @@
+
+
+
+
+
+ hsfs-sdk
+ com.logicalclocks
+ 3.2.0-SNAPSHOT
+
+ 4.0.0
+
+ hsfs-spark
+
+
+ 3.1.1.3
+
+
+
+
+ com.logicalclocks
+ hsfs
+ ${project.version}
+ compile
+
+
+ javax.xml.bind
+ jaxb-api
+
+
+
+
+
+ com.logicalclocks
+ deequ_${scala-short.version}
+ ${deequ.version}
+
+
+ org.apache.spark
+ *
+
+
+ org.scala-lang
+ *
+
+
+
+
+
+ org.apache.spark
+ spark-core_${scala-short.version}
+ ${spark.version}
+ provided
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+
+
+
+ org.apache.spark
+ spark-sql_${scala-short.version}
+ ${spark.version}
+ provided
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+
+
+
+
+
+ org.apache.spark
+ spark-avro_${scala-short.version}
+ ${spark.version}
+ provided
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
+
+
+
+
+
+ io.hops.hudi
+ hudi-spark3-bundle_${scala-short.version}
+ ${hudi.version}
+ provided
+
+
+ org.apache.httpcomponents
+ *
+
+
+ com.fasterxml.jackson.core
+ *
+
+
+
+
+
+ io.hops.hudi
+ hudi-utilities-bundle_${scala-short.version}
+ ${hudi.version}
+ provided
+
+
+ org.apache.httpcomponents
+ *
+
+
+ com.fasterxml.jackson.core
+ *
+
+
+
+
+
+ org.apache.spark
+ spark-hive_${scala-short.version}
+ ${spark.version}
+ test
+
+
+
diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java
rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java
index 6be008ab89..63d9db749f 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java
+++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java
@@ -18,6 +18,9 @@
package com.logicalclocks.hsfs.spark;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.logicalclocks.hsfs.spark.constructor.Query;
+import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine;
+import com.logicalclocks.hsfs.spark.engine.StatisticsEngine;
import com.logicalclocks.hsfs.EntityEndpointType;
import com.logicalclocks.hsfs.ExternalDataFormat;
import com.logicalclocks.hsfs.Feature;
@@ -33,9 +36,6 @@
import com.logicalclocks.hsfs.FeatureGroupBase;
import com.logicalclocks.hsfs.metadata.OnDemandOptions;
import com.logicalclocks.hsfs.metadata.Statistics;
-import com.logicalclocks.hsfs.spark.constructor.Query;
-import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine;
-import com.logicalclocks.hsfs.spark.engine.StatisticsEngine;
import lombok.AllArgsConstructor;
import lombok.Builder;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java
rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java
index f36c29a516..c4750e4a58 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java
+++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java
@@ -19,6 +19,9 @@
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.logicalclocks.hsfs.spark.constructor.Query;
+import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine;
+import com.logicalclocks.hsfs.spark.engine.StatisticsEngine;
import com.logicalclocks.hsfs.EntityEndpointType;
import com.logicalclocks.hsfs.Feature;
@@ -31,9 +34,6 @@
import com.logicalclocks.hsfs.engine.CodeEngine;
import com.logicalclocks.hsfs.FeatureGroupBase;
import com.logicalclocks.hsfs.metadata.Statistics;
-import com.logicalclocks.hsfs.spark.constructor.Query;
-import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine;
-import com.logicalclocks.hsfs.spark.engine.StatisticsEngine;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.NonNull;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java
similarity index 98%
rename from java/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java
rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java
index 3fdd43f06b..9c56c09060 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java
+++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java
@@ -17,6 +17,10 @@
package com.logicalclocks.hsfs.spark;
+import com.logicalclocks.hsfs.spark.constructor.Query;
+import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine;
+import com.logicalclocks.hsfs.spark.engine.FeatureViewEngine;
+import com.logicalclocks.hsfs.spark.engine.SparkEngine;
import com.logicalclocks.hsfs.FeatureStoreBase;
import com.logicalclocks.hsfs.FeatureStoreException;
import com.logicalclocks.hsfs.StatisticsConfig;
@@ -27,11 +31,6 @@
import com.logicalclocks.hsfs.metadata.StorageConnectorApi;
import com.logicalclocks.hsfs.metadata.TrainingDatasetApi;
-import com.logicalclocks.hsfs.spark.constructor.Query;
-import com.logicalclocks.hsfs.spark.engine.FeatureViewEngine;
-import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine;
-import com.logicalclocks.hsfs.spark.engine.SparkEngine;
-
import lombok.NonNull;
import org.apache.spark.sql.Dataset;
@@ -102,8 +101,8 @@ public FeatureGroup getFeatureGroup(@NonNull String name, @NonNull Integer versi
*/
public FeatureGroup getFeatureGroup(String name) throws FeatureStoreException, IOException {
LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `"
- + DEFAULT_VERSION + "`.");
- return getFeatureGroup(name, DEFAULT_VERSION);
+ + FeatureStoreBase.DEFAULT_VERSION + "`.");
+ return getFeatureGroup(name, FeatureStoreBase.DEFAULT_VERSION);
}
/**
@@ -326,8 +325,8 @@ public FeatureGroup getOrCreateFeatureGroup(String name, Integer version, String
@Override
public StreamFeatureGroup getStreamFeatureGroup(String name) throws FeatureStoreException, IOException {
LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `"
- + DEFAULT_VERSION + "`.");
- return getStreamFeatureGroup(name, DEFAULT_VERSION);
+ + FeatureStoreBase.DEFAULT_VERSION + "`.");
+ return getStreamFeatureGroup(name, FeatureStoreBase.DEFAULT_VERSION);
}
/**
@@ -625,8 +624,8 @@ public ExternalFeatureGroup getExternalFeatureGroup(@NonNull String name, @NonNu
@Override
public ExternalFeatureGroup getExternalFeatureGroup(String name) throws FeatureStoreException, IOException {
LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `"
- + DEFAULT_VERSION + "`.");
- return getExternalFeatureGroup(name, DEFAULT_VERSION);
+ + FeatureStoreBase.DEFAULT_VERSION + "`.");
+ return getExternalFeatureGroup(name, FeatureStoreBase.DEFAULT_VERSION);
}
/**
@@ -881,8 +880,8 @@ public ExternalFeatureGroup getOnDemandFeatureGroup(@NonNull String name, @NonNu
@Deprecated
public ExternalFeatureGroup getOnDemandFeatureGroup(String name) throws FeatureStoreException, IOException {
LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `"
- + DEFAULT_VERSION + "`.");
- return getExternalFeatureGroup(name, DEFAULT_VERSION);
+ + FeatureStoreBase.DEFAULT_VERSION + "`.");
+ return getExternalFeatureGroup(name, FeatureStoreBase.DEFAULT_VERSION);
}
@Deprecated
@@ -992,8 +991,8 @@ public FeatureView getFeatureView(@NonNull String name, @NonNull Integer version
@Override
public FeatureView getFeatureView(String name) throws FeatureStoreException, IOException {
LOGGER.info("VersionWarning: No version provided for getting feature view `" + name + "`, defaulting to `"
- + DEFAULT_VERSION + "`.");
- return getFeatureView(name, DEFAULT_VERSION);
+ + FeatureStoreBase.DEFAULT_VERSION + "`.");
+ return getFeatureView(name, FeatureStoreBase.DEFAULT_VERSION);
}
@Override
@@ -1016,8 +1015,8 @@ public TrainingDataset getTrainingDataset(@NonNull String name, @NonNull Integer
@Deprecated
public TrainingDataset getTrainingDataset(String name) throws FeatureStoreException, IOException {
LOGGER.info("VersionWarning: No version provided for getting training dataset `" + name + "`, defaulting to `"
- + DEFAULT_VERSION + "`.");
- return getTrainingDataset(name, DEFAULT_VERSION);
+ + FeatureStoreBase.DEFAULT_VERSION + "`.");
+ return getTrainingDataset(name, FeatureStoreBase.DEFAULT_VERSION);
}
@Deprecated
diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java
similarity index 99%
rename from java/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java
rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java
index 3e9f8c7aba..b7f0d1043e 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java
+++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java
@@ -20,6 +20,8 @@
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.logicalclocks.hsfs.spark.constructor.Query;
+import com.logicalclocks.hsfs.spark.engine.FeatureViewEngine;
import com.logicalclocks.hsfs.DataFormat;
import com.logicalclocks.hsfs.FeatureStoreException;
import com.logicalclocks.hsfs.FeatureViewBase;
@@ -30,8 +32,6 @@
import com.logicalclocks.hsfs.constructor.Filter;
import com.logicalclocks.hsfs.constructor.FilterLogic;
import com.logicalclocks.hsfs.engine.FeatureGroupUtils;
-import com.logicalclocks.hsfs.spark.constructor.Query;
-import com.logicalclocks.hsfs.spark.engine.FeatureViewEngine;
import lombok.NoArgsConstructor;
import lombok.NonNull;
@@ -90,7 +90,7 @@ public FeatureViewBuilder description(String description) {
public FeatureViewBuilder query(Query query) {
this.query = query;
if (query.isTimeTravel()) {
- LOGGER.info("`as_of` argument in the `Query` will be ignored because "
+ FeatureViewBase.LOGGER.info("`as_of` argument in the `Query` will be ignored because "
+ "feature view does not support time travel query.");
}
return this;
@@ -137,7 +137,7 @@ public FeatureView(@NonNull String name, Integer version, @NonNull Query query,
*/
@Override
public void delete() throws FeatureStoreException, IOException {
- LOGGER.warn("JobWarning: All jobs associated to feature view `" + name + "`, version `"
+ FeatureViewBase.LOGGER.warn("JobWarning: All jobs associated to feature view `" + name + "`, version `"
+ version + "` will be removed.");
featureViewEngine.delete(this.featureStore, this.name, this.version);
}
diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java
rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java
index 674dba1982..e3b63a427f 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java
+++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java
@@ -17,6 +17,7 @@
package com.logicalclocks.hsfs.spark;
+import com.logicalclocks.hsfs.spark.engine.SparkEngine;
import com.logicalclocks.hsfs.FeatureStoreException;
import com.logicalclocks.hsfs.HopsworksConnectionBase;
import com.logicalclocks.hsfs.SecretStore;
@@ -24,7 +25,6 @@
import com.logicalclocks.hsfs.metadata.HopsworksHttpClient;
import com.logicalclocks.hsfs.metadata.HopsworksInternalClient;
-import com.logicalclocks.hsfs.spark.engine.SparkEngine;
import lombok.Builder;
import software.amazon.awssdk.regions.Region;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java
rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java
index c4503d33f9..c9ff950a37 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java
+++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java
@@ -17,10 +17,10 @@
package com.logicalclocks.hsfs.spark;
+import com.logicalclocks.hsfs.spark.constructor.Query;
+import com.logicalclocks.hsfs.spark.engine.SparkEngine;
import com.logicalclocks.hsfs.DataFormat;
import com.logicalclocks.hsfs.Split;
-import com.logicalclocks.hsfs.spark.engine.SparkEngine;
-import com.logicalclocks.hsfs.spark.constructor.Query;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java
similarity index 100%
rename from java/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java
rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java
index 06a5e3e4af..2008bb1856 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java
+++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java
@@ -19,6 +19,9 @@
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.logicalclocks.hsfs.spark.constructor.Query;
+import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine;
+import com.logicalclocks.hsfs.spark.engine.StatisticsEngine;
import com.logicalclocks.hsfs.EntityEndpointType;
import com.logicalclocks.hsfs.Feature;
import com.logicalclocks.hsfs.FeatureStoreException;
@@ -29,9 +32,6 @@
import com.logicalclocks.hsfs.engine.CodeEngine;
import com.logicalclocks.hsfs.FeatureGroupBase;
import com.logicalclocks.hsfs.metadata.Statistics;
-import com.logicalclocks.hsfs.spark.constructor.Query;
-import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine;
-import com.logicalclocks.hsfs.spark.engine.StatisticsEngine;
import lombok.AllArgsConstructor;
import lombok.Builder;
diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java
similarity index 99%
rename from java/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java
rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java
index 4758b8eb57..e269c60425 100644
--- a/java/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java
+++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java
@@ -21,6 +21,9 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
+import com.logicalclocks.hsfs.spark.constructor.Query;
+import com.logicalclocks.hsfs.spark.engine.StatisticsEngine;
+import com.logicalclocks.hsfs.spark.engine.TrainingDatasetEngine;
import com.logicalclocks.hsfs.DataFormat;
import com.logicalclocks.hsfs.EntityEndpointType;
import com.logicalclocks.hsfs.FeatureStoreException;
@@ -36,9 +39,6 @@
import com.logicalclocks.hsfs.engine.CodeEngine;
import com.logicalclocks.hsfs.engine.FeatureGroupUtils;
import com.logicalclocks.hsfs.metadata.Statistics;
-import com.logicalclocks.hsfs.spark.constructor.Query;
-import com.logicalclocks.hsfs.spark.engine.StatisticsEngine;
-import com.logicalclocks.hsfs.spark.engine.TrainingDatasetEngine;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
@@ -339,7 +339,7 @@ public List