diff --git a/.github/workflows/mkdocs-master.yml b/.github/workflows/mkdocs-master.yml index 10ad5ad6f2..0ea51b9216 100644 --- a/.github/workflows/mkdocs-master.yml +++ b/.github/workflows/mkdocs-master.yml @@ -35,9 +35,15 @@ jobs: restore-keys: | ${{ runner.os }}-maven- + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'adopt' + - name: Build java doc documentation working-directory: ./java - run: mvn clean package javadoc:javadoc -DskipTests && cp -r target/apidocs ../docs/javadoc + run: mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc - name: setup git run: | diff --git a/.github/workflows/mkdocs-release.yml b/.github/workflows/mkdocs-release.yml index 1fdc28288c..f80e9ad2f4 100644 --- a/.github/workflows/mkdocs-release.yml +++ b/.github/workflows/mkdocs-release.yml @@ -42,9 +42,15 @@ jobs: restore-keys: | ${{ runner.os }}-maven- + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'adopt' + - name: Build java doc documentation working-directory: ./java - run: mvn clean package javadoc:javadoc -DskipTests && cp -r target/apidocs ../docs/javadoc + run: mvn clean install javadoc:javadoc javadoc:aggregate -DskipTests && cp -r target/site/apidocs ../docs/javadoc - name: setup git run: | diff --git a/java/hsfs/pom.xml b/java/hsfs/pom.xml new file mode 100644 index 0000000000..319f4a6794 --- /dev/null +++ b/java/hsfs/pom.xml @@ -0,0 +1,28 @@ + + + + hsfs-sdk + com.logicalclocks + 3.2.0-SNAPSHOT + + 4.0.0 + + hsfs + + + + 2.2.11 + + + + + + javax.xml.bind + jaxb-api + ${javax.version} + provided + + + diff --git a/java/src/main/java/com/logicalclocks/hsfs/DataFormat.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/DataFormat.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/DataFormat.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/DataFormat.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/DeltaStreamerJobConf.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/EntityEndpointType.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/ExternalDataFormat.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/Feature.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/Feature.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/Feature.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/Feature.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBase.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupBaseForApi.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureGroupCommit.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreBase.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureStoreException.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureType.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/FeatureType.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureType.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureViewBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureViewBase.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/FeatureViewBase.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/FeatureViewBase.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnectionBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/HopsworksConnectionBase.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/HopsworksConnectionBase.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/HopsworksConnectionBase.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/HudiOperationType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/HudiOperationType.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/HudiOperationType.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/HudiOperationType.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/JobConfiguration.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/JobConfiguration.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/JobConfiguration.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/JobConfiguration.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/Project.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/Project.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/Project.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/Project.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/SecretStore.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/SecretStore.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/SecretStore.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/SecretStore.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/SecurityProtocol.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/SecurityProtocol.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/SecurityProtocol.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/SecurityProtocol.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/Split.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/Split.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/Split.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/Split.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/SslEndpointIdentificationAlgorithm.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/SslEndpointIdentificationAlgorithm.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/SslEndpointIdentificationAlgorithm.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/SslEndpointIdentificationAlgorithm.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/StatisticsConfig.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/StatisticsConfig.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/StatisticsConfig.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/StatisticsConfig.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/Storage.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/Storage.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/Storage.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/Storage.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnector.java similarity index 96% rename from java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnector.java index 91c6b42154..9eeb0c82eb 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java +++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnector.java @@ -23,9 +23,8 @@ import com.google.common.base.Strings; import com.logicalclocks.hsfs.metadata.Option; import com.logicalclocks.hsfs.metadata.StorageConnectorApi; - import com.logicalclocks.hsfs.util.Constants; -import com.logicalclocks.hsfs.spark.engine.SparkEngine; + import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; @@ -33,10 +32,7 @@ import lombok.ToString; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.time.Instant; -import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -491,17 +487,11 @@ public static class BigqueryConnector extends StorageConnector { /** * Set spark options specific to BigQuery. * @return Map - * @throws IOException IOException */ @Override - public Map sparkOptions() throws IOException { + public Map sparkOptions() { Map options = new HashMap<>(); - // Base64 encode the credentials file - String localKeyPath = SparkEngine.getInstance().addFile(keyPath); - byte[] fileContent = Files.readAllBytes(Paths.get(localKeyPath)); - options.put(Constants.BIGQ_CREDENTIALS, Base64.getEncoder().encodeToString(fileContent)); - options.put(Constants.BIGQ_PARENT_PROJECT, parentProject); if (!Strings.isNullOrEmpty(materializationDataset)) { options.put(Constants.BIGQ_MATERIAL_DATASET, materializationDataset); diff --git a/java/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/StorageConnectorType.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/TimeTravelFormat.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/TimeTravelFormat.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/TimeTravelFormat.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/TimeTravelFormat.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetBase.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetBase.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetBase.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetFeature.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetFeature.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetFeature.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetFeature.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/TrainingDatasetType.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/TransformationFunction.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/TransformationFunction.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/TransformationFunction.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/TransformationFunction.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/FeatureGroupAlias.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FeatureGroupAlias.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/FeatureGroupAlias.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FeatureGroupAlias.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/Filter.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/Filter.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/Filter.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/Filter.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/FilterLogic.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FilterLogic.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/FilterLogic.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FilterLogic.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/FsQueryBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FsQueryBase.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/FsQueryBase.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/FsQueryBase.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/Join.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/Join.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/Join.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/Join.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/JoinType.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/JoinType.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/JoinType.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/JoinType.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/PreparedStatementParameter.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/PreparedStatementParameter.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/PreparedStatementParameter.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/PreparedStatementParameter.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/QueryBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/QueryBase.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/QueryBase.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/QueryBase.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/ServingPreparedStatement.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/ServingPreparedStatement.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/ServingPreparedStatement.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/ServingPreparedStatement.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterCondition.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterCondition.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterCondition.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterCondition.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterLogic.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterLogic.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterLogic.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/constructor/SqlFilterLogic.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/CodeEngine.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/CodeEngine.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/engine/CodeEngine.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/CodeEngine.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java index a2b0ee73c4..bbfeb738e5 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java +++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngineBase.java @@ -18,9 +18,9 @@ package com.logicalclocks.hsfs.engine; import com.logicalclocks.hsfs.metadata.FeatureGroupApi; -import com.logicalclocks.hsfs.FeatureGroupBase; import com.logicalclocks.hsfs.metadata.HopsworksClient; import com.logicalclocks.hsfs.metadata.TagsApi; +import com.logicalclocks.hsfs.FeatureGroupBase; import com.logicalclocks.hsfs.EntityEndpointType; import com.logicalclocks.hsfs.Feature; import com.logicalclocks.hsfs.FeatureStoreException; diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java index 1aa89eb856..aa5b7eaaa3 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java +++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupUtils.java @@ -17,16 +17,16 @@ package com.logicalclocks.hsfs.engine; -import com.logicalclocks.hsfs.Feature; -import com.logicalclocks.hsfs.FeatureStoreBase; import com.logicalclocks.hsfs.metadata.FeatureGroupApi; -import com.logicalclocks.hsfs.FeatureGroupBase; import com.logicalclocks.hsfs.metadata.HopsworksClient; import com.logicalclocks.hsfs.metadata.KafkaApi; +import com.logicalclocks.hsfs.metadata.Subject; +import com.logicalclocks.hsfs.Feature; +import com.logicalclocks.hsfs.FeatureStoreBase; +import com.logicalclocks.hsfs.FeatureGroupBase; import com.logicalclocks.hsfs.FeatureGroupCommit; import com.logicalclocks.hsfs.FeatureStoreException; -import com.logicalclocks.hsfs.metadata.Subject; import lombok.SneakyThrows; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java index 175a9fb053..2780f6eeee 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java +++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/FeatureViewEngineBase.java @@ -20,15 +20,15 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.logicalclocks.hsfs.constructor.Join; +import com.logicalclocks.hsfs.constructor.QueryBase; +import com.logicalclocks.hsfs.metadata.FeatureViewApi; +import com.logicalclocks.hsfs.metadata.TagsApi; import com.logicalclocks.hsfs.Feature; import com.logicalclocks.hsfs.FeatureGroupBaseForApi; import com.logicalclocks.hsfs.Split; import com.logicalclocks.hsfs.TrainingDatasetBase; -import com.logicalclocks.hsfs.constructor.Join; -import com.logicalclocks.hsfs.constructor.QueryBase; import com.logicalclocks.hsfs.FeatureGroupBase; -import com.logicalclocks.hsfs.metadata.FeatureViewApi; -import com.logicalclocks.hsfs.metadata.TagsApi; import com.logicalclocks.hsfs.EntityEndpointType; import com.logicalclocks.hsfs.FeatureStoreBase; import com.logicalclocks.hsfs.FeatureStoreException; diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/VectorServer.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/VectorServer.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/engine/VectorServer.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/engine/VectorServer.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/AuthorizationHandler.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Code.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Code.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Code.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Code.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/CodeApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/CodeApi.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/CodeApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/CodeApi.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Credentials.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupApi.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureStoreApi.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureViewApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureViewApi.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureViewApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/FeatureViewApi.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksClient.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHostnameVerifier.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksHttpClient.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksInternalClient.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/InternalException.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/KafkaApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/KafkaApi.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/KafkaApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/KafkaApi.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/KafkaClusterInfo.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/KafkaClusterInfo.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/KafkaClusterInfo.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/KafkaClusterInfo.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/OnDemandOptions.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/OnDemandOptions.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/OnDemandOptions.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/OnDemandOptions.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Option.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Option.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Option.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Option.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/PartitionDetails.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/PartitionDetails.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/PartitionDetails.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/PartitionDetails.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/ProjectApi.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java index 2486ed72a3..95c4035836 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java +++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/QueryConstructorApi.java @@ -18,10 +18,10 @@ package com.logicalclocks.hsfs.metadata; import com.damnhandy.uri.template.UriTemplate; -import com.logicalclocks.hsfs.FeatureGroupBase; import com.logicalclocks.hsfs.constructor.FeatureGroupAlias; import com.logicalclocks.hsfs.constructor.FsQueryBase; import com.logicalclocks.hsfs.constructor.QueryBase; +import com.logicalclocks.hsfs.FeatureGroupBase; import com.logicalclocks.hsfs.FeatureStoreBase; import com.logicalclocks.hsfs.FeatureStoreException; import org.apache.http.HttpHeaders; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/RestDto.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/SplitStatistics.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/SplitStatistics.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/SplitStatistics.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/SplitStatistics.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Statistics.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Statistics.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Statistics.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Statistics.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java similarity index 93% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java index 874c55fb75..a2725e4f7c 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java +++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StatisticsApi.java @@ -31,9 +31,6 @@ import java.io.IOException; -import static com.logicalclocks.hsfs.metadata.HopsworksClient.PROJECT_PATH; -import static com.logicalclocks.hsfs.metadata.HopsworksClient.getInstance; - public class StatisticsApi { private static final String ENTITY_ROOT_PATH = "{/entityType}"; @@ -65,7 +62,7 @@ public Statistics post(TrainingDatasetBase trainingDatasetBase, Statistics stati private Statistics post(Integer projectId, Integer featurestoreId, Integer entityId, Statistics statistics) throws FeatureStoreException, IOException { - String pathTemplate = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + STATISTICS_PATH; + String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + STATISTICS_PATH; String uri = UriTemplate.fromTemplate(pathTemplate) .set("projectId", projectId) @@ -79,7 +76,7 @@ private Statistics post(Integer projectId, Integer featurestoreId, Integer entit public Statistics post(FeatureViewBase featureViewBase, Integer trainingDataVersion, Statistics statistics) throws FeatureStoreException, IOException { - String pathTemplate = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + FV_STATISTICS_PATH; + String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + FV_STATISTICS_PATH; String uri = UriTemplate.fromTemplate(pathTemplate) .set("projectId", featureViewBase.getFeatureStore().getProjectId()) @@ -92,7 +89,7 @@ public Statistics post(FeatureViewBase featureViewBase, } private Statistics post(String uri, Statistics statistics) throws FeatureStoreException, IOException { - HopsworksClient hopsworksClient = getInstance(); + HopsworksClient hopsworksClient = HopsworksClient.getInstance(); LOGGER.info("Sending metadata request: " + uri); HttpPost postRequest = new HttpPost(uri); postRequest.setEntity(hopsworksClient.buildStringEntity(statistics)); @@ -114,7 +111,7 @@ public Statistics get(TrainingDatasetBase trainingDatasetBase, String commitTime private Statistics get(Integer projectId, Integer featurestoreId, Integer entityId, String commitTime) throws FeatureStoreException, IOException { HopsworksClient hopsworksClient = HopsworksClient.getInstance(); - String pathTemplate = PROJECT_PATH + String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + STATISTICS_PATH; @@ -152,7 +149,7 @@ public Statistics getLast(TrainingDatasetBase trainingDatasetBase) throws Featur private Statistics getLast(Integer projectId, Integer featurestoreId, Integer entityId) throws FeatureStoreException, IOException { HopsworksClient hopsworksClient = HopsworksClient.getInstance(); - String pathTemplate = PROJECT_PATH + String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + STATISTICS_PATH; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/StorageConnectorApi.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Subject.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Subject.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Subject.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Subject.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/Tags.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java similarity index 96% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java index 6c6b744c94..c6f5cb7470 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java +++ b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TagsApi.java @@ -38,16 +38,14 @@ import java.util.Map; import java.util.Optional; -import static com.logicalclocks.hsfs.metadata.HopsworksClient.PROJECT_PATH; - public class TagsApi { public static final String ENTITY_ROOT_PATH = "{/entityType}"; public static final String ENTITY_ID_PATH = ENTITY_ROOT_PATH + "{/entityId}"; public static final String TAGS_PATH = ENTITY_ID_PATH + "/tags{/name}{?value}"; - public static final String FV_TAGS_PATH = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + public static final String FV_TAGS_PATH = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + "/featureview{/fvName}/version{/fvVersion}/tags{/name}"; - public static final String FV_TD_TAGS_PATH = PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + public static final String FV_TD_TAGS_PATH = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + "/featureview{/fvName}/version{/fvVersion}" + "/trainingdatasets/version{/tdVersion}/tags{/name}"; @@ -62,7 +60,7 @@ public TagsApi(@NonNull EntityEndpointType entityType) { private void add(Integer projectId, Integer featurestoreId, Integer entityId, String name, Object value) throws FeatureStoreException, IOException { - String pathTemplate = PROJECT_PATH + String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + TAGS_PATH; @@ -147,7 +145,7 @@ private UriTemplate getFvTdUriTemplate(FeatureViewBase featureViewBase, Integer private Map get(Integer projectId, Integer featurestoreId, Integer entityId, Optional name) throws FeatureStoreException, IOException { - String pathTemplate = PROJECT_PATH + String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + TAGS_PATH; @@ -238,7 +236,7 @@ public Object parseTagValue(ObjectMapper objectMapper, Object value) throws IOEx private void deleteTag(Integer projectId, Integer featurestoreId, Integer entityId, String name) throws FeatureStoreException, IOException { - String pathTemplate = PROJECT_PATH + String pathTemplate = HopsworksClient.PROJECT_PATH + FeatureStoreApi.FEATURE_STORE_PATH + TAGS_PATH; diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TrainingDatasetApi.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/TransformationFunctionAttached.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TransformationFunctionAttached.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/TransformationFunctionAttached.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/TransformationFunctionAttached.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/UnauthorizedException.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/User.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/User.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/metadata/User.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/metadata/User.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/util/Constants.java b/java/hsfs/src/main/java/com/logicalclocks/hsfs/util/Constants.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/util/Constants.java rename to java/hsfs/src/main/java/com/logicalclocks/hsfs/util/Constants.java diff --git a/java/src/test/java/com/logicalclocks/TestHopsworksExternalClient.java b/java/hsfs/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java similarity index 97% rename from java/src/test/java/com/logicalclocks/TestHopsworksExternalClient.java rename to java/hsfs/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java index d330a9abb4..379f1a0459 100644 --- a/java/src/test/java/com/logicalclocks/TestHopsworksExternalClient.java +++ b/java/hsfs/src/test/java/com/logicalclocks/hsfs/TestHopsworksExternalClient.java @@ -14,10 +14,9 @@ * See the License for the specific language governing permissions and limitations under the License. * */ -package com.logicalclocks; +package com.logicalclocks.hsfs; import com.logicalclocks.hsfs.metadata.HopsworksExternalClient; -import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.metadata.Credentials; import io.specto.hoverfly.junit.core.SimulationSource; import io.specto.hoverfly.junit.dsl.HttpBodyConverter; diff --git a/java/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java b/java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java similarity index 97% rename from java/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java rename to java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java index 266bffae77..6d27912fa2 100644 --- a/java/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java +++ b/java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestHopsworksClient.java @@ -42,7 +42,7 @@ public void testStringEntitySerialization() throws IOException { Assertions.assertEquals("Content-Type: application/json; charset=UTF-8", stringEntity.getContentType().toString()); - String json = IOUtils.toString(stringEntity.getContent(), StandardCharsets.UTF_8); + String json = IOUtils.toString(stringEntity.getContent(), String.valueOf(StandardCharsets.UTF_8)); Assertions.assertEquals("{\"email\":\"test@test.com\",\"firstName\":\"test\",\"lastName\":\"de la Rúa Martínez\"}", json); } diff --git a/java/src/test/java/com/logicalclocks/hsfs/metadata/TestTagsApi.java b/java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestTagsApi.java similarity index 100% rename from java/src/test/java/com/logicalclocks/hsfs/metadata/TestTagsApi.java rename to java/hsfs/src/test/java/com/logicalclocks/hsfs/metadata/TestTagsApi.java diff --git a/java/pom.xml b/java/pom.xml index 3ec65ee63a..5b2083c88e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -5,8 +5,13 @@ 4.0.0 com.logicalclocks - hsfs + hsfs-sdk + pom 3.2.0-SNAPSHOT + + hsfs + spark + 1.8 @@ -19,7 +24,6 @@ 2.1.8 1.18.10 2.10.0 - 3.1.1.3 1.1.0.6-SNAPSHOT 0.10.0.3 2.10.40 @@ -31,6 +35,7 @@ 5.9.1 2.22.0 4.3.1 + 1.11.1 UTF-8 ${project.basedir}/delombok @@ -43,73 +48,6 @@ ${lombok.version} - - com.logicalclocks - deequ_${scala-short.version} - ${deequ.version} - - - org.apache.spark - * - - - org.scala-lang - * - - - - - - org.apache.spark - spark-core_${scala-short.version} - ${spark.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - - - - org.apache.spark - spark-sql_${scala-short.version} - ${spark.version} - provided - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core - - - - - - org.apache.spark - spark-avro_${scala-short.version} - ${spark.version} - provided - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-databind - - - - com.fasterxml.jackson.core jackson-databind @@ -208,41 +146,6 @@ provided - - - io.hops.hudi - hudi-spark3-bundle_${scala-short.version} - ${hudi.version} - provided - - - org.apache.httpcomponents - * - - - com.fasterxml.jackson.core - * - - - - - - io.hops.hudi - hudi-utilities-bundle_${scala-short.version} - ${hudi.version} - provided - - - org.apache.httpcomponents - * - - - com.fasterxml.jackson.core - * - - - - org.json json @@ -270,19 +173,33 @@ test - - org.apache.spark - spark-hive_${scala-short.version} - ${spark.version} - test - - org.mockito mockito-core ${mockito.version} test + + + + org.apache.avro + avro + ${avro.version} + + + + + org.scala-lang + scala-library + ${scala.version} + + + + + org.apache.commons + commons-io + 1.3.2 + @@ -389,26 +306,29 @@ - org.apache.maven.plugins maven-javadoc-plugin - 3.1.1 + 3.5.0 - 1.8 - 1.8 - ${delombok.output} + **/MainClass.java + + **/beam/constructor/* + **/flink/constructor/* - - attach-javadocs + aggregate - jar + aggregate + site + + + diff --git a/java/spark/pom.xml b/java/spark/pom.xml new file mode 100644 index 0000000000..d6b385d07c --- /dev/null +++ b/java/spark/pom.xml @@ -0,0 +1,158 @@ + + + + + + hsfs-sdk + com.logicalclocks + 3.2.0-SNAPSHOT + + 4.0.0 + + hsfs-spark + + + 3.1.1.3 + + + + + com.logicalclocks + hsfs + ${project.version} + compile + + + javax.xml.bind + jaxb-api + + + + + + com.logicalclocks + deequ_${scala-short.version} + ${deequ.version} + + + org.apache.spark + * + + + org.scala-lang + * + + + + + + org.apache.spark + spark-core_${scala-short.version} + ${spark.version} + provided + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + + + org.apache.spark + spark-sql_${scala-short.version} + ${spark.version} + provided + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + + + + org.apache.spark + spark-avro_${scala-short.version} + ${spark.version} + provided + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + + + + io.hops.hudi + hudi-spark3-bundle_${scala-short.version} + ${hudi.version} + provided + + + org.apache.httpcomponents + * + + + com.fasterxml.jackson.core + * + + + + + + io.hops.hudi + hudi-utilities-bundle_${scala-short.version} + ${hudi.version} + provided + + + org.apache.httpcomponents + * + + + com.fasterxml.jackson.core + * + + + + + + org.apache.spark + spark-hive_${scala-short.version} + ${spark.version} + test + + + diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java index 6be008ab89..63d9db749f 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/ExternalFeatureGroup.java @@ -18,6 +18,9 @@ package com.logicalclocks.hsfs.spark; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.logicalclocks.hsfs.spark.constructor.Query; +import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine; +import com.logicalclocks.hsfs.spark.engine.StatisticsEngine; import com.logicalclocks.hsfs.EntityEndpointType; import com.logicalclocks.hsfs.ExternalDataFormat; import com.logicalclocks.hsfs.Feature; @@ -33,9 +36,6 @@ import com.logicalclocks.hsfs.FeatureGroupBase; import com.logicalclocks.hsfs.metadata.OnDemandOptions; import com.logicalclocks.hsfs.metadata.Statistics; -import com.logicalclocks.hsfs.spark.constructor.Query; -import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine; -import com.logicalclocks.hsfs.spark.engine.StatisticsEngine; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java index f36c29a516..c4750e4a58 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureGroup.java @@ -19,6 +19,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.logicalclocks.hsfs.spark.constructor.Query; +import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine; +import com.logicalclocks.hsfs.spark.engine.StatisticsEngine; import com.logicalclocks.hsfs.EntityEndpointType; import com.logicalclocks.hsfs.Feature; @@ -31,9 +34,6 @@ import com.logicalclocks.hsfs.engine.CodeEngine; import com.logicalclocks.hsfs.FeatureGroupBase; import com.logicalclocks.hsfs.metadata.Statistics; -import com.logicalclocks.hsfs.spark.constructor.Query; -import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine; -import com.logicalclocks.hsfs.spark.engine.StatisticsEngine; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.NonNull; diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java similarity index 98% rename from java/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java index 3fdd43f06b..9c56c09060 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureStore.java @@ -17,6 +17,10 @@ package com.logicalclocks.hsfs.spark; +import com.logicalclocks.hsfs.spark.constructor.Query; +import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine; +import com.logicalclocks.hsfs.spark.engine.FeatureViewEngine; +import com.logicalclocks.hsfs.spark.engine.SparkEngine; import com.logicalclocks.hsfs.FeatureStoreBase; import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.StatisticsConfig; @@ -27,11 +31,6 @@ import com.logicalclocks.hsfs.metadata.StorageConnectorApi; import com.logicalclocks.hsfs.metadata.TrainingDatasetApi; -import com.logicalclocks.hsfs.spark.constructor.Query; -import com.logicalclocks.hsfs.spark.engine.FeatureViewEngine; -import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine; -import com.logicalclocks.hsfs.spark.engine.SparkEngine; - import lombok.NonNull; import org.apache.spark.sql.Dataset; @@ -102,8 +101,8 @@ public FeatureGroup getFeatureGroup(@NonNull String name, @NonNull Integer versi */ public FeatureGroup getFeatureGroup(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" - + DEFAULT_VERSION + "`."); - return getFeatureGroup(name, DEFAULT_VERSION); + + FeatureStoreBase.DEFAULT_VERSION + "`."); + return getFeatureGroup(name, FeatureStoreBase.DEFAULT_VERSION); } /** @@ -326,8 +325,8 @@ public FeatureGroup getOrCreateFeatureGroup(String name, Integer version, String @Override public StreamFeatureGroup getStreamFeatureGroup(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" - + DEFAULT_VERSION + "`."); - return getStreamFeatureGroup(name, DEFAULT_VERSION); + + FeatureStoreBase.DEFAULT_VERSION + "`."); + return getStreamFeatureGroup(name, FeatureStoreBase.DEFAULT_VERSION); } /** @@ -625,8 +624,8 @@ public ExternalFeatureGroup getExternalFeatureGroup(@NonNull String name, @NonNu @Override public ExternalFeatureGroup getExternalFeatureGroup(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" - + DEFAULT_VERSION + "`."); - return getExternalFeatureGroup(name, DEFAULT_VERSION); + + FeatureStoreBase.DEFAULT_VERSION + "`."); + return getExternalFeatureGroup(name, FeatureStoreBase.DEFAULT_VERSION); } /** @@ -881,8 +880,8 @@ public ExternalFeatureGroup getOnDemandFeatureGroup(@NonNull String name, @NonNu @Deprecated public ExternalFeatureGroup getOnDemandFeatureGroup(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" - + DEFAULT_VERSION + "`."); - return getExternalFeatureGroup(name, DEFAULT_VERSION); + + FeatureStoreBase.DEFAULT_VERSION + "`."); + return getExternalFeatureGroup(name, FeatureStoreBase.DEFAULT_VERSION); } @Deprecated @@ -992,8 +991,8 @@ public FeatureView getFeatureView(@NonNull String name, @NonNull Integer version @Override public FeatureView getFeatureView(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting feature view `" + name + "`, defaulting to `" - + DEFAULT_VERSION + "`."); - return getFeatureView(name, DEFAULT_VERSION); + + FeatureStoreBase.DEFAULT_VERSION + "`."); + return getFeatureView(name, FeatureStoreBase.DEFAULT_VERSION); } @Override @@ -1016,8 +1015,8 @@ public TrainingDataset getTrainingDataset(@NonNull String name, @NonNull Integer @Deprecated public TrainingDataset getTrainingDataset(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting training dataset `" + name + "`, defaulting to `" - + DEFAULT_VERSION + "`."); - return getTrainingDataset(name, DEFAULT_VERSION); + + FeatureStoreBase.DEFAULT_VERSION + "`."); + return getTrainingDataset(name, FeatureStoreBase.DEFAULT_VERSION); } @Deprecated diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java similarity index 99% rename from java/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java index 3e9f8c7aba..b7f0d1043e 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/FeatureView.java @@ -20,6 +20,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.logicalclocks.hsfs.spark.constructor.Query; +import com.logicalclocks.hsfs.spark.engine.FeatureViewEngine; import com.logicalclocks.hsfs.DataFormat; import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.FeatureViewBase; @@ -30,8 +32,6 @@ import com.logicalclocks.hsfs.constructor.Filter; import com.logicalclocks.hsfs.constructor.FilterLogic; import com.logicalclocks.hsfs.engine.FeatureGroupUtils; -import com.logicalclocks.hsfs.spark.constructor.Query; -import com.logicalclocks.hsfs.spark.engine.FeatureViewEngine; import lombok.NoArgsConstructor; import lombok.NonNull; @@ -90,7 +90,7 @@ public FeatureViewBuilder description(String description) { public FeatureViewBuilder query(Query query) { this.query = query; if (query.isTimeTravel()) { - LOGGER.info("`as_of` argument in the `Query` will be ignored because " + FeatureViewBase.LOGGER.info("`as_of` argument in the `Query` will be ignored because " + "feature view does not support time travel query."); } return this; @@ -137,7 +137,7 @@ public FeatureView(@NonNull String name, Integer version, @NonNull Query query, */ @Override public void delete() throws FeatureStoreException, IOException { - LOGGER.warn("JobWarning: All jobs associated to feature view `" + name + "`, version `" + FeatureViewBase.LOGGER.warn("JobWarning: All jobs associated to feature view `" + name + "`, version `" + version + "` will be removed."); featureViewEngine.delete(this.featureStore, this.name, this.version); } diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java index 674dba1982..e3b63a427f 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/HopsworksConnection.java @@ -17,6 +17,7 @@ package com.logicalclocks.hsfs.spark; +import com.logicalclocks.hsfs.spark.engine.SparkEngine; import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.HopsworksConnectionBase; import com.logicalclocks.hsfs.SecretStore; @@ -24,7 +25,6 @@ import com.logicalclocks.hsfs.metadata.HopsworksHttpClient; import com.logicalclocks.hsfs.metadata.HopsworksInternalClient; -import com.logicalclocks.hsfs.spark.engine.SparkEngine; import lombok.Builder; import software.amazon.awssdk.regions.Region; diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java index c4503d33f9..c9ff950a37 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/MainClass.java @@ -17,10 +17,10 @@ package com.logicalclocks.hsfs.spark; +import com.logicalclocks.hsfs.spark.constructor.Query; +import com.logicalclocks.hsfs.spark.engine.SparkEngine; import com.logicalclocks.hsfs.DataFormat; import com.logicalclocks.hsfs.Split; -import com.logicalclocks.hsfs.spark.engine.SparkEngine; -import com.logicalclocks.hsfs.spark.constructor.Query; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java index 06a5e3e4af..2008bb1856 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/StreamFeatureGroup.java @@ -19,6 +19,9 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.logicalclocks.hsfs.spark.constructor.Query; +import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine; +import com.logicalclocks.hsfs.spark.engine.StatisticsEngine; import com.logicalclocks.hsfs.EntityEndpointType; import com.logicalclocks.hsfs.Feature; import com.logicalclocks.hsfs.FeatureStoreException; @@ -29,9 +32,6 @@ import com.logicalclocks.hsfs.engine.CodeEngine; import com.logicalclocks.hsfs.FeatureGroupBase; import com.logicalclocks.hsfs.metadata.Statistics; -import com.logicalclocks.hsfs.spark.constructor.Query; -import com.logicalclocks.hsfs.spark.engine.FeatureGroupEngine; -import com.logicalclocks.hsfs.spark.engine.StatisticsEngine; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java similarity index 99% rename from java/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java index 4758b8eb57..e269c60425 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDataset.java @@ -21,6 +21,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Strings; import com.google.common.collect.Lists; +import com.logicalclocks.hsfs.spark.constructor.Query; +import com.logicalclocks.hsfs.spark.engine.StatisticsEngine; +import com.logicalclocks.hsfs.spark.engine.TrainingDatasetEngine; import com.logicalclocks.hsfs.DataFormat; import com.logicalclocks.hsfs.EntityEndpointType; import com.logicalclocks.hsfs.FeatureStoreException; @@ -36,9 +39,6 @@ import com.logicalclocks.hsfs.engine.CodeEngine; import com.logicalclocks.hsfs.engine.FeatureGroupUtils; import com.logicalclocks.hsfs.metadata.Statistics; -import com.logicalclocks.hsfs.spark.constructor.Query; -import com.logicalclocks.hsfs.spark.engine.StatisticsEngine; -import com.logicalclocks.hsfs.spark.engine.TrainingDatasetEngine; import lombok.Builder; import lombok.Getter; import lombok.NoArgsConstructor; @@ -339,7 +339,7 @@ public List getServingVector(Map entry) throws SQLExcept */ @Override public void delete() throws FeatureStoreException, IOException { - LOGGER.warn("JobWarning: All jobs associated to training dataset `" + name + "`, version `" + TrainingDatasetBase.LOGGER.warn("JobWarning: All jobs associated to training dataset `" + name + "`, version `" + version + "` will be removed."); trainingDatasetEngine.delete(this); } diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/TrainingDatasetBundle.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDatasetBundle.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/TrainingDatasetBundle.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDatasetBundle.java index 664c0252a5..c63c7f343a 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/TrainingDatasetBundle.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/TrainingDatasetBundle.java @@ -19,8 +19,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.Lists; -import com.logicalclocks.hsfs.Split; import com.logicalclocks.hsfs.spark.engine.SparkEngine; +import com.logicalclocks.hsfs.Split; import lombok.Getter; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/constructor/FsQuery.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/constructor/FsQuery.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/constructor/FsQuery.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/constructor/FsQuery.java index 72b52126ae..569e3db632 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/constructor/FsQuery.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/constructor/FsQuery.java @@ -18,12 +18,12 @@ package com.logicalclocks.hsfs.spark.constructor; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.logicalclocks.hsfs.spark.engine.SparkEngine; import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.constructor.FeatureGroupAlias; import com.logicalclocks.hsfs.constructor.FsQueryBase; import com.logicalclocks.hsfs.spark.ExternalFeatureGroup; import com.logicalclocks.hsfs.spark.StreamFeatureGroup; -import com.logicalclocks.hsfs.spark.engine.SparkEngine; import lombok.AllArgsConstructor; import java.io.IOException; diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/constructor/Query.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/constructor/Query.java similarity index 94% rename from java/src/main/java/com/logicalclocks/hsfs/spark/constructor/Query.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/constructor/Query.java index 43429b69a9..8f533e39db 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/constructor/Query.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/constructor/Query.java @@ -71,7 +71,7 @@ public Dataset read(boolean online, Map readOptions) throws queryConstructorApi.constructQuery(leftFeatureGroup.getFeatureStore(), this, FsQuery.class); if (online) { - LOGGER.info("Executing query: " + fsQuery.getStorageQuery(Storage.ONLINE)); + QueryBase.LOGGER.info("Executing query: " + fsQuery.getStorageQuery(Storage.ONLINE)); StorageConnector.JdbcConnector onlineConnector = storageConnectorApi.getOnlineStorageConnector( leftFeatureGroup.getFeatureStore(), StorageConnector.JdbcConnector.class); @@ -80,7 +80,7 @@ public Dataset read(boolean online, Map readOptions) throws fsQuery.registerOnDemandFeatureGroups(); fsQuery.registerHudiFeatureGroups(readOptions); - LOGGER.info("Executing query: " + fsQuery.getStorageQuery(Storage.OFFLINE)); + QueryBase.LOGGER.info("Executing query: " + fsQuery.getStorageQuery(Storage.OFFLINE)); return SparkEngine.getInstance().sql(fsQuery.getStorageQuery(Storage.OFFLINE)); } } diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/FeatureGroupEngine.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/FeatureGroupEngine.java similarity index 99% rename from java/src/main/java/com/logicalclocks/hsfs/spark/engine/FeatureGroupEngine.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/FeatureGroupEngine.java index 61024b68dc..22cc333389 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/FeatureGroupEngine.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/FeatureGroupEngine.java @@ -17,6 +17,7 @@ package com.logicalclocks.hsfs.spark.engine; +import com.logicalclocks.hsfs.spark.engine.hudi.HudiEngine; import com.logicalclocks.hsfs.Feature; import com.logicalclocks.hsfs.FeatureGroupCommit; import com.logicalclocks.hsfs.FeatureStoreException; @@ -32,7 +33,6 @@ import com.logicalclocks.hsfs.spark.FeatureGroup; import com.logicalclocks.hsfs.spark.FeatureStore; -import com.logicalclocks.hsfs.spark.engine.hudi.HudiEngine; import lombok.SneakyThrows; import org.apache.spark.sql.Dataset; @@ -235,7 +235,7 @@ public FeatureGroup saveFeatureGroupMetaData(FeatureGroup featureGroup, List onlineFeatureGroupToAvro(FeatureGroupBase featureGroupBase, return dataset.select( to_avro(concat(pks.stream().map(name -> col(name).cast("string")) .toArray(Column[]::new))).alias("key"), - to_avro(struct(featureGroupBase.getDeserializedAvroSchema().getFields().stream() + to_avro(functions.struct(featureGroupBase.getDeserializedAvroSchema().getFields().stream() .map(f -> col(f.name())).toArray(Column[]::new)), featureGroupBase.getEncodedAvroSchema()).alias("value")); } diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/StatisticsEngine.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/StatisticsEngine.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/engine/StatisticsEngine.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/StatisticsEngine.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetEngine.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetEngine.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetEngine.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetEngine.java index bc816062b4..f2799c8908 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetEngine.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetEngine.java @@ -18,6 +18,7 @@ package com.logicalclocks.hsfs.spark.engine; import com.google.common.collect.Maps; +import com.logicalclocks.hsfs.spark.constructor.Query; import com.logicalclocks.hsfs.EntityEndpointType; import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.Storage; @@ -25,7 +26,6 @@ import com.logicalclocks.hsfs.metadata.TagsApi; import com.logicalclocks.hsfs.metadata.TrainingDatasetApi; import com.logicalclocks.hsfs.spark.TrainingDataset; -import com.logicalclocks.hsfs.spark.constructor.Query; import com.logicalclocks.hsfs.spark.util.StorageConnectorUtils; import org.apache.hadoop.fs.Path; diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetUtils.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetUtils.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetUtils.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/TrainingDatasetUtils.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerAvroDeserializer.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerAvroDeserializer.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerAvroDeserializer.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerAvroDeserializer.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerConfig.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerConfig.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerConfig.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerConfig.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerKafkaSource.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerKafkaSource.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerKafkaSource.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerKafkaSource.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerSchemaProvider.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerSchemaProvider.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerSchemaProvider.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerSchemaProvider.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerTransformer.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerTransformer.java similarity index 100% rename from java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerTransformer.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/DeltaStreamerTransformer.java diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java similarity index 97% rename from java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java index b80f70a486..afddae7fb7 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java @@ -112,16 +112,16 @@ public class HudiEngine { protected static final String COMMIT_METADATA_KEYPREFIX_OPT_KEY = "hoodie.datasource.write.commitmeta.key.prefix"; protected static final String DELTASTREAMER_CHECKPOINT_KEY = "deltastreamer.checkpoint.key"; protected static final String INITIAL_CHECKPOINT_STRING = "initialCheckPointString"; - protected static final String FEATURE_GROUP_SCHEMA = "com.logicalclocks.hsfs.spark.StreamFeatureGroup.avroSchema"; + protected static final String FEATURE_GROUP_SCHEMA = "StreamFeatureGroup.avroSchema"; protected static final String FEATURE_GROUP_ENCODED_SCHEMA = - "com.logicalclocks.hsfs.spark.StreamFeatureGroup.encodedAvroSchema"; + "StreamFeatureGroup.encodedAvroSchema"; protected static final String FEATURE_GROUP_COMPLEX_FEATURES = - "com.logicalclocks.hsfs.spark.StreamFeatureGroup.complexFeatures"; - protected static final String KAFKA_SOURCE = "com.logicalclocks.hsfs.spark.engine.hudi.DeltaStreamerKafkaSource"; + "StreamFeatureGroup.complexFeatures"; + protected static final String KAFKA_SOURCE = "DeltaStreamerKafkaSource"; protected static final String SCHEMA_PROVIDER = - "com.logicalclocks.hsfs.spark.engine.hudi.DeltaStreamerSchemaProvider"; + "DeltaStreamerSchemaProvider"; protected static final String DELTA_STREAMER_TRANSFORMER = - "com.logicalclocks.hsfs.spark.engine.hudi.DeltaStreamerTransformer"; + "DeltaStreamerTransformer"; protected static final String DELTA_SOURCE_ORDERING_FIELD_OPT_KEY = "sourceOrderingField"; protected static final String MIN_SYNC_INTERVAL_SECONDS = "minSyncIntervalSeconds"; diff --git a/java/src/main/java/com/logicalclocks/hsfs/spark/util/StorageConnectorUtils.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/util/StorageConnectorUtils.java similarity index 97% rename from java/src/main/java/com/logicalclocks/hsfs/spark/util/StorageConnectorUtils.java rename to java/spark/src/main/java/com/logicalclocks/hsfs/spark/util/StorageConnectorUtils.java index 3bd3c130b4..2c40fbccad 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/spark/util/StorageConnectorUtils.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/util/StorageConnectorUtils.java @@ -18,16 +18,19 @@ package com.logicalclocks.hsfs.spark.util; import com.google.common.base.Strings; +import com.logicalclocks.hsfs.spark.engine.SparkEngine; import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.StorageConnector; import com.logicalclocks.hsfs.util.Constants; -import com.logicalclocks.hsfs.spark.engine.SparkEngine; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import javax.ws.rs.NotSupportedException; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.Arrays; +import java.util.Base64; import java.util.Map; public class StorageConnectorUtils { @@ -173,6 +176,12 @@ public Dataset read(StorageConnector.BigqueryConnector connector, String qu Map options, String path) throws FeatureStoreException, IOException { Map readOptions = connector.sparkOptions(); + + // Base64 encode the credentials file + String localKeyPath = SparkEngine.getInstance().addFile(connector.getKeyPath()); + byte[] fileContent = Files.readAllBytes(Paths.get(localKeyPath)); + options.put(Constants.BIGQ_CREDENTIALS, Base64.getEncoder().encodeToString(fileContent)); + // merge user spark options on top of default spark options if (options != null && !options.isEmpty()) { readOptions.putAll(options); diff --git a/java/src/test/java/com/logicalclocks/hsfs/spark/TestExternalFeatureGroup.java b/java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestExternalFeatureGroup.java similarity index 100% rename from java/src/test/java/com/logicalclocks/hsfs/spark/TestExternalFeatureGroup.java rename to java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestExternalFeatureGroup.java diff --git a/java/src/test/java/com/logicalclocks/hsfs/spark/TestFeature.java b/java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestFeature.java similarity index 100% rename from java/src/test/java/com/logicalclocks/hsfs/spark/TestFeature.java rename to java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestFeature.java diff --git a/java/src/test/java/com/logicalclocks/hsfs/spark/TestFeatureGroup.java b/java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestFeatureGroup.java similarity index 100% rename from java/src/test/java/com/logicalclocks/hsfs/spark/TestFeatureGroup.java rename to java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestFeatureGroup.java diff --git a/java/src/test/java/com/logicalclocks/TestStorageConnector.java b/java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestStorageConnector.java similarity index 94% rename from java/src/test/java/com/logicalclocks/TestStorageConnector.java rename to java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestStorageConnector.java index ceb74bb294..5ebed968c2 100644 --- a/java/src/test/java/com/logicalclocks/TestStorageConnector.java +++ b/java/spark/src/test/java/com/logicalclocks/hsfs/spark/TestStorageConnector.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2022. Hopsworks AB + * Copyright (c) 2022-2023. Hopsworks AB * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ * */ -package com.logicalclocks; +package com.logicalclocks.hsfs.spark; import com.logicalclocks.hsfs.FeatureStoreException; import com.logicalclocks.hsfs.StorageConnectorType; @@ -39,6 +39,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Base64; import java.util.Map; @@ -57,12 +58,15 @@ public void testBigQueryCredentialsBase64Encoded(@TempDir Path tempDir) throws I } else { bigqueryConnector.setKeyPath("file://" + credentialsFile); } + // Act - Map sparkOptions = bigqueryConnector.sparkOptions(); + // Base64 encode the credentials file + String localKeyPath = SparkEngine.getInstance().addFile(bigqueryConnector.getKeyPath()); + String fileContent = Base64.getEncoder().encodeToString(Files.readAllBytes(Paths.get(localKeyPath))); // Assert Assertions.assertEquals(credentials, - new String(Base64.getDecoder().decode(sparkOptions.get(Constants.BIGQ_CREDENTIALS)), StandardCharsets.UTF_8)); + new String(Base64.getDecoder().decode(fileContent), StandardCharsets.UTF_8)); } @Test @@ -104,6 +108,7 @@ public void testGcsConnectorCredentials(@TempDir Path tempDir) throws IOExceptio gcsConnector.setKeyPath("file://" + credentialsFile); } gcsConnector.setStorageConnectorType(StorageConnectorType.GCS); + // Act SparkEngine.getInstance().setupConnectorHadoopConf(gcsConnector); SparkContext sc = SparkEngine.getInstance().getSparkSession().sparkContext(); @@ -141,6 +146,7 @@ public void testGcsConnectorCredentials_encrypted(@TempDir Path tempDir) throws gcsConnector.setAlgorithm("AES256"); gcsConnector.setEncryptionKey("encryptionkey"); gcsConnector.setEncryptionKeyHash("encryptionkeyhash"); + // Act SparkEngine.getInstance().setupConnectorHadoopConf(gcsConnector); SparkContext sc = SparkEngine.getInstance().getSparkSession().sparkContext(); diff --git a/java/src/test/java/com/logicalclocks/hsfs/spark/engine/TestFeatureViewEngine.java b/java/spark/src/test/java/com/logicalclocks/hsfs/spark/engine/TestFeatureViewEngine.java similarity index 100% rename from java/src/test/java/com/logicalclocks/hsfs/spark/engine/TestFeatureViewEngine.java rename to java/spark/src/test/java/com/logicalclocks/hsfs/spark/engine/TestFeatureViewEngine.java diff --git a/java/src/test/java/com/logicalclocks/hsfs/spark/engine/TestHudiEngine.java b/java/spark/src/test/java/com/logicalclocks/hsfs/spark/engine/TestHudiEngine.java similarity index 100% rename from java/src/test/java/com/logicalclocks/hsfs/spark/engine/TestHudiEngine.java rename to java/spark/src/test/java/com/logicalclocks/hsfs/spark/engine/TestHudiEngine.java diff --git a/java/src/test/java/com/logicalclocks/hsfs/spark/engine/TestSparkEngine.java b/java/spark/src/test/java/com/logicalclocks/hsfs/spark/engine/TestSparkEngine.java similarity index 100% rename from java/src/test/java/com/logicalclocks/hsfs/spark/engine/TestSparkEngine.java rename to java/spark/src/test/java/com/logicalclocks/hsfs/spark/engine/TestSparkEngine.java diff --git a/java/spark/src/test/resources/hadoop/bin/winutils.exe b/java/spark/src/test/resources/hadoop/bin/winutils.exe new file mode 100644 index 0000000000..aef881bc2a Binary files /dev/null and b/java/spark/src/test/resources/hadoop/bin/winutils.exe differ diff --git a/java/spark/src/test/resources/system.properties b/java/spark/src/test/resources/system.properties new file mode 100644 index 0000000000..3d39703a4f --- /dev/null +++ b/java/spark/src/test/resources/system.properties @@ -0,0 +1 @@ +spark.master=local[1]