diff --git a/README.md b/README.md index 352d7c3f..27a0b8b9 100644 --- a/README.md +++ b/README.md @@ -44,15 +44,15 @@ Run indiviudal test Run SHC examples - ./bin/spark-submit --verbose --class org.apache.spark.sql.execution.datasources.hbase.examples.HBaseSource --master yarn-cluster --packages com.hortonworks:shc-core:1.1.0-2.1-s_2.11 --repositories http://repo.hortonworks.com/content/groups/public/ --files /usr/hdp/current/hbase-client/conf/hbase-site.xml shc-examples-1.1.0-2.1-s_2.11-SNAPSHOT.jar + ./bin/spark-submit --verbose --class org.apache.spark.sql.execution.datasources.hbase.examples.HBaseSource --master yarn-cluster --packages com.hortonworks:shc-core:1.1.1-2.1-s_2.11 --repositories http://repo.hortonworks.com/content/groups/public/ --files /usr/hdp/current/hbase-client/conf/hbase-site.xml shc-examples-1.1.1-2.1-s_2.11-SNAPSHOT.jar The following illustrates how to run your application in real hbase cluster. You need to provide the hbase-site.xml. It may subject to change based on your specific cluster configuration. - ./bin/spark-submit --class your.application.class --master yarn-client --packages com.hortonworks:shc-core:1.1.0-2.1-s_2.11 --repositories http://repo.hortonworks.com/content/groups/public/ --files /etc/hbase/conf/hbase-site.xml /To/your/application/jar + ./bin/spark-submit --class your.application.class --master yarn-client --packages com.hortonworks:shc-core:1.1.1-2.1-s_2.11 --repositories http://repo.hortonworks.com/content/groups/public/ --files /etc/hbase/conf/hbase-site.xml /To/your/application/jar Running Spark applications with this connector, HBase jars of version 1.1.2 will be pulled by default. If Phoenix is enabled on HBase cluster, you need to use "--jars" to pass "phoenix-server.jar". For example: - ./bin/spark-submit --class your.application.class --master yarn-client --packages com.hortonworks:shc-core:1.1.0-2.1-s_2.11 --repositories http://repo.hortonworks.com/content/groups/public/ --jars /usr/hdp/current/phoenix-client/phoenix-server.jar --files /etc/hbase/conf/hbase-site.xml /To/your/application/jar + ./bin/spark-submit --class your.application.class --master yarn-client --packages com.hortonworks:shc-core:1.1.1-2.1-s_2.11 --repositories http://repo.hortonworks.com/content/groups/public/ --jars /usr/hdp/current/phoenix-client/phoenix-server.jar --files /etc/hbase/conf/hbase-site.xml /To/your/application/jar ## Application Usage The following illustrates the basic procedure on how to use the connector. For more details and advanced use case, such as Avro and composite key support, please refer to the examples in the repository. @@ -120,15 +120,15 @@ Given a DataFrame with specified schema, above will create an HBase table with 5 ## Configuring Spark-package Users can use the Spark-on-HBase connector as a standard Spark package. To include the package in your Spark application use: -_**Note**: com.hortonworks:shc-core:1.1.0-2.1-s_2.11 has not been uploaded to [spark-packages.org](https://spark-packages.org/package/hortonworks-spark/shc), but will be there soon._ +_**Note**: com.hortonworks:shc-core:1.1.1-2.1-s_2.11 has not been uploaded to [spark-packages.org](https://spark-packages.org/package/hortonworks-spark/shc), but will be there soon._ spark-shell, pyspark, or spark-submit - $SPARK_HOME/bin/spark-shell --packages com.hortonworks:shc-core:1.1.0-2.1-s_2.11 + $SPARK_HOME/bin/spark-shell --packages com.hortonworks:shc-core:1.1.1-2.1-s_2.11 Users can include the package as the dependency in your SBT file as well. The format is the spark-package-name:version in build.sbt file. - libraryDependencies += “com.hortonworks/shc-core:1.1.0-2.1-s_2.11” + libraryDependencies += “com.hortonworks/shc-core:1.1.1-2.1-s_2.11” ## Running in secure cluster @@ -142,9 +142,9 @@ Suppose hrt_qa is a headless account, user can use following command for kinit: kinit -k -t /tmp/hrt_qa.headless.keytab hrt_qa - /usr/hdp/current/spark-client/bin/spark-submit --class your.application.class --master yarn-client --files /etc/hbase/conf/hbase-site.xml --packages com.hortonworks:shc-core:1.1.0-2.1-s_2.11 --repositories http://repo.hortonworks.com/content/groups/public/ /To/your/application/jar + /usr/hdp/current/spark-client/bin/spark-submit --class your.application.class --master yarn-client --files /etc/hbase/conf/hbase-site.xml --packages com.hortonworks:shc-core:1.1.1-2.1-s_2.11 --repositories http://repo.hortonworks.com/content/groups/public/ /To/your/application/jar - /usr/hdp/current/spark-client/bin/spark-submit --class your.application.class --master yarn-cluster --files /etc/hbase/conf/hbase-site.xml --packages com.hortonworks:shc-core:1.1.0-2.1-s_2.11 --repositories http://repo.hortonworks.com/content/groups/public/ /To/your/application/jar + /usr/hdp/current/spark-client/bin/spark-submit --class your.application.class --master yarn-cluster --files /etc/hbase/conf/hbase-site.xml --packages com.hortonworks:shc-core:1.1.1-2.1-s_2.11 --repositories http://repo.hortonworks.com/content/groups/public/ /To/your/application/jar If the solution above does not work and you encounter errors like : diff --git a/core/pom.xml b/core/pom.xml index 2eaddaa6..e9e75a67 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -24,12 +24,12 @@ com.hortonworks shc - 1.1.1-2.1-s_2.11-SNAPSHOT + 1.1.1-2.1-s_2.11 ../pom.xml shc-core - 1.1.1-2.1-s_2.11-SNAPSHOT + 1.1.1-2.1-s_2.11 jar HBase Spark Connector Project Core diff --git a/examples/pom.xml b/examples/pom.xml index 51ce3ece..39a83f20 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -24,12 +24,12 @@ com.hortonworks shc - 1.1.1-2.1-s_2.11-SNAPSHOT + 1.1.1-2.1-s_2.11 ../pom.xml shc-examples - 1.1.1-2.1-s_2.11-SNAPSHOT + 1.1.1-2.1-s_2.11 jar HBase Spark Connector Project Examples diff --git a/pom.xml b/pom.xml index 008f4228..dc3950c8 100644 --- a/pom.xml +++ b/pom.xml @@ -28,7 +28,7 @@ com.hortonworks shc - 1.1.1-2.1-s_2.11-SNAPSHOT + 1.1.1-2.1-s_2.11 pom HBase Spark Connector Project Parent POM https://github.com/hortonworks-spark/shc @@ -42,7 +42,7 @@ - 2.1.0 + 2.1.1 1.1.2 4.9.0-HBase-1.1 ${project.build.directory}/spark-test-classpath.txt