From 6668ce1a9e3a99de686f1b692bfe598465b3d28d Mon Sep 17 00:00:00 2001 From: Mike Drob Date: Fri, 5 Oct 2018 16:24:33 -0500 Subject: [PATCH] HBASE-21273 Move classes out of o.a.spark packages --- .../org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java | 2 +- .../scala/org/apache/hadoop/hbase/spark/DefaultSource.scala | 1 - .../org/apache/hadoop/hbase/spark/DynamicLogicExpression.scala | 3 +-- .../hbase/spark/datasources}/DataTypeParserWrapper.scala | 2 +- .../hbase/spark/datasources}/HBaseTableCatalog.scala | 10 +++------- .../hadoop/hbase/spark/datasources/HBaseTableScanRDD.scala | 1 - .../hbase => hadoop/hbase/spark/datasources}/Utils.scala | 2 +- .../hadoop/hbase/spark/example/datasources/AvroSource.scala | 2 +- .../hadoop/hbase/spark/example/datasources/DataType.scala | 2 +- .../hadoop/hbase/spark/example/datasources/HBaseSource.scala | 2 +- .../org/apache/hadoop/hbase/spark/DefaultSourceSuite.scala | 3 +-- .../org/apache/hadoop/hbase/spark/HBaseCatalogSuite.scala | 3 +-- .../org/apache/hadoop/hbase/spark/PartitionFilterSuite.scala | 5 ++--- 13 files changed, 14 insertions(+), 24 deletions(-) rename hbase-spark/src/main/scala/org/apache/{spark/sql/datasources/hbase => hadoop/hbase/spark/datasources}/DataTypeParserWrapper.scala (96%) rename hbase-spark/src/main/scala/org/apache/{spark/sql/datasources/hbase => hadoop/hbase/spark/datasources}/HBaseTableCatalog.scala (97%) rename hbase-spark/src/main/scala/org/apache/{spark/sql/datasources/hbase => hadoop/hbase/spark/datasources}/Utils.scala (98%) diff --git a/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java b/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java index b1d0ba7e62..a17d2e68fb 100644 --- a/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java +++ b/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java @@ -32,11 +32,11 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.filter.Filter.ReturnCode; import org.apache.hadoop.hbase.filter.FilterBase; import org.apache.hadoop.hbase.spark.datasources.BytesEncoder; +import org.apache.hadoop.hbase.spark.datasources.Field; import org.apache.hadoop.hbase.spark.datasources.JavaBytesEncoder; import org.apache.hadoop.hbase.spark.protobuf.generated.SparkFilterProtos; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.spark.sql.datasources.hbase.Field; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DefaultSource.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DefaultSource.scala index 9e139540e5..4e05695fc7 100644 --- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DefaultSource.scala +++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DefaultSource.scala @@ -34,7 +34,6 @@ import org.apache.hadoop.hbase.TableName import org.apache.hadoop.hbase.CellUtil import org.apache.hadoop.mapred.JobConf import org.apache.spark.rdd.RDD -import org.apache.spark.sql.datasources.hbase.{Utils, Field, HBaseTableCatalog} import org.apache.spark.sql.{DataFrame, SaveMode, Row, SQLContext} import org.apache.spark.sql.sources._ import org.apache.spark.sql.types._ diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DynamicLogicExpression.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DynamicLogicExpression.scala index 283838f52b..4c35a7b4e2 100644 --- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DynamicLogicExpression.scala +++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/DynamicLogicExpression.scala @@ -23,8 +23,7 @@ import org.apache.yetus.audience.InterfaceAudience; import org.apache.hadoop.hbase.spark.datasources.{BytesEncoder, JavaBytesEncoder} import org.apache.hadoop.hbase.spark.datasources.JavaBytesEncoder.JavaBytesEncoder import org.apache.hadoop.hbase.util.Bytes -import org.apache.spark.sql.datasources.hbase.{Field, Utils} -import org.apache.spark.sql.types._ + /** * Dynamic logic for SQL push down logic there is an instance for most * common operations and a pass through for other operations not covered here diff --git a/hbase-spark/src/main/scala/org/apache/spark/sql/datasources/hbase/DataTypeParserWrapper.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/DataTypeParserWrapper.scala similarity index 96% rename from hbase-spark/src/main/scala/org/apache/spark/sql/datasources/hbase/DataTypeParserWrapper.scala rename to hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/DataTypeParserWrapper.scala index 430785222a..c0ccc92882 100644 --- a/hbase-spark/src/main/scala/org/apache/spark/sql/datasources/hbase/DataTypeParserWrapper.scala +++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/DataTypeParserWrapper.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.spark.sql.datasources.hbase +package org.apache.hadoop.hbase.spark.datasources import org.apache.spark.sql.catalyst.parser.CatalystSqlParser import org.apache.spark.sql.types.DataType diff --git a/hbase-spark/src/main/scala/org/apache/spark/sql/datasources/hbase/HBaseTableCatalog.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableCatalog.scala similarity index 97% rename from hbase-spark/src/main/scala/org/apache/spark/sql/datasources/hbase/HBaseTableCatalog.scala rename to hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableCatalog.scala index 6cb0b18706..d2a8a3e845 100644 --- a/hbase-spark/src/main/scala/org/apache/spark/sql/datasources/hbase/HBaseTableCatalog.scala +++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableCatalog.scala @@ -15,21 +15,17 @@ * limitations under the License. */ -package org.apache.spark.sql.datasources.hbase +package org.apache.hadoop.hbase.spark.datasources import org.apache.avro.Schema -import org.apache.yetus.audience.InterfaceAudience; -import org.apache.hadoop.hbase.spark.Logging -import org.apache.hadoop.hbase.spark.SchemaConverters -import org.apache.hadoop.hbase.spark.datasources._ -import org.apache.hadoop.hbase.spark.hbase._ +import org.apache.yetus.audience.InterfaceAudience +import org.apache.hadoop.hbase.spark.{Logging, SchemaConverters} import org.apache.hadoop.hbase.util.Bytes import org.apache.spark.sql.types._ import org.json4s.jackson.JsonMethods._ import scala.collection.mutable -// Due the access issue defined in spark, we have to locate the file in this package. // The definition of each column cell, which may be composite type // TODO: add avro support @InterfaceAudience.Private diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableScanRDD.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableScanRDD.scala index 6a656677e1..6c06811d12 100644 --- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableScanRDD.scala +++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/HBaseTableScanRDD.scala @@ -25,7 +25,6 @@ import org.apache.hadoop.hbase.spark._ import org.apache.hadoop.hbase.spark.hbase._ import org.apache.hadoop.hbase.spark.datasources.HBaseResources._ import org.apache.hadoop.hbase.util.ShutdownHookManager -import org.apache.spark.sql.datasources.hbase.Field import org.apache.spark.{SparkEnv, TaskContext, Partition} import org.apache.spark.rdd.RDD diff --git a/hbase-spark/src/main/scala/org/apache/spark/sql/datasources/hbase/Utils.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Utils.scala similarity index 98% rename from hbase-spark/src/main/scala/org/apache/spark/sql/datasources/hbase/Utils.scala rename to hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Utils.scala index 5db5965f61..093c6ac04e 100644 --- a/hbase-spark/src/main/scala/org/apache/spark/sql/datasources/hbase/Utils.scala +++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/datasources/Utils.scala @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.spark.sql.datasources.hbase +package org.apache.hadoop.hbase.spark.datasources import org.apache.hadoop.hbase.spark.AvroSerdes import org.apache.hadoop.hbase.util.Bytes diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/AvroSource.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/AvroSource.scala index fda3c785ea..068b1af6ac 100644 --- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/AvroSource.scala +++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/AvroSource.scala @@ -20,7 +20,7 @@ package org.apache.hadoop.hbase.spark.example.datasources import org.apache.avro.Schema import org.apache.avro.generic.GenericData import org.apache.hadoop.hbase.spark.AvroSerdes -import org.apache.spark.sql.datasources.hbase.HBaseTableCatalog +import org.apache.hadoop.hbase.spark.datasources.HBaseTableCatalog import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SQLContext import org.apache.spark.SparkConf diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/DataType.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/DataType.scala index 06303716af..ac7e7766de 100644 --- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/DataType.scala +++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/DataType.scala @@ -17,7 +17,7 @@ package org.apache.hadoop.hbase.spark.example.datasources -import org.apache.spark.sql.datasources.hbase.HBaseTableCatalog +import org.apache.hadoop.hbase.spark.datasources.HBaseTableCatalog import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SQLContext import org.apache.spark.SparkConf diff --git a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/HBaseSource.scala b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/HBaseSource.scala index b414a379f5..6accae0925 100644 --- a/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/HBaseSource.scala +++ b/hbase-spark/src/main/scala/org/apache/hadoop/hbase/spark/example/datasources/HBaseSource.scala @@ -17,7 +17,7 @@ package org.apache.hadoop.hbase.spark.example.datasources -import org.apache.spark.sql.datasources.hbase.HBaseTableCatalog +import org.apache.hadoop.hbase.spark.datasources.HBaseTableCatalog import org.apache.spark.sql.DataFrame import org.apache.spark.sql.SQLContext import org.apache.spark.SparkConf diff --git a/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/DefaultSourceSuite.scala b/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/DefaultSourceSuite.scala index 89b68f4cde..afe515bce9 100644 --- a/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/DefaultSourceSuite.scala +++ b/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/DefaultSourceSuite.scala @@ -20,10 +20,9 @@ package org.apache.hadoop.hbase.spark import org.apache.avro.Schema import org.apache.avro.generic.GenericData import org.apache.hadoop.hbase.client.{ConnectionFactory, Put} -import org.apache.hadoop.hbase.spark.datasources.HBaseSparkConf +import org.apache.hadoop.hbase.spark.datasources.{HBaseSparkConf, HBaseTableCatalog} import org.apache.hadoop.hbase.util.Bytes import org.apache.hadoop.hbase.{HBaseTestingUtility, TableName} -import org.apache.spark.sql.datasources.hbase.HBaseTableCatalog import org.apache.spark.sql.functions._ import org.apache.spark.sql.{DataFrame, SQLContext} import org.apache.spark.{SparkConf, SparkContext} diff --git a/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/HBaseCatalogSuite.scala b/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/HBaseCatalogSuite.scala index d8af4a8791..74bf912cc8 100644 --- a/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/HBaseCatalogSuite.scala +++ b/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/HBaseCatalogSuite.scala @@ -17,9 +17,8 @@ package org.apache.hadoop.hbase.spark -import org.apache.hadoop.hbase.spark.datasources.{DoubleSerDes, SerDes} +import org.apache.hadoop.hbase.spark.datasources.{DataTypeParserWrapper, DoubleSerDes, HBaseTableCatalog} import org.apache.hadoop.hbase.util.Bytes -import org.apache.spark.sql.datasources.hbase.{DataTypeParserWrapper, HBaseTableCatalog} import org.apache.spark.sql.types._ import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, FunSuite} diff --git a/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/PartitionFilterSuite.scala b/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/PartitionFilterSuite.scala index 034d450615..496008473f 100644 --- a/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/PartitionFilterSuite.scala +++ b/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/PartitionFilterSuite.scala @@ -17,9 +17,8 @@ package org.apache.hadoop.hbase.spark -import org.apache.hadoop.hbase.spark.datasources.HBaseSparkConf -import org.apache.hadoop.hbase.{TableName, HBaseTestingUtility} -import org.apache.spark.sql.datasources.hbase.HBaseTableCatalog +import org.apache.hadoop.hbase.spark.datasources.{HBaseSparkConf, HBaseTableCatalog} +import org.apache.hadoop.hbase.{HBaseTestingUtility, TableName} import org.apache.spark.sql.{DataFrame, SQLContext} import org.apache.spark.{SparkConf, SparkContext} import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, FunSuite} -- 2.16.1