diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java index 924e233293..93b927ee58 100644 --- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java +++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/AbstractRecordWriter.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Table; @@ -185,7 +186,19 @@ protected int getBucket(Object row) throws SerializationError { } ObjectInspector[] inspectors = getBucketObjectInspectors(); Object[] bucketFields = getBucketFields(row); - return ObjectInspectorUtils.getBucketNumber(bucketFields, inspectors, totalBuckets); + String versionStr = tbl.getParameters().get(hive_metastoreConstants.TABLE_BUCKETING_VERSION); + int bucketingVersion = 1; + if (versionStr != null) { + try { + bucketingVersion = Integer.parseInt(versionStr); + } catch (NumberFormatException e) { + // Do nothing, should not happen. + } + } + + return bucketingVersion == 2 ? + ObjectInspectorUtils.getBucketNumber(bucketFields, inspectors, totalBuckets) : + ObjectInspectorUtils.getBucketNumberOld(bucketFields, inspectors, totalBuckets); } @Override diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/BucketIdResolver.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/BucketIdResolver.java index 5dd0b8ea5b..ca1cfed2f8 100644 --- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/BucketIdResolver.java +++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/BucketIdResolver.java @@ -17,12 +17,14 @@ */ package org.apache.hive.hcatalog.streaming.mutate.worker; +import org.apache.hadoop.hive.serde2.SerDeException; + /** Computes and appends bucket ids to records that are due to be inserted. */ public interface BucketIdResolver { - Object attachBucketIdToRecord(Object record); + Object attachBucketIdToRecord(Object record) throws SerDeException; /** See: {@link org.apache.hadoop.hive.ql.exec.ReduceSinkOperator#computeBucketNumber(Object, int)}. */ - int computeBucketId(Object record); + int computeBucketId(Object record) throws SerDeException; } diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/BucketIdResolverImpl.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/BucketIdResolverImpl.java index 7c2cadefa7..e9c7f7a9bf 100644 --- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/BucketIdResolverImpl.java +++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/BucketIdResolverImpl.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.ql.io.AcidOutputFormat; import org.apache.hadoop.hive.ql.io.BucketCodec; import org.apache.hadoop.hive.ql.io.RecordIdentifier; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector; diff --git a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/MutatorCoordinator.java b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/MutatorCoordinator.java index ad14c7265f..3867d934bd 100644 --- a/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/MutatorCoordinator.java +++ b/hcatalog/streaming/src/java/org/apache/hive/hcatalog/streaming/mutate/worker/MutatorCoordinator.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.io.BucketCodec; import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.ql.io.RecordUpdater; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hive.hcatalog.streaming.mutate.client.AcidTable; import org.slf4j.Logger; @@ -212,7 +213,12 @@ private RecordIdentifier extractRecordIdentifier(OperationType operationType, Li RecordIdentifier recordIdentifier = recordInspector.extractRecordIdentifier(record); int bucketIdFromRecord = BucketCodec.determineVersion( recordIdentifier.getBucketProperty()).decodeWriterId(recordIdentifier.getBucketProperty()); - int computedBucketId = bucketIdResolver.computeBucketId(record); + int computedBucketId = -1; + try { + computedBucketId = bucketIdResolver.computeBucketId(record); + } catch (SerDeException e) { + throw new BucketIdException("SerDeException in fetching bucket Id"); + } if (operationType != OperationType.DELETE && bucketIdFromRecord != computedBucketId) { throw new BucketIdException("RecordIdentifier.bucketId != computed bucketId (" + computedBucketId + ") for record " + recordIdentifier + " in partition " + newPartitionValues + "."); diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestBucketIdResolverImpl.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestBucketIdResolverImpl.java index 03c28a33c8..5ad6372e94 100644 --- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestBucketIdResolverImpl.java +++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/worker/TestBucketIdResolverImpl.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.io.AcidOutputFormat; import org.apache.hadoop.hive.ql.io.BucketCodec; import org.apache.hadoop.hive.ql.io.RecordIdentifier; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hive.hcatalog.streaming.mutate.MutableRecord; import org.junit.Test; @@ -41,7 +42,11 @@ @Test public void testAttachBucketIdToRecord() { MutableRecord record = new MutableRecord(1, "hello"); - capturingBucketIdResolver.attachBucketIdToRecord(record); + try { + capturingBucketIdResolver.attachBucketIdToRecord(record); + } catch (SerDeException e) { + assert false; + } assertThat(record.rowId, is(new RecordIdentifier(-1L, BucketCodec.V1.encode(new AcidOutputFormat.Options(null).bucket(1)), -1L))); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java index c084fa054c..5eb05bc0c0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java @@ -66,10 +66,7 @@ import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.ql.stats.StatsCollectionContext; import org.apache.hadoop.hive.ql.stats.StatsPublisher; -import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; -import org.apache.hadoop.hive.serde2.SerDeException; -import org.apache.hadoop.hive.serde2.SerDeStats; -import org.apache.hadoop.hive.serde2.Serializer; +import org.apache.hadoop.hive.serde2.*; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; @@ -86,6 +83,7 @@ import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hive.common.util.HiveStringUtils; +import org.apache.hive.common.util.Murmur3; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1050,7 +1048,9 @@ assert getConf().getWriteType() != AcidUtils.Operation.DELETE && for(int i = 0; i < partitionEval.length; i++) { bucketFieldValues[i] = partitionEval[i].evaluate(row); } - int keyHashCode = ObjectInspectorUtils.getBucketHashCode(bucketFieldValues, partitionObjectInspectors); + int keyHashCode = conf.getTableInfo().getBucketingVersion() == 2 ? + ObjectInspectorUtils.getBucketHashCode(bucketFieldValues, partitionObjectInspectors) : + ObjectInspectorUtils.getBucketHashCodeOld(bucketFieldValues, partitionObjectInspectors); key.setHashCode(keyHashCode); int bucketNum = prtner.getBucket(key, null, totalFiles); return bucketMap.get(bucketNum); @@ -1578,4 +1578,5 @@ private Configuration unsetNestedColumnPaths(Configuration conf) { private boolean isNativeTable() { return !conf.getTableInfo().isNonNative(); } + } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java index d59bf1fb6e..bb91eeae4b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java @@ -478,6 +478,7 @@ system.registerGenericUDF("when", GenericUDFWhen.class); system.registerGenericUDF("nullif", GenericUDFNullif.class); system.registerGenericUDF("hash", GenericUDFHash.class); + system.registerGenericUDF("murmur_hash", GenericUDFMurmurHash.class); system.registerGenericUDF("coalesce", GenericUDFCoalesce.class); system.registerGenericUDF("index", GenericUDFIndex.class); system.registerGenericUDF("in_file", GenericUDFInFile.class); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java index d4363fdf91..29f747592f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java @@ -39,8 +39,10 @@ import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; +import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.Serializer; +import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; @@ -52,7 +54,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.OutputCollector; -import org.apache.hadoop.util.hash.MurmurHash; +import org.apache.hive.common.util.Murmur3; /** * Reduce Sink Operator sends output to the reduce stage. @@ -61,7 +63,6 @@ implements Serializable, TopNHash.BinaryCollector { private static final long serialVersionUID = 1L; - private static final MurmurHash hash = (MurmurHash) MurmurHash.getInstance(); private transient ObjectInspector[] partitionObjectInspectors; private transient ObjectInspector[] bucketObjectInspectors; @@ -118,7 +119,7 @@ * This two dimensional array holds key data and a corresponding Union object * which contains the tag identifying the aggregate expression for distinct columns. * - * If there is no distict expression, cachedKeys is simply like this. + * If there is no distinct expression, cachedKeys is simply like this. * cachedKeys[0] = [col0][col1] * * with two distict expression, union(tag:key) is attatched for each distinct expression @@ -134,6 +135,8 @@ protected transient long cntr = 1; protected transient long logEveryNRows = 0; + private int bucketingVersion; + /** Kryo ctor. */ protected ReduceSinkOperator() { super(); @@ -308,7 +311,7 @@ public void process(Object row, int tag) throws HiveException { } // Determine distKeyLength (w/o distincts), and then add the first if present. - populateCachedDistributionKeys(row, 0); + populateCachedDistributionKeys(row); // replace bucketing columns with hashcode % numBuckets int bucketNumber = -1; @@ -330,12 +333,18 @@ public void process(Object row, int tag) throws HiveException { final int hashCode; // distKeyLength doesn't include tag, but includes buckNum in cachedKeys[0] - if (useUniformHash && partitionEval.length > 0) { - hashCode = computeMurmurHash(firstKey); + LOG.info("UniformHash = " + useUniformHash + " bucketing version = " + bucketingVersion); + /*if (useUniformHash && partitionEval.length > 0) { + //hashCode = computeMurmurHash(firstKey.getBytes()); + hashCode = computeMurmurHashForBucketing(row); } else { - hashCode = computeHashCode(row, bucketNumber); - } - + if (partitionEval.length > 0 && bucketingVersion == 2) { + hashCode = computeMurmurHashForBucketing(row); + } else { + hashCode = computeHashCode(row, bucketNumber); + } + }*/ + hashCode = computeHashCode(row, bucketNumber); firstKey.setHashCode(hashCode); /* @@ -379,20 +388,25 @@ public void process(Object row, int tag) throws HiveException { } } - private int computeBucketNumber(Object row, int numBuckets) throws HiveException { + private int computeBucketNumber(Object row, int numBuckets) + throws HiveException, SerDeException { Object[] bucketFieldValues = new Object[bucketEval.length]; for (int i = 0; i < bucketEval.length; i++) { bucketFieldValues[i] = bucketEval[i].evaluate(row); } - return ObjectInspectorUtils.getBucketNumber(bucketFieldValues, bucketObjectInspectors, numBuckets); + if (bucketingVersion == 2) { + return ObjectInspectorUtils.getBucketNumber(bucketFieldValues, bucketObjectInspectors, numBuckets); + } else { + return ObjectInspectorUtils.getBucketNumberOld(bucketFieldValues, bucketObjectInspectors, numBuckets); + } } - private void populateCachedDistributionKeys(Object row, int index) throws HiveException { + private void populateCachedDistributionKeys(Object row) throws HiveException { for (int i = 0; i < numDistributionKeys; i++) { - cachedKeys[index][i] = keyEval[i].evaluate(row); + cachedKeys[0][i] = keyEval[i].evaluate(row); } if (cachedKeys[0].length > numDistributionKeys) { - cachedKeys[index][numDistributionKeys] = null; + cachedKeys[0][numDistributionKeys] = null; } } @@ -413,8 +427,8 @@ private void populateCachedDistinctKeys(Object row, int index) throws HiveExcept union.setTag((byte) index); } - protected final int computeMurmurHash(HiveKey firstKey) { - return hash.hash(firstKey.getBytes(), firstKey.getDistKeyLength(), 0); + protected final int computeMurmurHash(byte[] keyBytes) { + return Murmur3.hash32(keyBytes); } /** @@ -439,7 +453,11 @@ private int computeHashCode(Object row, int buckNum) throws HiveException { for(int i = 0; i < partitionEval.length; i++) { bucketFieldValues[i] = partitionEval[i].evaluate(row); } - keyHashCode = ObjectInspectorUtils.getBucketHashCode(bucketFieldValues, partitionObjectInspectors); + if (bucketingVersion == 2) { + keyHashCode = ObjectInspectorUtils.getBucketHashCode(bucketFieldValues, partitionObjectInspectors); + } else { + keyHashCode = ObjectInspectorUtils.getBucketHashCodeOld(bucketFieldValues, partitionObjectInspectors); + } } int hashCode = buckNum < 0 ? keyHashCode : keyHashCode * 31 + buckNum; if (LOG.isTraceEnabled()) { @@ -586,4 +604,12 @@ public String getReduceOutputName() { public void setOutputCollector(OutputCollector _out) { this.out = _out; } + + public void setBucketingVersion(int bucketingVersion) { + this.bucketingVersion = bucketingVersion; + } + + public int getBucketingVersion() { + return bucketingVersion; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/keyseries/VectorKeySeriesSerializedImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/keyseries/VectorKeySeriesSerializedImpl.java index 86f466fc4e..a3b076fa0a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/keyseries/VectorKeySeriesSerializedImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/keyseries/VectorKeySeriesSerializedImpl.java @@ -24,6 +24,7 @@ import org.apache.hive.common.util.HashCodeUtil; import com.google.common.base.Preconditions; +import org.apache.hive.common.util.Murmur3; /** * Implementation of base serialization interface. @@ -103,7 +104,7 @@ protected void computeSerializedHashCodes() { byte[] bytes = output.getData(); for (int i = 0; i < nonNullKeyCount; i++) { keyLength = serializedKeyLengths[i]; - hashCodes[i] = HashCodeUtil.murmurHash(bytes, offset, keyLength); + hashCodes[i] = Murmur3.hash32(bytes, offset, keyLength, 0); offset += keyLength; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java index cc876d54c1..75cc4610f1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkCommonOperator.java @@ -139,6 +139,9 @@ // Debug display. protected transient long batchCounter; + // Bucketing version + protected int bucketingVersion; + //--------------------------------------------------------------------------- /** Kryo ctor. */ @@ -448,4 +451,8 @@ public VectorizationContext getInputVectorizationContext() { public VectorDesc getVectorDesc() { return vectorDesc; } + + public void setBucketingVersion(int bucketingVersion) { + this.bucketingVersion = bucketingVersion; + } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java index 1bc3fdabac..53004b72a6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/reducesink/VectorReduceSinkObjectHashOperator.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.VectorDesc; +import org.apache.hadoop.hive.serde2.ByteStream; import org.apache.hadoop.hive.serde2.ByteStream.Output; import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -40,6 +41,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import com.google.common.base.Preconditions; +import org.apache.hive.common.util.Murmur3; /** * This class is the object hash (not Uniform Hash) operator class for native vectorized reduce sink. @@ -226,61 +228,110 @@ public void process(Object row, int tag) throws HiveException { int[] selected = batch.selected; final int size = batch.size; - for (int logical = 0; logical < size; logical++) { - final int batchIndex = (selectedInUse ? selected[logical] : logical); - - final int hashCode; - if (isEmptyBuckets) { - if (isEmptyPartitions) { - hashCode = nonPartitionRandom.nextInt(); - } else { + + // EmptyBuckets = true + if (isEmptyBuckets) { + if (isEmptyPartitions) { + for (int logical = 0; logical< size; logical++) { + final int batchIndex = (selectedInUse ? selected[logical] : logical); + final int hashCode = nonPartitionRandom.nextInt(); + postProcess(batch, batchIndex, tag, hashCode); + } + } else { // isEmptyPartition = false + for (int logical = 0; logical< size; logical++) { + final int batchIndex = (selectedInUse ? selected[logical] : logical); partitionVectorExtractRow.extractRow(batch, batchIndex, partitionFieldValues); - hashCode = + final int hashCode = bucketingVersion == 2 ? ObjectInspectorUtils.getBucketHashCode( + partitionFieldValues, partitionObjectInspectors) : + ObjectInspectorUtils.getBucketHashCodeOld( partitionFieldValues, partitionObjectInspectors); + postProcess(batch, batchIndex, tag, hashCode); } - } else { - bucketVectorExtractRow.extractRow(batch, batchIndex, bucketFieldValues); - final int bucketNum = - ObjectInspectorUtils.getBucketNumber( + } + } else { // EmptyBuckets = false + if (isEmptyPartitions) { + for (int logical = 0; logical< size; logical++) { + final int batchIndex = (selectedInUse ? selected[logical] : logical); + bucketVectorExtractRow.extractRow(batch, batchIndex, bucketFieldValues); + final int bucketNum = bucketingVersion == 2 ? + ObjectInspectorUtils.getBucketNumber(bucketFieldValues, + bucketObjectInspectors, numBuckets) : + ObjectInspectorUtils.getBucketNumberOld( bucketFieldValues, bucketObjectInspectors, numBuckets); - if (isEmptyPartitions) { - hashCode = nonPartitionRandom.nextInt() * 31 + bucketNum; - } else { + final int hashCode = nonPartitionRandom.nextInt() * 31 + bucketNum; + postProcess(batch, batchIndex, tag, hashCode); + } + } else { // isEmptyPartition = false + for (int logical = 0; logical< size; logical++) { + final int batchIndex = (selectedInUse ? selected[logical] : logical); partitionVectorExtractRow.extractRow(batch, batchIndex, partitionFieldValues); - hashCode = - ObjectInspectorUtils.getBucketHashCode( - partitionFieldValues, partitionObjectInspectors) * 31 + bucketNum; + bucketVectorExtractRow.extractRow(batch, batchIndex, bucketFieldValues); + final int hashCode, bucketNum; + if (bucketingVersion == 2) { + bucketNum = + ObjectInspectorUtils.getBucketNumber( + bucketFieldValues, bucketObjectInspectors, numBuckets); + hashCode = ObjectInspectorUtils.getBucketHashCode( + partitionFieldValues, partitionObjectInspectors) * 31 + bucketNum; + } else { // old bucketing logic + bucketNum = + ObjectInspectorUtils.getBucketNumberOld( + bucketFieldValues, bucketObjectInspectors, numBuckets); + hashCode = ObjectInspectorUtils.getBucketHashCodeOld( + partitionFieldValues, partitionObjectInspectors) * 31 + bucketNum; + } + postProcess(batch, batchIndex, tag, hashCode); } } + } + } catch (Exception e) { + throw new HiveException(e); + } + } - if (!isEmptyKey) { - keyBinarySortableSerializeWrite.reset(); - keyVectorSerializeRow.serializeWrite(batch, batchIndex); - - // One serialized key for 1 or more rows for the duplicate keys. - final int keyLength = keyOutput.getLength(); - if (tag == -1 || reduceSkipTag) { - keyWritable.set(keyOutput.getData(), 0, keyLength); - } else { - keyWritable.setSize(keyLength + 1); - System.arraycopy(keyOutput.getData(), 0, keyWritable.get(), 0, keyLength); - keyWritable.get()[keyLength] = reduceTagByte; - } - keyWritable.setDistKeyLength(keyLength); - } + private void processKey(VectorizedRowBatch batch, int batchIndex, int tag) + throws HiveException{ + if (isEmptyKey) return; - keyWritable.setHashCode(hashCode); + try { + keyBinarySortableSerializeWrite.reset(); + keyVectorSerializeRow.serializeWrite(batch, batchIndex); + + // One serialized key for 1 or more rows for the duplicate keys. + final int keyLength = keyOutput.getLength(); + if (tag == -1 || reduceSkipTag) { + keyWritable.set(keyOutput.getData(), 0, keyLength); + } else { + keyWritable.setSize(keyLength + 1); + System.arraycopy(keyOutput.getData(), 0, keyWritable.get(), 0, keyLength); + keyWritable.get()[keyLength] = reduceTagByte; + } + keyWritable.setDistKeyLength(keyLength); + } catch (Exception e) { + throw new HiveException(e); + } + } - if (!isEmptyValue) { - valueLazyBinarySerializeWrite.reset(); - valueVectorSerializeRow.serializeWrite(batch, batchIndex); + private void processValue(VectorizedRowBatch batch, int batchIndex) throws HiveException { + if (isEmptyValue) return; - valueBytesWritable.set(valueOutput.getData(), 0, valueOutput.getLength()); - } + try { + valueLazyBinarySerializeWrite.reset(); + valueVectorSerializeRow.serializeWrite(batch, batchIndex); - collect(keyWritable, valueBytesWritable); - } + valueBytesWritable.set(valueOutput.getData(), 0, valueOutput.getLength()); + } catch (Exception e) { + throw new HiveException(e); + } + } + + private void postProcess(VectorizedRowBatch batch, int batchIndex, int tag, int hashCode) throws HiveException { + try { + processKey(batch, batchIndex, tag); + keyWritable.setHashCode(hashCode); + processValue(batch, batchIndex); + collect(keyWritable, valueBytesWritable); } catch (Exception e) { throw new HiveException(e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java index a51fdd322f..694ec9504b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java @@ -188,6 +188,9 @@ public void setTTable(org.apache.hadoop.hive.metastore.api.Table tTable) { // set create time t.setCreateTime((int) (System.currentTimeMillis() / 1000)); } + // Explictly set the bucketing version + t.getParameters().put(hive_metastoreConstants.TABLE_BUCKETING_VERSION, + "2"); return t; } @@ -450,6 +453,19 @@ public StructField getField(String fld) { } } + public int getBucketingVersion() { + String versionStr = getProperty(hive_metastoreConstants.TABLE_BUCKETING_VERSION); + int bucketingVersion = 1; + if (versionStr != null) { + try { + bucketingVersion = Integer.parseInt(versionStr); + } catch (NumberFormatException e) { + // Do nothing + } + } + return bucketingVersion; + } + @Override public String toString() { return tTable.getTableName(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index a235f3fbf4..3b7a410e82 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.Stack; +import com.google.common.base.Preconditions; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -180,7 +181,8 @@ MapJoinOperator mapJoinOp = convertJoinMapJoin(joinOp, context, mapJoinConversionPos, true); // map join operator by default has no bucket cols and num of reduce sinks // reduced by 1 - mapJoinOp.setOpTraits(new OpTraits(null, -1, null, joinOp.getOpTraits().getNumReduceSinks())); + mapJoinOp.setOpTraits(new OpTraits(null, -1, null, joinOp.getOpTraits().getNumReduceSinks(), + joinOp.getOpTraits().getBucketingVersion())); mapJoinOp.setStatistics(joinOp.getStatistics()); // propagate this change till the next RS for (Operator childOp : mapJoinOp.getChildOperators()) { @@ -378,7 +380,7 @@ private void convertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcContext cont joinOp.getSchema()); int numReduceSinks = joinOp.getOpTraits().getNumReduceSinks(); OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, - joinOp.getOpTraits().getSortCols(), numReduceSinks); + joinOp.getOpTraits().getSortCols(), numReduceSinks, joinOp.getOpTraits().getBucketingVersion()); mergeJoinOp.setOpTraits(opTraits); mergeJoinOp.setStatistics(joinOp.getStatistics()); @@ -445,7 +447,8 @@ private void setAllChildrenTraits(Operator currentOp, Op return; } currentOp.setOpTraits(new OpTraits(opTraits.getBucketColNames(), - opTraits.getNumBuckets(), opTraits.getSortCols(), opTraits.getNumReduceSinks())); + opTraits.getNumBuckets(), opTraits.getSortCols(), opTraits.getNumReduceSinks(), + opTraits.getBucketingVersion())); for (Operator childOp : currentOp.getChildOperators()) { if ((childOp instanceof ReduceSinkOperator) || (childOp instanceof GroupByOperator)) { break; @@ -498,7 +501,8 @@ private boolean convertJoinBucketMapJoin(JoinOperator joinOp, OptimizeTezProcCon // we can set the traits for this join operator opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(), - tezBucketJoinProcCtx.getNumBuckets(), null, joinOp.getOpTraits().getNumReduceSinks()); + tezBucketJoinProcCtx.getNumBuckets(), null, joinOp.getOpTraits().getNumReduceSinks(), + joinOp.getOpTraits().getBucketingVersion()); mapJoinOp.setOpTraits(opTraits); mapJoinOp.setStatistics(joinOp.getStatistics()); setNumberOfBucketsOnChildren(mapJoinOp); @@ -594,6 +598,38 @@ private boolean checkConvertJoinSMBJoin(JoinOperator joinOp, OptimizeTezProcCont numBuckets = bigTableRS.getConf().getNumReducers(); } tezBucketJoinProcCtx.setNumBuckets(numBuckets); + + // With bucketing using two different versions. Version 1 for exiting + // tables and version 2 for new tables. All the inputs to the SMB must be + // from same version. This only applies to tables read directly and not + // intermediate outputs of joins/groupbys + int bucketingVersion = -1; + for (Operator parentOp : joinOp.getParentOperators()) { + // Check if the parent is coming from a table scan, if so, what is the version of it. + assert parentOp.getParentOperators() != null && parentOp.getParentOperators().size() == 1; + Operator op = parentOp.getParentOperators().get(0); + while(op != null && !(op instanceof TableScanOperator + || op instanceof ReduceSinkOperator + || op instanceof CommonJoinOperator)) { + // If op has parents it is guaranteed to be 1. + List> parents = op.getParentOperators(); + Preconditions.checkState(parents.size() == 0 || parents.size() == 1); + op = parents.size() == 1 ? parents.get(0) : null; + } + + if (op instanceof TableScanOperator) { + int localVersion = ((TableScanOperator)op).getConf(). + getTableMetadata().getBucketingVersion(); + if (bucketingVersion == -1) { + bucketingVersion = localVersion; + } else if (bucketingVersion != localVersion) { + // versions dont match, return false. + LOG.debug("SMB Join can't be performed due to bucketing version mismatch"); + return false; + } + } + } + LOG.info("We can convert the join to an SMB join."); return true; } @@ -1169,7 +1205,8 @@ private boolean convertJoinDynamicPartitionedHashJoin(JoinOperator joinOp, Optim joinOp.getOpTraits().getBucketColNames(), numReducers, null, - joinOp.getOpTraits().getNumReduceSinks()); + joinOp.getOpTraits().getNumReduceSinks(), + joinOp.getOpTraits().getBucketingVersion()); mapJoinOp.setOpTraits(opTraits); mapJoinOp.setStatistics(joinOp.getStatistics()); // propagate this change till the next RS diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java index 5f65f638ca..d1672b229e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FixedBucketPruningOptimizer.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hive.ql.parse.PrunedPartitionList; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; +import org.apache.hadoop.hive.serde2.SerDeException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter; @@ -84,8 +85,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, @Override protected void generatePredicate(NodeProcessorCtx procCtx, - FilterOperator fop, TableScanOperator top) throws SemanticException, - UDFArgumentException { + FilterOperator fop, TableScanOperator top) throws SemanticException { FixedBucketPruningOptimizerCtxt ctxt = ((FixedBucketPruningOptimizerCtxt) procCtx); Table tbl = top.getConf().getTableMetadata(); if (tbl.getNumBuckets() > 0) { @@ -122,8 +122,7 @@ protected void generatePredicate(NodeProcessorCtx procCtx, @Override protected void generatePredicate(NodeProcessorCtx procCtx, - FilterOperator fop, TableScanOperator top) throws SemanticException, - UDFArgumentException { + FilterOperator fop, TableScanOperator top) throws SemanticException { FixedBucketPruningOptimizerCtxt ctxt = ((FixedBucketPruningOptimizerCtxt) procCtx); if (ctxt.getNumBuckets() <= 0 || ctxt.getBucketCols().size() != 1) { // bucketing isn't consistent or there are >1 bucket columns @@ -225,6 +224,9 @@ protected void generatePredicate(NodeProcessorCtx procCtx, bs.clear(); PrimitiveObjectInspector bucketOI = (PrimitiveObjectInspector)bucketField.getFieldObjectInspector(); PrimitiveObjectInspector constOI = PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(bucketOI.getPrimitiveCategory()); + // Fetch the bucketing version from table scan operator + int bucketingVersion = top.getConf().getTableMetadata().getBucketingVersion(); + for (Object literal: literals) { PrimitiveObjectInspector origOI = PrimitiveObjectInspectorFactory.getPrimitiveObjectInspectorFromClass(literal.getClass()); Converter conv = ObjectInspectorConverters.getConverter(origOI, constOI); @@ -233,10 +235,12 @@ protected void generatePredicate(NodeProcessorCtx procCtx, return; } Object convCols[] = new Object[] {conv.convert(literal)}; - int n = ObjectInspectorUtils.getBucketNumber(convCols, new ObjectInspector[]{constOI}, ctxt.getNumBuckets()); + int n = bucketingVersion == 2 ? + ObjectInspectorUtils.getBucketNumber(convCols, new ObjectInspector[]{constOI}, ctxt.getNumBuckets()) : + ObjectInspectorUtils.getBucketNumberOld(convCols, new ObjectInspector[]{constOI}, ctxt.getNumBuckets()); bs.set(n); - if (ctxt.isCompat()) { - int h = ObjectInspectorUtils.getBucketHashCode(convCols, new ObjectInspector[]{constOI}); + if (bucketingVersion == 1 && ctxt.isCompat()) { + int h = ObjectInspectorUtils.getBucketHashCodeOld(convCols, new ObjectInspector[]{constOI}); // -ve hashcodes had conversion to positive done in different ways in the past // abs() is now obsolete and all inserts now use & Integer.MAX_VALUE // the compat mode assumes that old data could've been loaded using the other conversion diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java index 2be3c9b9a2..1626e26782 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; +import org.apache.hadoop.hive.serde2.SerDeException; /** * Operator factory for pruning processing of operator graph We find @@ -101,7 +102,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, * @throws UDFArgumentException */ protected abstract void generatePredicate(NodeProcessorCtx procCtx, FilterOperator fop, - TableScanOperator top) throws SemanticException, UDFArgumentException; + TableScanOperator top) throws SemanticException; /** * Add pruning predicate. * diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java index 3c8e61d47b..e6c5b9d9ce 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/AnnotateWithOpTraits.java @@ -22,16 +22,7 @@ import java.util.LinkedHashMap; import java.util.Map; -import org.apache.hadoop.hive.ql.exec.DemuxOperator; -import org.apache.hadoop.hive.ql.exec.GroupByOperator; -import org.apache.hadoop.hive.ql.exec.JoinOperator; -import org.apache.hadoop.hive.ql.exec.MapJoinOperator; -import org.apache.hadoop.hive.ql.exec.MuxOperator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator; -import org.apache.hadoop.hive.ql.exec.SelectOperator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; -import org.apache.hadoop.hive.ql.exec.UnionOperator; +import org.apache.hadoop.hive.ql.exec.*; import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher; import org.apache.hadoop.hive.ql.lib.Dispatcher; import org.apache.hadoop.hive.ql.lib.GraphWalker; @@ -78,6 +69,8 @@ public ParseContext transform(ParseContext pctx) throws SemanticException { OpTraitsRulesProcFactory.getGroupByRule()); opRules.put(new RuleRegExp("SEL", SelectOperator.getOperatorName() + "%"), OpTraitsRulesProcFactory.getSelectRule()); + /*opRules.put(new RuleRegExp("FS", FileSinkOperator.getOperatorName() + "%"), + OpTraitsRulesProcFactory.getFileSinkRule());*/ // The dispatcher fires the processor corresponding to the closest matching // rule and passes the context along diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java index 69d9f3125a..614511238d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java @@ -23,12 +23,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.ql.exec.GroupByOperator; -import org.apache.hadoop.hive.ql.exec.JoinOperator; -import org.apache.hadoop.hive.ql.exec.Operator; -import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; -import org.apache.hadoop.hive.ql.exec.SelectOperator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; +import org.apache.hadoop.hive.ql.exec.*; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; @@ -92,10 +87,12 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, List> listBucketCols = new ArrayList>(); int numBuckets = -1; int numReduceSinks = 1; + int bucketingVersion = -1; OpTraits parentOpTraits = rs.getParentOperators().get(0).getOpTraits(); if (parentOpTraits != null) { numBuckets = parentOpTraits.getNumBuckets(); numReduceSinks += parentOpTraits.getNumReduceSinks(); + bucketingVersion = parentOpTraits.getBucketingVersion(); } List bucketCols = new ArrayList<>(); @@ -134,8 +131,10 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, } listBucketCols.add(bucketCols); - OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listBucketCols, numReduceSinks); + OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, + listBucketCols, numReduceSinks, bucketingVersion); rs.setOpTraits(opTraits); + rs.setBucketingVersion(bucketingVersion); return null; } } @@ -213,7 +212,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, sortedColsList.add(sortCols); } // num reduce sinks hardcoded to 0 because TS has no parents - OpTraits opTraits = new OpTraits(bucketColsList, numBuckets, sortedColsList, 0); + OpTraits opTraits = new OpTraits(bucketColsList, numBuckets, + sortedColsList, 0, table.getBucketingVersion()); ts.setOpTraits(opTraits); return null; } @@ -239,12 +239,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, List> listBucketCols = new ArrayList>(); int numReduceSinks = 0; + int bucketingVersion = -1; OpTraits parentOpTraits = gbyOp.getParentOperators().get(0).getOpTraits(); if (parentOpTraits != null) { numReduceSinks = parentOpTraits.getNumReduceSinks(); + bucketingVersion = parentOpTraits.getBucketingVersion(); } listBucketCols.add(gbyKeys); - OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols, numReduceSinks); + OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols, + numReduceSinks, bucketingVersion); gbyOp.setOpTraits(opTraits); return null; } @@ -298,12 +301,15 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, int numBuckets = -1; int numReduceSinks = 0; + int bucketingVersion = -1; OpTraits parentOpTraits = selOp.getParentOperators().get(0).getOpTraits(); if (parentOpTraits != null) { numBuckets = parentOpTraits.getNumBuckets(); numReduceSinks = parentOpTraits.getNumReduceSinks(); + bucketingVersion = parentOpTraits.getBucketingVersion(); } - OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols, numReduceSinks); + OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols, + numReduceSinks, bucketingVersion); selOp.setOpTraits(opTraits); return null; } @@ -338,7 +344,7 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, pos++; } - joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList, numReduceSinks)); + joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList, numReduceSinks, 2)); return null; } @@ -392,6 +398,8 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, Operator operator = (Operator) nd; int numReduceSinks = 0; + int bucketingVersion = -1; + boolean bucketingVersionSeen = false; for (Operator parentOp : operator.getParentOperators()) { if (parentOp.getOpTraits() == null) { continue; @@ -399,8 +407,17 @@ public Object process(Node nd, Stack stack, NodeProcessorCtx procCtx, if (parentOp.getOpTraits().getNumReduceSinks() > numReduceSinks) { numReduceSinks = parentOp.getOpTraits().getNumReduceSinks(); } + // If there is mismatch in bucketingVersion, then it should be set to + // -1, that way SMB will be disabled. + if (bucketingVersion == -1 && !bucketingVersionSeen) { + bucketingVersion = parentOp.getOpTraits().getBucketingVersion(); + bucketingVersionSeen = true; + } else if (bucketingVersion != parentOp.getOpTraits().getBucketingVersion()) { + bucketingVersion = -1; + } } - OpTraits opTraits = new OpTraits(null, -1, null, numReduceSinks); + OpTraits opTraits = new OpTraits(null, -1, + null, numReduceSinks, bucketingVersion); operator.setOpTraits(opTraits); return null; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java index 13a2fc478f..ae36a967da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java @@ -40,6 +40,7 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.hadoop.hive.ql.exec.vector.reducesink.*; import org.apache.hadoop.hive.ql.parse.spark.SparkPartitionPruningSinkOperator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,11 +69,6 @@ import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinOuterMultiKeyOperator; import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinOuterStringOperator; import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFOperator; -import org.apache.hadoop.hive.ql.exec.vector.reducesink.VectorReduceSinkEmptyKeyOperator; -import org.apache.hadoop.hive.ql.exec.vector.reducesink.VectorReduceSinkLongOperator; -import org.apache.hadoop.hive.ql.exec.vector.reducesink.VectorReduceSinkMultiKeyOperator; -import org.apache.hadoop.hive.ql.exec.vector.reducesink.VectorReduceSinkObjectHashOperator; -import org.apache.hadoop.hive.ql.exec.vector.reducesink.VectorReduceSinkStringOperator; import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type; @@ -3566,6 +3562,9 @@ private boolean canSpecializeMapJoin(Operator op, MapJoi LOG.info("Vectorizer vectorizeOperator reduce sink class " + opClass.getSimpleName()); + // Get the bucketing version + int bucketingVersion = ((ReduceSinkOperator)op).getBucketingVersion(); + Operator vectorOp = null; try { vectorOp = OperatorFactory.getVectorOperator( @@ -3577,6 +3576,10 @@ private boolean canSpecializeMapJoin(Operator op, MapJoi throw new HiveException(e); } + // Set the bucketing version + Preconditions.checkArgument(vectorOp instanceof VectorReduceSinkCommonOperator); + ((VectorReduceSinkCommonOperator)vectorOp).setBucketingVersion(bucketingVersion); + return vectorOp; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java index 7b1fd5f206..8e75db9e08 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java @@ -121,7 +121,8 @@ } // we can set the traits for this join operator - OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null, joinOp.getOpTraits().getNumReduceSinks()); + OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null, + joinOp.getOpTraits().getNumReduceSinks(), joinOp.getOpTraits().getBucketingVersion()); mapJoinOp.setOpTraits(opTraits); mapJoinOp.setStatistics(joinOp.getStatistics()); setNumberOfBucketsOnChildren(mapJoinOp); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d1609e1186..372a502ed3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -219,12 +219,8 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.ResourceType; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator; +import org.apache.hadoop.hive.ql.udf.generic.*; import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCardinalityViolation; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; -import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF; import org.apache.hadoop.hive.ql.util.ResourceDownloader; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.Deserializer; @@ -10480,7 +10476,8 @@ private Operator genUnionPlan(String unionalias, String leftalias, */ private ExprNodeDesc genSamplePredicate(TableSample ts, List bucketCols, boolean useBucketCols, String alias, - RowResolver rwsch, QBMetaData qbm, ExprNodeDesc planExpr) + RowResolver rwsch, QBMetaData qbm, ExprNodeDesc planExpr, + int bucketingVersion) throws SemanticException { ExprNodeDesc numeratorExpr = new ExprNodeConstantDesc( @@ -10510,22 +10507,19 @@ private ExprNodeDesc genSamplePredicate(TableSample ts, ExprNodeDesc equalsExpr = null; { ExprNodeDesc hashfnExpr = new ExprNodeGenericFuncDesc( - TypeInfoFactory.intTypeInfo, new GenericUDFHash(), args); - assert (hashfnExpr != null); + TypeInfoFactory.intTypeInfo, + bucketingVersion == 2 ? new GenericUDFMurmurHash() : new GenericUDFHash(), args); LOG.info("hashfnExpr = " + hashfnExpr); ExprNodeDesc andExpr = TypeCheckProcFactory.DefaultExprProcessor .getFuncExprNodeDesc("&", hashfnExpr, intMaxExpr); - assert (andExpr != null); LOG.info("andExpr = " + andExpr); ExprNodeDesc modExpr = TypeCheckProcFactory.DefaultExprProcessor .getFuncExprNodeDesc("%", andExpr, denominatorExpr); - assert (modExpr != null); LOG.info("modExpr = " + modExpr); LOG.info("numeratorExpr = " + numeratorExpr); equalsExpr = TypeCheckProcFactory.DefaultExprProcessor .getFuncExprNodeDesc("==", modExpr, numeratorExpr); LOG.info("equalsExpr = " + equalsExpr); - assert (equalsExpr != null); } return equalsExpr; } @@ -10694,7 +10688,8 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { // later LOG.info("No need for sample filter"); ExprNodeDesc samplePredicate = genSamplePredicate(ts, tabBucketCols, - colsEqual, alias, rwsch, qb.getMetaData(), null); + colsEqual, alias, rwsch, qb.getMetaData(), null, + tab.getBucketingVersion()); FilterDesc filterDesc = new FilterDesc( samplePredicate, true, new SampleDesc(ts.getNumerator(), ts.getDenominator(), tabBucketCols, true)); @@ -10706,7 +10701,8 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { // create tableOp to be filterDesc and set as child to 'top' LOG.info("Need sample filter"); ExprNodeDesc samplePredicate = genSamplePredicate(ts, tabBucketCols, - colsEqual, alias, rwsch, qb.getMetaData(), null); + colsEqual, alias, rwsch, qb.getMetaData(), null, + tab.getBucketingVersion()); FilterDesc filterDesc = new FilterDesc(samplePredicate, true); filterDesc.setGenerated(true); op = OperatorFactory.getAndMakeChild(filterDesc, @@ -10737,7 +10733,8 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { tsSample.setInputPruning(true); qb.getParseInfo().setTabSample(alias, tsSample); ExprNodeDesc samplePred = genSamplePredicate(tsSample, tab - .getBucketCols(), true, alias, rwsch, qb.getMetaData(), null); + .getBucketCols(), true, alias, rwsch, qb.getMetaData(), null, + tab.getBucketingVersion()); FilterDesc filterDesc = new FilterDesc(samplePred, true, new SampleDesc(tsSample.getNumerator(), tsSample .getDenominator(), tab.getBucketCols(), true)); @@ -10756,7 +10753,7 @@ private Operator genTablePlan(String alias, QB qb) throws SemanticException { .getFuncExprNodeDesc("rand", new ExprNodeConstantDesc(Integer .valueOf(460476415))); ExprNodeDesc samplePred = genSamplePredicate(tsSample, null, false, - alias, rwsch, qb.getMetaData(), randFunc); + alias, rwsch, qb.getMetaData(), randFunc, tab.getBucketingVersion()); FilterDesc filterDesc = new FilterDesc(samplePred, true); filterDesc.setGenerated(true); op = OperatorFactory.getAndMakeChild(filterDesc, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java index 9621c3be53..d3b62ce799 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java @@ -22,17 +22,20 @@ public class OpTraits { - List> bucketColNames; - List> sortColNames; - int numBuckets; - int numReduceSinks; + private List> bucketColNames; + private List> sortColNames; + private int numBuckets; + private int numReduceSinks; + private int bucketingVersion; public OpTraits(List> bucketColNames, int numBuckets, - List> sortColNames, int numReduceSinks) { + List> sortColNames, int numReduceSinks, + int bucketingVersion) { this.bucketColNames = bucketColNames; this.numBuckets = numBuckets; this.sortColNames = sortColNames; this.numReduceSinks = numReduceSinks; + this.bucketingVersion = bucketingVersion; } public List> getBucketColNames() { @@ -68,10 +71,17 @@ public int getNumReduceSinks() { return this.numReduceSinks; } - + public void setBucketingVersion(int bucketingVersion) { + this.bucketingVersion = bucketingVersion; + } + + public int getBucketingVersion() { + return bucketingVersion; + } + @Override public String toString() { return "{ bucket column names: " + bucketColNames + "; sort column names: " - + sortColNames + "; bucket count: " + numBuckets + " }"; + + sortColNames + "; bucket count: " + numBuckets + "; bucketing version: " + bucketingVersion + " }"; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java index 25b91899de..64a8d32988 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableDesc.java @@ -184,6 +184,20 @@ public boolean isNonNative() { return (properties.getProperty(hive_metastoreConstants.META_TABLE_STORAGE) != null); } + // This code exact copy of getBucketingVersion defined in metadata Table. + public int getBucketingVersion() { + String versionStr = properties.getProperty(hive_metastoreConstants.TABLE_BUCKETING_VERSION); + int bucketingVersion = 1; + if (versionStr != null) { + try { + bucketingVersion = Integer.parseInt(versionStr); + } catch (NumberFormatException e) { + // Do nothing + } + } + return bucketingVersion; + } + @Override public Object clone() { TableDesc ret = new TableDesc(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFHash.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFHash.java index 7cd571815d..1a758436fa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFHash.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFHash.java @@ -29,6 +29,7 @@ /** * GenericUDF Class for computing hash values. */ +@Deprecated @Description(name = "hash", value = "_FUNC_(a1, a2, ...) - Returns a hash value of the arguments") public class GenericUDFHash extends GenericUDF { private transient ObjectInspector[] argumentOIs; @@ -48,7 +49,7 @@ public Object evaluate(DeferredObject[] arguments) throws HiveException { for(int i = 0; i < arguments.length; i++) { fieldValues[i] = arguments[i].get(); } - int r = ObjectInspectorUtils.getBucketHashCode(fieldValues, argumentOIs); + int r = ObjectInspectorUtils.getBucketHashCodeOld(fieldValues, argumentOIs); result.set(r); return result; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMurmurHash.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMurmurHash.java new file mode 100644 index 0000000000..a101795892 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFMurmurHash.java @@ -0,0 +1,46 @@ +package org.apache.hadoop.hive.ql.udf.generic; + +import org.apache.hadoop.hive.ql.exec.Description; +import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.SerDeException; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; +import org.apache.hadoop.io.IntWritable; +import org.apache.hive.common.util.Murmur3; + +/** + * GenericUDF Class for computing murmurhash values. + */ +@Description(name = "hash", value = "_FUNC_(a1, a2, ...) - Returns a hash value of the arguments") +public class GenericUDFMurmurHash extends GenericUDF { + private transient ObjectInspector[] argumentOIs; + + @Override + public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentTypeException { + + argumentOIs = arguments; + return PrimitiveObjectInspectorFactory.writableIntObjectInspector; + } + + private final IntWritable result = new IntWritable(); + + @Override + public Object evaluate(DeferredObject[] arguments) throws HiveException { + Object[] fieldValues = new Object[arguments.length]; + for(int i = 0; i < arguments.length; i++) { + fieldValues[i] = arguments[i].get(); + } + int r = ObjectInspectorUtils.getBucketHashCode(fieldValues, argumentOIs); + result.set(r); + return result; + } + + @Override + public String getDisplayString(String[] children) { + return getStandardDisplayString("hash", children, ","); + } + +} diff --git a/ql/src/test/queries/clientpositive/insert_update_delete.q b/ql/src/test/queries/clientpositive/insert_update_delete.q index fb00346a69..0b0b913728 100644 --- a/ql/src/test/queries/clientpositive/insert_update_delete.q +++ b/ql/src/test/queries/clientpositive/insert_update_delete.q @@ -2,13 +2,18 @@ set hive.mapred.mode=nonstrict; set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; - +--create table acid_iud(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc; +--create table acid_iud(a int, b varchar(128)) stored as orc TBLPROPERTIES ('transactional'='true'); create table acid_iud(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true'); +explain select * from src, srcpart where src.key = srcpart.key; insert into table acid_iud select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10; +explain select * from src, srcpart where src.key = srcpart.key; select a,b from acid_iud order by a; +explain select * from src, srcpart where src.key = srcpart.key; +explain update acid_iud set b = 'fred'; update acid_iud set b = 'fred'; select a,b from acid_iud order by a; diff --git a/ql/src/test/queries/clientpositive/schema_evol_orc_acid_table_update_llap_io.q b/ql/src/test/queries/clientpositive/schema_evol_orc_acid_table_update_llap_io.q index b72ded6bd0..4ca1aa93a7 100644 --- a/ql/src/test/queries/clientpositive/schema_evol_orc_acid_table_update_llap_io.q +++ b/ql/src/test/queries/clientpositive/schema_evol_orc_acid_table_update_llap_io.q @@ -44,10 +44,13 @@ alter table table5 add columns(c int, d string); insert into table table5 SELECT insert_num, int1, 'new', int1, string1 FROM schema_evolution_data_2; select a,b,c,d from table5; +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/table5; -- UPDATE New Columns +explain update table5 set c=99; update table5 set c=99; +dfs -lsr ${hiveconf:hive.metastore.warehouse.dir}/table5; select a,b,c,d from table5; alter table table5 compact 'major'; diff --git a/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_3.q b/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_3.q index ff4cde2c30..3db1563f88 100644 --- a/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_3.q +++ b/ql/src/test/queries/clientpositive/tez_dynpart_hashjoin_3.q @@ -14,8 +14,8 @@ limit 1; set hive.auto.convert.join=true; set hive.optimize.dynamic.partition.hashjoin=true; -set hive.auto.convert.join.noconditionaltask.size=200000; -set hive.exec.reducers.bytes.per.reducer=200000; +set hive.auto.convert.join.noconditionaltask.size=20000; +set hive.exec.reducers.bytes.per.reducer=2000; explain select a.* diff --git a/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out b/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out index 6a5ace0af6..58257a4c47 100644 --- a/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/acid_bucket_pruning.q.out @@ -44,8 +44,8 @@ STAGE PLANS: TableScan alias: acidtbldefault filterExpr: (a = 1) (type: boolean) - buckets included: [1,] of 16 - Statistics: Num rows: 1765 Data size: 6712 Basic stats: COMPLETE Column stats: NONE + buckets included: [5,] of 16 + Statistics: Num rows: 1839 Data size: 6996 Basic stats: COMPLETE Column stats: NONE GatherStats: false Filter Operator isSamplingPred: false @@ -90,6 +90,7 @@ STAGE PLANS: properties: bucket_count 16 bucket_field_name a + bucketing_version 2 column.name.delimiter , columns a columns.comments @@ -101,7 +102,7 @@ STAGE PLANS: serialization.ddl struct acidtbldefault { i32 a} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 33194 + totalSize 34578 transactional true transactional_properties default #### A masked pattern was here #### @@ -112,6 +113,7 @@ STAGE PLANS: properties: bucket_count 16 bucket_field_name a + bucketing_version 2 column.name.delimiter , columns a columns.comments @@ -123,7 +125,7 @@ STAGE PLANS: serialization.ddl struct acidtbldefault { i32 a} serialization.format 1 serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde - totalSize 33194 + totalSize 34578 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out b/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out index c87a0a64f2..c8b831df79 100644 --- a/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out +++ b/ql/src/test/results/clientpositive/llap/alter_merge_stats_orc.q.out @@ -89,6 +89,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + bucketing_version 2 numFiles 3 numRows 1500 rawDataSize 141000 @@ -139,6 +140,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 1500 rawDataSize 141000 diff --git a/ql/src/test/results/clientpositive/llap/autoColumnStats_10.q.out b/ql/src/test/results/clientpositive/llap/autoColumnStats_10.q.out index 9ae59e6943..6634966edf 100644 --- a/ql/src/test/results/clientpositive/llap/autoColumnStats_10.q.out +++ b/ql/src/test/results/clientpositive/llap/autoColumnStats_10.q.out @@ -29,6 +29,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + bucketing_version 2 numFiles 0 numRows 0 rawDataSize 0 @@ -75,6 +76,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + bucketing_version 2 numFiles 1 numRows 1 rawDataSize 8 @@ -118,6 +120,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} + bucketing_version 2 #### A masked pattern was here #### numFiles 1 numRows 1 @@ -203,6 +206,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"insert_num\":\"true\"}} + bucketing_version 2 #### A masked pattern was here #### numFiles 2 numRows 2 @@ -293,6 +297,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"c1\":\"true\",\"c2\":\"true\",\"insert_num\":\"true\"}} + bucketing_version 2 numFiles 0 numRows 0 rawDataSize 0 @@ -339,6 +344,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 1 rawDataSize 8 @@ -382,6 +388,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 #### A masked pattern was here #### numFiles 1 numRows 1 @@ -467,6 +474,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 #### A masked pattern was here #### numFiles 2 numRows 2 diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out index 2e02499ebd..daa8d6a15a 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_1.q.out @@ -165,6 +165,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -252,6 +253,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -300,6 +302,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -439,6 +442,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -526,6 +530,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -574,6 +579,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -713,6 +719,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -800,6 +807,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -848,6 +856,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out index e8973a3198..81f9f87b8f 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_11.q.out @@ -175,6 +175,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -253,6 +254,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -302,6 +304,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -465,6 +468,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -552,6 +556,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -600,6 +605,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -735,6 +741,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -818,6 +825,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -866,6 +874,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1001,6 +1010,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1063,6 +1073,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1111,6 +1122,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1197,6 +1209,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1245,6 +1258,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out index 2c7decd16f..5ce2d4057d 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_12.q.out @@ -202,6 +202,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 3 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -268,6 +269,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -351,6 +353,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -399,6 +402,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -470,6 +474,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 3 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out index 054b0d00be..942738dae1 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_2.q.out @@ -147,6 +147,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -234,6 +235,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -282,6 +284,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -421,6 +424,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -508,6 +512,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -556,6 +561,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out index 5c3fb0fc02..b389dcc639 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_3.q.out @@ -147,6 +147,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -195,6 +196,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -283,6 +285,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -421,6 +424,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -469,6 +473,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -557,6 +562,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -695,6 +701,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -743,6 +750,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -831,6 +839,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out index 95d329862c..612f3ead58 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_4.q.out @@ -163,6 +163,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -211,6 +212,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -299,6 +301,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -437,6 +440,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -485,6 +489,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -573,6 +578,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -711,6 +717,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -759,6 +766,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -847,6 +855,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out index bbe1252b64..c56c82c977 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_5.q.out @@ -103,6 +103,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -125,6 +126,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -189,6 +191,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -211,6 +214,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -320,6 +324,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -342,6 +347,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -406,6 +412,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -428,6 +435,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -537,6 +545,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -559,6 +568,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -623,6 +633,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -645,6 +656,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out index 53c685cb11..b8923760b8 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_7.q.out @@ -180,6 +180,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -228,6 +229,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -316,6 +318,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -364,6 +367,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -505,6 +509,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -553,6 +558,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -641,6 +647,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -689,6 +696,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -830,6 +838,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -878,6 +887,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -966,6 +976,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1014,6 +1025,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out index 80116eca07..4d10634a8c 100644 --- a/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out +++ b/ql/src/test/results/clientpositive/llap/auto_sortmerge_join_8.q.out @@ -180,6 +180,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -228,6 +229,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -316,6 +318,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -364,6 +367,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -505,6 +509,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -553,6 +558,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -641,6 +647,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -689,6 +696,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -830,6 +838,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -878,6 +887,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -966,6 +976,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1014,6 +1025,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/bucket2.q.out b/ql/src/test/results/clientpositive/llap/bucket2.q.out index 10fc171ae1..e0dce94987 100644 --- a/ql/src/test/results/clientpositive/llap/bucket2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket2.q.out @@ -59,6 +59,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -80,6 +81,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -121,6 +123,7 @@ STAGE PLANS: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -191,6 +194,7 @@ STAGE PLANS: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/bucket3.q.out b/ql/src/test/results/clientpositive/llap/bucket3.q.out index 9ae5166ded..05404c7de9 100644 --- a/ql/src/test/results/clientpositive/llap/bucket3.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket3.q.out @@ -60,6 +60,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -81,6 +82,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -122,6 +124,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -211,6 +214,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/bucket4.q.out b/ql/src/test/results/clientpositive/llap/bucket4.q.out index 0a1c49797b..9260877693 100644 --- a/ql/src/test/results/clientpositive/llap/bucket4.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket4.q.out @@ -60,6 +60,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -81,6 +82,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -123,6 +125,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -194,6 +197,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/bucket5.q.out b/ql/src/test/results/clientpositive/llap/bucket5.q.out index 6649344727..dd5f9a423e 100644 --- a/ql/src/test/results/clientpositive/llap/bucket5.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket5.q.out @@ -93,6 +93,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -115,6 +116,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -158,6 +160,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -246,6 +249,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -330,6 +334,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -369,6 +374,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -426,6 +432,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -457,6 +464,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -479,6 +487,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -519,6 +528,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -550,6 +560,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -572,6 +583,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -637,6 +649,7 @@ Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} SORTBUCKETCOLSPREFIX TRUE + bucketing_version 2 numFiles 2 numRows 500 rawDataSize 5312 @@ -661,16 +674,16 @@ POSTHOOK: query: SELECT * FROM bucketed_table TABLESAMPLE (BUCKET 1 OUT OF 2) s POSTHOOK: type: QUERY POSTHOOK: Input: default@bucketed_table POSTHOOK: Output: hdfs://### HDFS PATH ### -0 val_0 -0 val_0 -0 val_0 -2 val_2 4 val_4 -8 val_8 -10 val_10 12 val_12 12 val_12 -18 val_18 +24 val_24 +24 val_24 +26 val_26 +26 val_26 +30 val_30 +42 val_42 +42 val_42 PREHOOK: query: SELECT * FROM bucketed_table TABLESAMPLE (BUCKET 2 OUT OF 2) s LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@bucketed_table @@ -682,13 +695,13 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 5 val_5 5 val_5 5 val_5 -9 val_9 11 val_11 -15 val_15 -15 val_15 17 val_17 19 val_19 27 val_27 +41 val_41 +47 val_47 +53 val_53 PREHOOK: query: SELECT COUNT(DISTINCT INPUT__FILE__NAME) FROM bucketed_table PREHOOK: type: QUERY PREHOOK: Input: default@bucketed_table diff --git a/ql/src/test/results/clientpositive/llap/bucket_many.q.out b/ql/src/test/results/clientpositive/llap/bucket_many.q.out index 22027457ed..350cc0d80f 100644 --- a/ql/src/test/results/clientpositive/llap/bucket_many.q.out +++ b/ql/src/test/results/clientpositive/llap/bucket_many.q.out @@ -60,6 +60,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -81,6 +82,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -122,6 +124,7 @@ STAGE PLANS: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count 256 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -204,6 +207,7 @@ STAGE PLANS: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count 256 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out index 61db738021..bbd689a247 100644 --- a/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin1.q.out @@ -439,6 +439,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -460,6 +461,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -540,6 +542,7 @@ STAGE PLANS: properties: bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -587,6 +590,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -668,6 +672,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -859,6 +864,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -880,6 +886,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -960,6 +967,7 @@ STAGE PLANS: properties: bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1007,6 +1015,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -1088,6 +1097,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out index d6f541db7c..633e6b8a6c 100644 --- a/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin2.q.out @@ -176,6 +176,7 @@ STAGE PLANS: properties: bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -254,6 +255,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -301,6 +303,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -382,6 +385,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -602,6 +606,7 @@ STAGE PLANS: properties: bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -680,6 +685,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -727,6 +733,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -808,6 +815,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -1045,6 +1053,7 @@ STAGE PLANS: properties: bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1123,6 +1132,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1171,6 +1181,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1219,6 +1230,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -1300,6 +1312,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out index e37a618bdc..e73c2af465 100644 --- a/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin3.q.out @@ -200,6 +200,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -278,6 +279,7 @@ STAGE PLANS: properties: bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -325,6 +327,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -406,6 +409,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -626,6 +630,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -704,6 +709,7 @@ STAGE PLANS: properties: bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -751,6 +757,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -832,6 +839,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out index c8deac4253..23ef650acc 100644 --- a/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin4.q.out @@ -175,6 +175,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -196,6 +197,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -251,6 +253,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -272,6 +275,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -321,6 +325,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -402,6 +407,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -585,6 +591,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -606,6 +613,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -661,6 +669,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -682,6 +691,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -731,6 +741,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -812,6 +823,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments diff --git a/ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out b/ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out index 1e87e1a423..ad740c8416 100644 --- a/ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketmapjoin7.q.out @@ -134,6 +134,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -209,6 +210,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out b/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out index 83070e6422..fa6724808a 100644 --- a/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketpruning1.q.out @@ -34,7 +34,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: (key = 1) (type: boolean) - buckets included: [1,] of 16 + buckets included: [0,1,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -158,7 +158,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: (key = 17) (type: boolean) - buckets included: [1,] of 16 + buckets included: [0,1,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -220,7 +220,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: (key = 17) (type: boolean) - buckets included: [1,] of 16 + buckets included: [0,1,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -282,7 +282,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: (key = 11) (type: boolean) - buckets included: [11,] of 16 + buckets included: [0,11,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -344,7 +344,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: ((key = 1) and (ds = '2008-04-08')) (type: boolean) - buckets included: [1,] of 16 + buckets included: [0,1,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -406,7 +406,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: ((key = 1) and (ds = '2008-04-08') and (value = 'One')) (type: boolean) - buckets included: [1,] of 16 + buckets included: [0,1,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -468,7 +468,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: ((value = 'One') and (key = 1) and (ds = '2008-04-08')) (type: boolean) - buckets included: [1,] of 16 + buckets included: [0,1,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -530,7 +530,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: (key) IN (2, 3) (type: boolean) - buckets included: [2,3,] of 16 + buckets included: [0,2,3,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -592,7 +592,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: ((key) IN (2, 3) and (ds = '2008-04-08')) (type: boolean) - buckets included: [2,3,] of 16 + buckets included: [0,2,3,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -654,7 +654,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: ((key) IN (2, 3) and (ds = '2008-04-08') and (value = 'One')) (type: boolean) - buckets included: [2,3,] of 16 + buckets included: [0,2,3,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -716,7 +716,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: ((key) IN (2, 3) and (value = 'One') and (ds = '2008-04-08')) (type: boolean) - buckets included: [2,3,] of 16 + buckets included: [0,2,3,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator @@ -900,7 +900,7 @@ STAGE PLANS: TableScan alias: srcbucket_pruned filterExpr: (key = -15) (type: boolean) - buckets included: [1,15,] of 16 + buckets included: [0,15,] of 16 Statistics: Num rows: 1 Data size: 372 Basic stats: COMPLETE Column stats: PARTIAL GatherStats: false Filter Operator diff --git a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out index 49827b4c5b..79da100543 100644 --- a/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out +++ b/ql/src/test/results/clientpositive/llap/bucketsortoptimize_insert_2.q.out @@ -252,6 +252,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table3 POSTHOOK: Input: default@test_table3@ds=1 #### A masked pattern was here #### +4 val_4val_4 1 0 val_0val_0 1 0 val_0val_0 1 0 val_0val_0 1 @@ -262,7 +263,6 @@ POSTHOOK: Input: default@test_table3@ds=1 0 val_0val_0 1 0 val_0val_0 1 2 val_2val_2 1 -4 val_4val_4 1 8 val_8val_8 1 PREHOOK: query: select * from test_table3 tablesample (bucket 2 out of 2) s where ds = '1' PREHOOK: type: QUERY @@ -462,6 +462,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table3 POSTHOOK: Input: default@test_table3@ds=1 #### A masked pattern was here #### +4 val_4val_4 1 +4 val_4val_4 1 0 val_0val_0 1 0 val_0val_0 1 0 val_0val_0 1 @@ -482,8 +484,6 @@ POSTHOOK: Input: default@test_table3@ds=1 0 val_0val_0 1 2 val_2val_2 1 2 val_2val_2 1 -4 val_4val_4 1 -4 val_4val_4 1 8 val_8val_8 1 8 val_8val_8 1 PREHOOK: query: select * from test_table3 tablesample (bucket 2 out of 2) s where ds = '1' @@ -694,6 +694,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table3 POSTHOOK: Input: default@test_table3@ds=1 #### A masked pattern was here #### +4 val_4val_4 1 +4 val_4val_4 1 0 val_0val_0 1 0 val_0val_0 1 0 val_0val_0 1 @@ -714,8 +716,6 @@ POSTHOOK: Input: default@test_table3@ds=1 0 val_0val_0 1 2 val_2val_2 1 2 val_2val_2 1 -4 val_4val_4 1 -4 val_4val_4 1 8 val_8val_8 1 8 val_8val_8 1 PREHOOK: query: select * from test_table3 tablesample (bucket 2 out of 2) s where ds = '1' @@ -936,6 +936,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table3 POSTHOOK: Input: default@test_table3@ds=1 #### A masked pattern was here #### +4 val_4val_4 1 0 val_0val_0 1 0 val_0val_0 1 0 val_0val_0 1 @@ -946,7 +947,6 @@ POSTHOOK: Input: default@test_table3@ds=1 0 val_0val_0 1 0 val_0val_0 1 2 val_2val_2 1 -4 val_4val_4 1 8 val_8val_8 1 PREHOOK: query: select * from test_table3 tablesample (bucket 2 out of 2) s where ds = '1' PREHOOK: type: QUERY @@ -1156,6 +1156,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table3 POSTHOOK: Input: default@test_table3@ds=1 #### A masked pattern was here #### +4 val_4val_4val_4val_4 1 0 val_0val_0val_0val_0 1 0 val_0val_0val_0val_0 1 0 val_0val_0val_0val_0 1 @@ -1166,7 +1167,6 @@ POSTHOOK: Input: default@test_table3@ds=1 0 val_0val_0val_0val_0 1 0 val_0val_0val_0val_0 1 2 val_2val_2val_2val_2 1 -4 val_4val_4val_4val_4 1 8 val_8val_8val_8val_8 1 PREHOOK: query: select * from test_table3 tablesample (bucket 2 out of 2) s where ds = '1' PREHOOK: type: QUERY @@ -1376,6 +1376,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table3 POSTHOOK: Input: default@test_table3@ds=1 #### A masked pattern was here #### +4 val_2val_2 1 +16 val_8val_8 1 0 val_0val_0 1 0 val_0val_0 1 0 val_0val_0 1 @@ -1385,7 +1387,6 @@ POSTHOOK: Input: default@test_table3@ds=1 0 val_0val_0 1 0 val_0val_0 1 0 val_0val_0 1 -4 val_2val_2 1 8 val_4val_4 1 10 val_5val_5 1 10 val_5val_5 1 @@ -1396,7 +1397,6 @@ POSTHOOK: Input: default@test_table3@ds=1 10 val_5val_5 1 10 val_5val_5 1 10 val_5val_5 1 -16 val_8val_8 1 18 val_9val_9 1 PREHOOK: query: select * from test_table3 tablesample (bucket 2 out of 2) s where ds = '1' PREHOOK: type: QUERY diff --git a/ql/src/test/results/clientpositive/llap/cbo_views.q.out b/ql/src/test/results/clientpositive/llap/cbo_views.q.out index 25ea4cf50b..44e5501f6d 100644 --- a/ql/src/test/results/clientpositive/llap/cbo_views.q.out +++ b/ql/src/test/results/clientpositive/llap/cbo_views.q.out @@ -50,9 +50,9 @@ POSTHOOK: Input: default@cbo_t1 POSTHOOK: Input: default@cbo_t1@dt=2014 POSTHOOK: Input: default@v1 #### A masked pattern was here #### -1 NULL 1 +1 PREHOOK: query: select count(v1.c_int) from v1 join cbo_t2 on v1.c_int = cbo_t2.c_int PREHOOK: type: QUERY PREHOOK: Input: default@cbo_t1 diff --git a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out index fb7e872696..4c73fec88f 100644 --- a/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out +++ b/ql/src/test/results/clientpositive/llap/columnStatsUpdateForStatsOptimizer_1.q.out @@ -38,6 +38,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 3 rawDataSize 21 @@ -80,6 +81,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 3 rawDataSize 21 @@ -214,6 +216,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"month\":\"true\",\"year\":\"true\"}} + bucketing_version 2 numFiles 1 numRows 3 rawDataSize 21 @@ -281,6 +284,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numRows 4 rawDataSize 28 @@ -459,6 +463,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"year\":\"true\"}} + bucketing_version 2 numFiles 2 numRows 4 rawDataSize 28 @@ -593,6 +598,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"month\":\"true\",\"year\":\"true\"}} + bucketing_version 2 numFiles 2 numRows 4 rawDataSize 28 @@ -965,6 +971,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 numFiles 1 numRows 0 rawDataSize 0 @@ -1007,6 +1014,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 500 rawDataSize 5312 diff --git a/ql/src/test/results/clientpositive/llap/column_names_with_leading_and_trailing_spaces.q.out b/ql/src/test/results/clientpositive/llap/column_names_with_leading_and_trailing_spaces.q.out index b327bf8e14..e83a370391 100644 --- a/ql/src/test/results/clientpositive/llap/column_names_with_leading_and_trailing_spaces.q.out +++ b/ql/src/test/results/clientpositive/llap/column_names_with_leading_and_trailing_spaces.q.out @@ -25,6 +25,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\" left\":\"true\",\" middle \":\"true\",\"right \":\"true\"}} + bucketing_version 2 numFiles 0 numRows 0 rawDataSize 0 diff --git a/ql/src/test/results/clientpositive/llap/column_table_stats.q.out b/ql/src/test/results/clientpositive/llap/column_table_stats.q.out index 7f3cc02618..1457932726 100644 --- a/ql/src/test/results/clientpositive/llap/column_table_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/column_table_stats.q.out @@ -35,6 +35,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 numFiles 1 numRows 0 rawDataSize 0 @@ -102,6 +103,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -122,6 +124,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -211,6 +214,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + bucketing_version 2 numFiles 1 numRows 500 rawDataSize 5312 @@ -287,6 +291,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 numFiles 2 numPartitions 2 numRows 0 @@ -383,6 +388,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -430,6 +436,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -536,6 +543,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numPartitions 2 numRows 1000 @@ -693,6 +701,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 numFiles 2 numPartitions 2 numRows 0 @@ -789,6 +798,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -836,6 +846,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -942,6 +953,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numPartitions 2 numRows 1000 @@ -1099,6 +1111,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 numFiles 2 numPartitions 2 numRows 0 @@ -1195,6 +1208,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1295,6 +1309,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 numFiles 2 numPartitions 2 numRows 500 diff --git a/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out b/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out index 2c8e9ab6ed..5ed5dbf96d 100644 --- a/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out +++ b/ql/src/test/results/clientpositive/llap/column_table_stats_orc.q.out @@ -38,6 +38,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + bucketing_version 2 numFiles 1 numRows 1 rawDataSize 170 @@ -106,6 +107,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -127,6 +129,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -215,6 +218,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"key\":\"true\",\"value\":\"true\"}} + bucketing_version 2 numFiles 1 numRows 1 rawDataSize 170 @@ -290,6 +294,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numPartitions 2 numRows 2 @@ -383,6 +388,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -427,6 +433,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -532,6 +539,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numPartitions 2 numRows 2 @@ -688,6 +696,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numPartitions 2 numRows 2 @@ -781,6 +790,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -881,6 +891,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numPartitions 2 numRows 2 diff --git a/ql/src/test/results/clientpositive/llap/ctas.q.out b/ql/src/test/results/clientpositive/llap/ctas.q.out index 7240da636b..0b85984d64 100644 --- a/ql/src/test/results/clientpositive/llap/ctas.q.out +++ b/ql/src/test/results/clientpositive/llap/ctas.q.out @@ -154,6 +154,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 10 rawDataSize 96 @@ -310,6 +311,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 10 rawDataSize 96 @@ -466,6 +468,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 10 rawDataSize 120 @@ -529,6 +532,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 10 rawDataSize 120 @@ -686,6 +690,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 10 rawDataSize 96 diff --git a/ql/src/test/results/clientpositive/llap/default_constraint.q.out b/ql/src/test/results/clientpositive/llap/default_constraint.q.out index 72ff7af046..c73c3a1ae8 100644 --- a/ql/src/test/results/clientpositive/llap/default_constraint.q.out +++ b/ql/src/test/results/clientpositive/llap/default_constraint.q.out @@ -31,6 +31,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 transactional true transactional_properties default #### A masked pattern was here #### @@ -301,6 +302,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 #### A masked pattern was here #### # Storage Information @@ -530,6 +532,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 transactional true transactional_properties default #### A masked pattern was here #### @@ -788,6 +791,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 transactional true transactional_properties default #### A masked pattern was here #### @@ -1059,6 +1063,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 #### A masked pattern was here #### # Storage Information @@ -1336,6 +1341,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 transactional true transactional_properties default #### A masked pattern was here #### @@ -1491,6 +1497,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {} + bucketing_version 2 transactional true transactional_properties default #### A masked pattern was here #### @@ -1658,6 +1665,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {} + bucketing_version 2 #### A masked pattern was here #### transactional true transactional_properties default @@ -1736,6 +1744,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {} + bucketing_version 2 #### A masked pattern was here #### transactional true transactional_properties default @@ -1911,6 +1920,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {} + bucketing_version 2 #### A masked pattern was here #### transactional true transactional_properties default @@ -1988,6 +1998,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {} + bucketing_version 2 #### A masked pattern was here #### transactional true transactional_properties default @@ -2066,6 +2077,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {} + bucketing_version 2 #### A masked pattern was here #### transactional true transactional_properties default @@ -2253,6 +2265,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 #### A masked pattern was here #### # Storage Information @@ -2484,6 +2497,7 @@ Retention: 0 #### A masked pattern was here #### Table Type: MANAGED_TABLE Table Parameters: + bucketing_version 2 transactional true transactional_properties default #### A masked pattern was here #### @@ -2639,6 +2653,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {} + bucketing_version 2 transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out b/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out index aeca5184c8..48bc7ace76 100644 --- a/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out +++ b/ql/src/test/results/clientpositive/llap/deleteAnalyze.q.out @@ -50,6 +50,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"amount\":\"true\",\"id\":\"true\",\"item\":\"true\",\"sales_tax\":\"true\"}} + bucketing_version 2 numFiles 1 numRows 2 rawDataSize 634 @@ -116,6 +117,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"amount\":\"true\",\"id\":\"true\",\"item\":\"true\",\"sales_tax\":\"true\"}} + bucketing_version 2 numFiles 0 numRows 0 rawDataSize 0 diff --git a/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out b/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out index e45d49e602..a5ad10e945 100644 --- a/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out +++ b/ql/src/test/results/clientpositive/llap/disable_merge_for_bucketing.q.out @@ -59,6 +59,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -80,6 +81,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -121,6 +123,7 @@ STAGE PLANS: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -191,6 +194,7 @@ STAGE PLANS: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out index f468b90c64..260636393e 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_partition_pruning.q.out @@ -14,8 +14,8 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### -2008-04-08 2008-04-09 +2008-04-08 PREHOOK: query: select distinct hr from srcpart PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -3679,8 +3679,8 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### -2008-04-08 2008-04-09 +2008-04-08 PREHOOK: query: EXPLAIN select ds from (select distinct(ds) as ds from srcpart union all select distinct(ds) as ds from srcpart) s where s.ds in (select max(srcpart.ds) from srcpart union all select min(srcpart.ds) from srcpart) PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN select ds from (select distinct(ds) as ds from srcpart union all select distinct(ds) as ds from srcpart) s where s.ds in (select max(srcpart.ds) from srcpart union all select min(srcpart.ds) from srcpart) @@ -5843,8 +5843,8 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### -2008-04-08 2008-04-09 +2008-04-08 PREHOOK: query: create table srcpart_orc (key int, value string) partitioned by (ds string, hr int) stored as orc PREHOOK: type: CREATETABLE PREHOOK: Output: database:default diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out index 3d711c3225..86e41116d2 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out @@ -1608,6 +1608,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1651,6 +1652,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1742,6 +1744,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key1,value1 columns.comments @@ -1785,6 +1788,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key1,value1 columns.comments diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_user_level.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_user_level.q.out index af4004efd3..9b73c3440d 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_user_level.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_user_level.q.out @@ -874,6 +874,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -919,6 +920,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -1013,6 +1015,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key1,value1 columns.comments @@ -1058,6 +1061,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key1,value1 columns.comments diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out index 97f7cd159f..34ff2ceab9 100644 --- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out +++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization_acid.q.out @@ -94,19 +94,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_part - Statistics: Num rows: 158 Data size: 60414 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 160 Data size: 60861 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 'foo') (type: boolean) - Statistics: Num rows: 5 Data size: 1911 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1901 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1911 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1901 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 5 Data size: 1911 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1901 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 @@ -115,10 +115,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5 Data size: 1911 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1901 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 1911 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1901 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -189,7 +189,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_part - Statistics: Num rows: 157 Data size: 102975 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 158 Data size: 103606 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1355 Basic stats: COMPLETE Column stats: PARTIAL @@ -380,7 +380,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_part_sdpo - Statistics: Num rows: 174 Data size: 66399 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 176 Data size: 67193 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1908 Basic stats: COMPLETE Column stats: NONE @@ -475,7 +475,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_part_sdpo - Statistics: Num rows: 169 Data size: 110820 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 171 Data size: 111982 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1355 Basic stats: COMPLETE Column stats: PARTIAL @@ -675,19 +675,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_2l_part - Statistics: Num rows: 154 Data size: 59496 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 155 Data size: 59903 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 'foo') (type: boolean) - Statistics: Num rows: 5 Data size: 1931 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1932 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1931 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1932 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 5 Data size: 1931 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1932 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 @@ -696,10 +696,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 5 Data size: 1931 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1932 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 1931 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1932 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1095,19 +1095,19 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_2l_part_sdpo - Statistics: Num rows: 154 Data size: 59496 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 155 Data size: 59903 Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (key = 'foo') (type: boolean) - Statistics: Num rows: 5 Data size: 1931 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1932 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 5 Data size: 1931 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1932 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: struct) sort order: + Map-reduce partition columns: UDFToInteger(_col0) (type: int) - Statistics: Num rows: 5 Data size: 1931 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1932 Basic stats: COMPLETE Column stats: NONE Execution mode: llap LLAP IO: may be used (ACID table) Reducer 2 @@ -1116,10 +1116,10 @@ STAGE PLANS: Select Operator expressions: KEY.reducesinkkey0 (type: struct), 'foo' (type: string), 'bar' (type: string), '2008-04-08' (type: string), 11 (type: int) outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 5 Data size: 1931 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1932 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 5 Data size: 1931 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 1932 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat @@ -1515,7 +1515,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: acid_2l_part_sdpo_no_cp - Statistics: Num rows: 95 Data size: 81408 Basic stats: COMPLETE Column stats: PARTIAL + Statistics: Num rows: 96 Data size: 82100 Basic stats: COMPLETE Column stats: PARTIAL Filter Operator predicate: (key = 'foo') (type: boolean) Statistics: Num rows: 5 Data size: 1860 Basic stats: COMPLETE Column stats: PARTIAL diff --git a/ql/src/test/results/clientpositive/llap/except_distinct.q.out b/ql/src/test/results/clientpositive/llap/except_distinct.q.out index cc35b120fc..2e0dea2857 100644 --- a/ql/src/test/results/clientpositive/llap/except_distinct.q.out +++ b/ql/src/test/results/clientpositive/llap/except_distinct.q.out @@ -199,8 +199,8 @@ POSTHOOK: Input: default@a POSTHOOK: Input: default@b POSTHOOK: Output: hdfs://### HDFS PATH ### 2 3 -2 20 1 2 +2 20 PREHOOK: query: explain select * from src except distinct select * from src PREHOOK: type: QUERY POSTHOOK: query: explain select * from src except distinct select * from src diff --git a/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out index 8a6b70daec..aa02521ceb 100644 --- a/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out +++ b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out @@ -65,9 +65,9 @@ Stage-0 <-Reducer 11 [CONTAINS] llap Reduce Output Operator [RS_53] PartitionCols:_col0, _col1 - Select Operator [SEL_49] (rows=67/61 width=177) + Select Operator [SEL_49] (rows=67/69 width=177) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_82] (rows=67/61 width=177) + Merge Join Operator [MERGEJOIN_82] (rows=67/69 width=177) Conds:RS_46._col2=RS_47._col0(Inner),Output:["_col1","_col2"] <-Map 17 [SIMPLE_EDGE] llap SHUFFLE [RS_47] @@ -81,7 +81,7 @@ Stage-0 <-Reducer 10 [SIMPLE_EDGE] llap SHUFFLE [RS_46] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_81] (rows=42/52 width=177) + Merge Join Operator [MERGEJOIN_81] (rows=42/59 width=177) Conds:RS_43._col1=RS_44._col1(Inner),Output:["_col1","_col2"] <-Map 9 [SIMPLE_EDGE] llap SHUFFLE [RS_44] @@ -95,9 +95,9 @@ Stage-0 <-Reducer 15 [SIMPLE_EDGE] llap SHUFFLE [RS_43] PartitionCols:_col1 - Select Operator [SEL_36] (rows=525/319 width=178) + Select Operator [SEL_36] (rows=525/323 width=178) Output:["_col1"] - Group By Operator [GBY_35] (rows=525/319 width=178) + Group By Operator [GBY_35] (rows=525/323 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 14 [SIMPLE_EDGE] <-Map 13 [CONTAINS] llap @@ -121,9 +121,9 @@ Stage-0 <-Reducer 5 [CONTAINS] llap Reduce Output Operator [RS_53] PartitionCols:_col0, _col1 - Select Operator [SEL_24] (rows=67/61 width=177) + Select Operator [SEL_24] (rows=67/69 width=177) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_80] (rows=67/61 width=177) + Merge Join Operator [MERGEJOIN_80] (rows=67/69 width=177) Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col1","_col2"] <-Map 12 [SIMPLE_EDGE] llap SHUFFLE [RS_22] @@ -137,7 +137,7 @@ Stage-0 <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_21] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_79] (rows=42/52 width=177) + Merge Join Operator [MERGEJOIN_79] (rows=42/59 width=177) Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col1","_col2"] <-Map 9 [SIMPLE_EDGE] llap SHUFFLE [RS_19] @@ -146,9 +146,9 @@ Stage-0 <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col1 - Select Operator [SEL_11] (rows=525/319 width=178) + Select Operator [SEL_11] (rows=525/323 width=178) Output:["_col1"] - Group By Operator [GBY_10] (rows=525/319 width=178) + Group By Operator [GBY_10] (rows=525/323 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 2 [SIMPLE_EDGE] <-Map 1 [CONTAINS] llap @@ -265,9 +265,9 @@ Stage-0 <-Reducer 15 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 - Select Operator [SEL_107] (rows=199/61 width=177) + Select Operator [SEL_107] (rows=199/83 width=177) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_162] (rows=199/61 width=177) + Merge Join Operator [MERGEJOIN_162] (rows=199/83 width=177) Conds:RS_104._col2=RS_105._col0(Inner),Output:["_col2","_col5"] <-Map 16 [SIMPLE_EDGE] llap SHUFFLE [RS_105] @@ -281,7 +281,7 @@ Stage-0 <-Reducer 14 [SIMPLE_EDGE] llap SHUFFLE [RS_104] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_161] (rows=123/52 width=86) + Merge Join Operator [MERGEJOIN_161] (rows=123/63 width=86) Conds:RS_101._col1=RS_102._col1(Inner),Output:["_col2"] <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_102] @@ -295,9 +295,9 @@ Stage-0 <-Reducer 31 [SIMPLE_EDGE] llap SHUFFLE [RS_101] PartitionCols:_col1 - Select Operator [SEL_94] (rows=1525/319 width=178) + Select Operator [SEL_94] (rows=1525/510 width=178) Output:["_col1"] - Group By Operator [GBY_93] (rows=1525/319 width=178) + Group By Operator [GBY_93] (rows=1525/510 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 30 [SIMPLE_EDGE] <-Map 34 [CONTAINS] llap @@ -312,9 +312,9 @@ Stage-0 <-Reducer 29 [CONTAINS] llap Reduce Output Operator [RS_92] PartitionCols:_col1, _col0 - Select Operator [SEL_85] (rows=1025/319 width=178) + Select Operator [SEL_85] (rows=1025/423 width=178) Output:["_col0","_col1"] - Group By Operator [GBY_84] (rows=1025/319 width=178) + Group By Operator [GBY_84] (rows=1025/423 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 28 [SIMPLE_EDGE] <-Map 33 [CONTAINS] llap @@ -329,9 +329,9 @@ Stage-0 <-Reducer 27 [CONTAINS] llap Reduce Output Operator [RS_83] PartitionCols:_col1, _col0 - Select Operator [SEL_76] (rows=525/319 width=178) + Select Operator [SEL_76] (rows=525/323 width=178) Output:["_col0","_col1"] - Group By Operator [GBY_75] (rows=525/319 width=178) + Group By Operator [GBY_75] (rows=525/323 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 26 [SIMPLE_EDGE] <-Map 25 [CONTAINS] llap @@ -361,9 +361,9 @@ Stage-0 <-Reducer 13 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 - Select Operator [SEL_58] (rows=132/61 width=177) + Select Operator [SEL_58] (rows=132/71 width=177) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_160] (rows=132/61 width=177) + Merge Join Operator [MERGEJOIN_160] (rows=132/71 width=177) Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col2","_col5"] <-Map 24 [SIMPLE_EDGE] llap SHUFFLE [RS_56] @@ -377,7 +377,7 @@ Stage-0 <-Reducer 12 [SIMPLE_EDGE] llap SHUFFLE [RS_55] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_159] (rows=82/52 width=86) + Merge Join Operator [MERGEJOIN_159] (rows=82/58 width=86) Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2"] <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_53] @@ -386,9 +386,9 @@ Stage-0 <-Reducer 21 [SIMPLE_EDGE] llap SHUFFLE [RS_52] PartitionCols:_col1 - Select Operator [SEL_45] (rows=1025/319 width=178) + Select Operator [SEL_45] (rows=1025/423 width=178) Output:["_col1"] - Group By Operator [GBY_44] (rows=1025/319 width=178) + Group By Operator [GBY_44] (rows=1025/423 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 20 [SIMPLE_EDGE] <-Map 23 [CONTAINS] llap @@ -403,9 +403,9 @@ Stage-0 <-Reducer 19 [CONTAINS] llap Reduce Output Operator [RS_43] PartitionCols:_col1, _col0 - Select Operator [SEL_36] (rows=525/319 width=178) + Select Operator [SEL_36] (rows=525/323 width=178) Output:["_col0","_col1"] - Group By Operator [GBY_35] (rows=525/319 width=178) + Group By Operator [GBY_35] (rows=525/323 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 18 [SIMPLE_EDGE] <-Map 17 [CONTAINS] llap @@ -429,9 +429,9 @@ Stage-0 <-Reducer 5 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 - Select Operator [SEL_24] (rows=67/61 width=177) + Select Operator [SEL_24] (rows=67/69 width=177) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_158] (rows=67/61 width=177) + Merge Join Operator [MERGEJOIN_158] (rows=67/69 width=177) Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col2","_col5"] <-Map 16 [SIMPLE_EDGE] llap SHUFFLE [RS_22] @@ -440,7 +440,7 @@ Stage-0 <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_21] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_157] (rows=42/52 width=86) + Merge Join Operator [MERGEJOIN_157] (rows=42/59 width=86) Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col2"] <-Map 11 [SIMPLE_EDGE] llap SHUFFLE [RS_19] @@ -449,9 +449,9 @@ Stage-0 <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col1 - Select Operator [SEL_11] (rows=525/319 width=178) + Select Operator [SEL_11] (rows=525/323 width=178) Output:["_col1"] - Group By Operator [GBY_10] (rows=525/319 width=178) + Group By Operator [GBY_10] (rows=525/323 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 2 [SIMPLE_EDGE] <-Map 1 [CONTAINS] llap @@ -1499,17 +1499,17 @@ Stage-5 Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] <-Reducer 9 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=2941/319 width=178) + Select Operator [SEL_1] (rows=2941/351 width=178) Output:["key","value"] - Group By Operator [GBY_112] (rows=2941/319 width=178) + Group By Operator [GBY_112] (rows=2941/351 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 8 [SIMPLE_EDGE] <-Reducer 16 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 - Select Operator [SEL_107] (rows=192/304 width=175) + Select Operator [SEL_107] (rows=192/333 width=175) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_164] (rows=192/304 width=175) + Merge Join Operator [MERGEJOIN_164] (rows=192/333 width=175) Conds:RS_104._col1=RS_105._col1(Inner),Output:["_col0","_col3"] <-Reducer 15 [SIMPLE_EDGE] llap SHUFFLE [RS_104] @@ -1537,9 +1537,9 @@ Stage-5 <-Reducer 35 [SIMPLE_EDGE] llap SHUFFLE [RS_105] PartitionCols:_col1 - Select Operator [SEL_100] (rows=1525/319 width=178) + Select Operator [SEL_100] (rows=1525/510 width=178) Output:["_col1"] - Group By Operator [GBY_99] (rows=1525/319 width=178) + Group By Operator [GBY_99] (rows=1525/510 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 34 [SIMPLE_EDGE] <-Map 38 [CONTAINS] llap @@ -1554,9 +1554,9 @@ Stage-5 <-Reducer 33 [CONTAINS] llap Reduce Output Operator [RS_98] PartitionCols:_col1, _col0 - Select Operator [SEL_91] (rows=1025/319 width=178) + Select Operator [SEL_91] (rows=1025/423 width=178) Output:["_col0","_col1"] - Group By Operator [GBY_90] (rows=1025/319 width=178) + Group By Operator [GBY_90] (rows=1025/423 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 32 [SIMPLE_EDGE] <-Map 37 [CONTAINS] llap @@ -1571,9 +1571,9 @@ Stage-5 <-Reducer 31 [CONTAINS] llap Reduce Output Operator [RS_89] PartitionCols:_col1, _col0 - Select Operator [SEL_82] (rows=525/319 width=178) + Select Operator [SEL_82] (rows=525/323 width=178) Output:["_col0","_col1"] - Group By Operator [GBY_81] (rows=525/319 width=178) + Group By Operator [GBY_81] (rows=525/323 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 30 [SIMPLE_EDGE] <-Map 29 [CONTAINS] llap @@ -1597,15 +1597,15 @@ Stage-5 <-Reducer 7 [CONTAINS] llap Reduce Output Operator [RS_111] PartitionCols:_col0, _col1 - Group By Operator [GBY_63] (rows=2749/309 width=178) + Group By Operator [GBY_63] (rows=2749/321 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 6 [SIMPLE_EDGE] <-Reducer 19 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 - Select Operator [SEL_58] (rows=2682/1056 width=178) + Select Operator [SEL_58] (rows=2682/1415 width=178) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_162] (rows=2682/1056 width=178) + Merge Join Operator [MERGEJOIN_162] (rows=2682/1415 width=178) Conds:RS_55._col2=RS_56._col0(Inner),Output:["_col2","_col5"] <-Map 27 [SIMPLE_EDGE] llap SHUFFLE [RS_56] @@ -1619,7 +1619,7 @@ Stage-5 <-Reducer 18 [SIMPLE_EDGE] llap SHUFFLE [RS_55] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_161] (rows=1658/512 width=87) + Merge Join Operator [MERGEJOIN_161] (rows=1658/687 width=87) Conds:RS_52._col1=RS_53._col1(Inner),Output:["_col2"] <-Map 17 [SIMPLE_EDGE] llap SHUFFLE [RS_53] @@ -1633,9 +1633,9 @@ Stage-5 <-Reducer 24 [SIMPLE_EDGE] llap SHUFFLE [RS_52] PartitionCols:_col1 - Select Operator [SEL_45] (rows=1025/319 width=178) + Select Operator [SEL_45] (rows=1025/423 width=178) Output:["_col1"] - Group By Operator [GBY_44] (rows=1025/319 width=178) + Group By Operator [GBY_44] (rows=1025/423 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 23 [SIMPLE_EDGE] <-Map 26 [CONTAINS] llap @@ -1650,9 +1650,9 @@ Stage-5 <-Reducer 22 [CONTAINS] llap Reduce Output Operator [RS_43] PartitionCols:_col1, _col0 - Select Operator [SEL_36] (rows=525/319 width=178) + Select Operator [SEL_36] (rows=525/323 width=178) Output:["_col0","_col1"] - Group By Operator [GBY_35] (rows=525/319 width=178) + Group By Operator [GBY_35] (rows=525/323 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 21 [SIMPLE_EDGE] <-Map 20 [CONTAINS] llap @@ -1676,9 +1676,9 @@ Stage-5 <-Reducer 5 [CONTAINS] llap Reduce Output Operator [RS_62] PartitionCols:_col0, _col1 - Select Operator [SEL_24] (rows=67/61 width=177) + Select Operator [SEL_24] (rows=67/69 width=177) Output:["_col0","_col1"] - Merge Join Operator [MERGEJOIN_160] (rows=67/61 width=177) + Merge Join Operator [MERGEJOIN_160] (rows=67/69 width=177) Conds:RS_21._col2=RS_22._col0(Inner),Output:["_col2","_col5"] <-Map 17 [SIMPLE_EDGE] llap SHUFFLE [RS_22] @@ -1691,7 +1691,7 @@ Stage-5 <-Reducer 4 [SIMPLE_EDGE] llap SHUFFLE [RS_21] PartitionCols:_col2 - Merge Join Operator [MERGEJOIN_159] (rows=42/52 width=86) + Merge Join Operator [MERGEJOIN_159] (rows=42/59 width=86) Conds:RS_18._col1=RS_19._col1(Inner),Output:["_col2"] <-Map 14 [SIMPLE_EDGE] llap SHUFFLE [RS_19] @@ -1700,9 +1700,9 @@ Stage-5 <-Reducer 3 [SIMPLE_EDGE] llap SHUFFLE [RS_18] PartitionCols:_col1 - Select Operator [SEL_11] (rows=525/319 width=178) + Select Operator [SEL_11] (rows=525/323 width=178) Output:["_col1"] - Group By Operator [GBY_10] (rows=525/319 width=178) + Group By Operator [GBY_10] (rows=525/323 width=178) Output:["_col0","_col1"],keys:KEY._col0, KEY._col1 <-Union 2 [SIMPLE_EDGE] <-Map 1 [CONTAINS] llap @@ -1729,7 +1729,7 @@ Stage-5 Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] <-Reducer 9 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=2941/319 width=178) + Select Operator [SEL_1] (rows=2941/351 width=178) Output:["key","value"] Please refer to the previous Group By Operator [GBY_112] Reducer 12 llap @@ -1738,7 +1738,7 @@ Stage-5 Output:["_col0","_col1"],aggregations:["compute_stats(VALUE._col0, 'hll')","compute_stats(VALUE._col2, 'hll')"] <-Reducer 9 [CUSTOM_SIMPLE_EDGE] llap PARTITION_ONLY_SHUFFLE [RS_2] - Select Operator [SEL_1] (rows=2941/319 width=178) + Select Operator [SEL_1] (rows=2941/351 width=178) Output:["key","value"] Please refer to the previous Group By Operator [GBY_112] Stage-6 diff --git a/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out b/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out index aa44d33420..2b1ec3c326 100644 --- a/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out +++ b/ql/src/test/results/clientpositive/llap/extrapolate_part_stats_partial_ndv.q.out @@ -293,6 +293,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt,zip columns.comments @@ -338,6 +339,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt,zip columns.comments @@ -383,6 +385,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt,zip columns.comments @@ -428,6 +431,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt,zip columns.comments @@ -666,6 +670,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt,zip columns.comments @@ -711,6 +716,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt,zip columns.comments @@ -756,6 +762,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt,zip columns.comments @@ -801,6 +808,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt,zip columns.comments @@ -1075,6 +1083,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments @@ -1121,6 +1130,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments @@ -1167,6 +1177,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments @@ -1213,6 +1224,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments @@ -1259,6 +1271,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments @@ -1305,6 +1318,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments @@ -1351,6 +1365,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments @@ -1397,6 +1412,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments @@ -1443,6 +1459,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments @@ -1489,6 +1506,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments @@ -1535,6 +1553,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns state,locid,cnt columns.comments diff --git a/ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out b/ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out index f69d8a2ae0..5c6dd26b8e 100644 --- a/ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out +++ b/ql/src/test/results/clientpositive/llap/filter_join_breaktask.q.out @@ -100,6 +100,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -177,6 +178,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -253,6 +255,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/filter_union.q.out b/ql/src/test/results/clientpositive/llap/filter_union.q.out index 0aab9e0a0e..813f49b828 100644 --- a/ql/src/test/results/clientpositive/llap/filter_union.q.out +++ b/ql/src/test/results/clientpositive/llap/filter_union.q.out @@ -75,6 +75,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -96,6 +97,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -153,6 +155,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -174,6 +177,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' diff --git a/ql/src/test/results/clientpositive/llap/groupby3.q.out b/ql/src/test/results/clientpositive/llap/groupby3.q.out index d050c4ec69..fa5bab8f53 100644 --- a/ql/src/test/results/clientpositive/llap/groupby3.q.out +++ b/ql/src/test/results/clientpositive/llap/groupby3.q.out @@ -192,4 +192,4 @@ POSTHOOK: query: SELECT dest1.* FROM dest1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### -130091.0 260.182 256.10355987055016 98.0 0.0 142.92680950752379 143.06995106518903 20428.07287599999 20469.010897795582 +130091.0 260.182 256.10355987055016 98.0 0.0 142.9268095075238 143.06995106518906 20428.072876000002 20469.010897795593 diff --git a/ql/src/test/results/clientpositive/llap/infer_bucket_sort_bucketed_table.q.out b/ql/src/test/results/clientpositive/llap/infer_bucket_sort_bucketed_table.q.out index 1520232a10..cab1caeb0d 100644 --- a/ql/src/test/results/clientpositive/llap/infer_bucket_sort_bucketed_table.q.out +++ b/ql/src/test/results/clientpositive/llap/infer_bucket_sort_bucketed_table.q.out @@ -67,7 +67,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table_bucketed POSTHOOK: Input: default@test_table_bucketed@part=1 #### A masked pattern was here #### -31 +101 PREHOOK: query: SELECT COUNT(*) FROM test_table_bucketed TABLESAMPLE (BUCKET 2 OUT OF 3) WHERE part = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table_bucketed @@ -78,7 +78,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table_bucketed POSTHOOK: Input: default@test_table_bucketed@part=1 #### A masked pattern was here #### -179 +4 PREHOOK: query: SELECT COUNT(*) FROM test_table_bucketed TABLESAMPLE (BUCKET 3 OUT OF 3) WHERE part = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table_bucketed @@ -89,7 +89,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table_bucketed POSTHOOK: Input: default@test_table_bucketed@part=1 #### A masked pattern was here #### -99 +204 PREHOOK: query: SELECT cnt FROM (SELECT INPUT__FILE__NAME, COUNT(*) cnt FROM test_table_bucketed WHERE part = '1' GROUP BY INPUT__FILE__NAME ORDER BY INPUT__FILE__NAME ASC LIMIT 3) a PREHOOK: type: QUERY @@ -102,6 +102,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table_bucketed POSTHOOK: Input: default@test_table_bucketed@part=1 #### A masked pattern was here #### -31 -179 -99 +101 +4 +204 diff --git a/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out b/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out index f8622dae76..927fa876b5 100644 --- a/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out +++ b/ql/src/test/results/clientpositive/llap/join32_lessSize.q.out @@ -90,6 +90,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -111,6 +112,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -166,6 +168,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -187,6 +190,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -266,6 +270,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -313,6 +318,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,val2 columns.comments @@ -394,6 +400,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,val2 columns.comments @@ -616,6 +623,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -637,6 +645,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -692,6 +701,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -713,6 +723,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -767,6 +778,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -788,6 +800,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -843,6 +856,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -864,6 +878,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -935,6 +950,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,val2 columns.comments @@ -1016,6 +1032,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,val2 columns.comments @@ -1245,6 +1262,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1314,6 +1332,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1335,6 +1354,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1390,6 +1410,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1411,6 +1432,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1460,6 +1482,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,val2 columns.comments @@ -1541,6 +1564,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,val2 columns.comments @@ -1772,6 +1796,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1820,6 +1845,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1841,6 +1867,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1896,6 +1923,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1917,6 +1945,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1966,6 +1995,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,val2 columns.comments @@ -2073,6 +2103,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","val2":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,val2 columns.comments diff --git a/ql/src/test/results/clientpositive/llap/lateral_view.q.out b/ql/src/test/results/clientpositive/llap/lateral_view.q.out index 20e44374ca..0e2408d510 100644 --- a/ql/src/test/results/clientpositive/llap/lateral_view.q.out +++ b/ql/src/test/results/clientpositive/llap/lateral_view.q.out @@ -716,16 +716,16 @@ LATERAL VIEW explode(value) myTable AS myCol POSTHOOK: type: QUERY POSTHOOK: Input: default@tmp_pyang_src_rcfile #### A masked pattern was here #### -["val_0"] val_0 -["val_10"] val_10 -["val_104"] val_104 ["val_105"] val_105 -["val_111"] val_111 -["val_113"] val_113 -["val_114"] val_114 -["val_116"] val_116 ["val_119"] val_119 +["val_10"] val_10 ["val_103"] val_103 +["val_111"] val_111 +["val_104"] val_104 +["val_113"] val_113 +["val_118"] val_118 +["val_0"] val_0 ["val_100"] val_100 ["val_11"] val_11 -["val_118"] val_118 +["val_114"] val_114 +["val_116"] val_116 diff --git a/ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out b/ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out index 2bf96d0aae..5e9d191885 100644 --- a/ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out +++ b/ql/src/test/results/clientpositive/llap/list_bucket_dml_10.q.out @@ -59,6 +59,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -108,6 +109,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -129,6 +131,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -201,6 +204,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out b/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out index 911b65a56d..aa0e6e4d7d 100644 --- a/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_nullscan.q.out @@ -87,6 +87,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,ds,hr columns.comments @@ -109,6 +110,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,ds,hr columns.comments diff --git a/ql/src/test/results/clientpositive/llap/llap_smb.q.out b/ql/src/test/results/clientpositive/llap/llap_smb.q.out index 4c0f6a01fe..747842d04b 100644 --- a/ql/src/test/results/clientpositive/llap/llap_smb.q.out +++ b/ql/src/test/results/clientpositive/llap/llap_smb.q.out @@ -172,26 +172,26 @@ POSTHOOK: Input: default@orc_a@y=2001/q=8 POSTHOOK: Input: default@orc_a@y=2001/q=9 POSTHOOK: Input: default@orc_b POSTHOOK: Output: hdfs://### HDFS PATH ### -2000 2 6578 -2001 8 9438 +2000 0 6721 2000 3 6149 -2000 5 5720 -2000 9 8151 -2001 0 6721 -2001 1 7493 -2001 2 6578 -2001 4 7865 -2001 9 8151 -2000 1 7493 -2000 7 6149 +2000 4 7865 2000 8 9438 -2001 6 5577 +2001 5 5720 2001 7 6149 -2000 0 6721 -2000 4 7865 +2001 9 8151 +2001 6 5577 +2000 1 7493 +2000 2 6578 2000 6 5577 +2000 7 6149 +2001 0 6721 +2001 1 7493 +2001 2 6578 2001 3 6149 -2001 5 5720 +2001 8 9438 +2000 5 5720 +2000 9 8151 +2001 4 7865 PREHOOK: query: explain select y,q,count(*) from orc_a a join orc_b b on a.id=b.id group by y,q PREHOOK: type: QUERY @@ -234,7 +234,7 @@ STAGE PLANS: 0 id (type: bigint) 1 id (type: bigint) outputColumnNames: _col2, _col3 - Statistics: Num rows: 987 Data size: 7896 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 988 Data size: 7904 Basic stats: COMPLETE Column stats: COMPLETE Group By Operator aggregations: count() keys: _col2 (type: int), _col3 (type: smallint) @@ -321,26 +321,26 @@ POSTHOOK: Input: default@orc_a@y=2001/q=8 POSTHOOK: Input: default@orc_a@y=2001/q=9 POSTHOOK: Input: default@orc_b POSTHOOK: Output: hdfs://### HDFS PATH ### -2000 2 6578 -2001 8 9438 +2000 0 6721 2000 3 6149 -2000 5 5720 -2000 9 8151 -2001 0 6721 -2001 1 7493 -2001 2 6578 -2001 4 7865 -2001 9 8151 -2000 1 7493 -2000 7 6149 +2000 4 7865 2000 8 9438 -2001 6 5577 +2001 5 5720 2001 7 6149 -2000 0 6721 -2000 4 7865 +2001 9 8151 +2001 6 5577 +2000 1 7493 +2000 2 6578 2000 6 5577 +2000 7 6149 +2001 0 6721 +2001 1 7493 +2001 2 6578 2001 3 6149 -2001 5 5720 +2001 8 9438 +2000 5 5720 +2000 9 8151 +2001 4 7865 PREHOOK: query: DROP TABLE orc_a PREHOOK: type: DROPTABLE PREHOOK: Input: default@orc_a diff --git a/ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out b/ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out index c4361de5b2..89be93356a 100644 --- a/ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out +++ b/ql/src/test/results/clientpositive/llap/mapjoin_mapjoin.q.out @@ -112,6 +112,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -160,6 +161,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -208,6 +210,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -256,6 +259,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -311,6 +315,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -332,6 +337,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -386,6 +392,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -407,6 +414,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out index 89523042a9..7cd24a8d3c 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out @@ -46,6 +46,7 @@ Retention: 0 Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 5 rawDataSize 1025 @@ -106,6 +107,7 @@ Retention: 0 Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 5 rawDataSize 580 @@ -240,6 +242,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view5 POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +bucketing_version 2 key value numFiles 1 numRows 5 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out index 86e3c56627..0feebf358c 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite.q.out @@ -65,6 +65,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +bucketing_version 2 numFiles 1 numRows 2 rawDataSize 408 @@ -97,6 +98,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view2 POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +bucketing_version 2 numFiles 1 numRows 2 rawDataSize 232 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_2.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_2.q.out index 9c3c336145..c68b79c99f 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_2.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_2.q.out @@ -393,10 +393,10 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_mat_view_4 #### A masked pattern was here #### -bob alfred -bonnie +bob charlie +bonnie calvin PREHOOK: query: create table cmv_basetable_2 (a int, b varchar(256), c decimal(10,2), d int) stored as orc TBLPROPERTIES ('transactional'='true') PREHOOK: type: CREATETABLE @@ -499,8 +499,8 @@ POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 POSTHOOK: Input: default@cmv_mat_view_5 #### A masked pattern was here #### -3 1 +3 PREHOOK: query: explain select cmv_basetable.a from cmv_basetable join cmv_basetable_2 on (cmv_basetable.a = cmv_basetable_2.a) diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out index 1ef7b876d8..03167ad9fa 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_3.q.out @@ -278,8 +278,8 @@ POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 POSTHOOK: Input: default@cmv_mat_view #### A masked pattern was here #### -3 1 +3 PREHOOK: query: insert into cmv_basetable_2 values (3, 'charlie', 15.8, 1) PREHOOK: type: QUERY @@ -431,9 +431,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 #### A masked pattern was here #### +1 3 3 -1 PREHOOK: query: EXPLAIN ALTER MATERIALIZED VIEW cmv_mat_view REBUILD PREHOOK: type: QUERY @@ -645,9 +645,9 @@ POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 POSTHOOK: Input: default@cmv_mat_view #### A masked pattern was here #### +1 3 3 -1 PREHOOK: query: DELETE FROM cmv_basetable_2 WHERE a = 3 PREHOOK: type: QUERY PREHOOK: Input: default@cmv_basetable_2 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out index 86e3c56627..0feebf358c 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_dummy.q.out @@ -65,6 +65,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +bucketing_version 2 numFiles 1 numRows 2 rawDataSize 408 @@ -97,6 +98,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view2 POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +bucketing_version 2 numFiles 1 numRows 2 rawDataSize 232 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out index 8930b80422..50c9b040d2 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_multi_db.q.out @@ -89,6 +89,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +bucketing_version 2 numFiles 1 numRows 2 rawDataSize 408 @@ -121,6 +122,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view2 POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +bucketing_version 2 numFiles 1 numRows 2 rawDataSize 232 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out index 1ef7b876d8..03167ad9fa 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_rebuild_dummy.q.out @@ -278,8 +278,8 @@ POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 POSTHOOK: Input: default@cmv_mat_view #### A masked pattern was here #### -3 1 +3 PREHOOK: query: insert into cmv_basetable_2 values (3, 'charlie', 15.8, 1) PREHOOK: type: QUERY @@ -431,9 +431,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 #### A masked pattern was here #### +1 3 3 -1 PREHOOK: query: EXPLAIN ALTER MATERIALIZED VIEW cmv_mat_view REBUILD PREHOOK: type: QUERY @@ -645,9 +645,9 @@ POSTHOOK: Input: default@cmv_basetable POSTHOOK: Input: default@cmv_basetable_2 POSTHOOK: Input: default@cmv_mat_view #### A masked pattern was here #### +1 3 3 -1 PREHOOK: query: DELETE FROM cmv_basetable_2 WHERE a = 3 PREHOOK: type: QUERY PREHOOK: Input: default@cmv_basetable_2 diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out index efd60d4345..bea8e9487c 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_describe.q.out @@ -67,6 +67,7 @@ Retention: 0 Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 comment this is the first view key foo numFiles 1 @@ -93,6 +94,7 @@ PREHOOK: type: SHOW_TBLPROPERTIES POSTHOOK: query: show tblproperties cmv_mat_view POSTHOOK: type: SHOW_TBLPROPERTIES COLUMN_STATS_ACCURATE {"BASIC_STATS":"true"} +bucketing_version 2 comment this is the first view key foo numFiles 1 @@ -154,6 +156,7 @@ Retention: 0 Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 comment this is the second view key alice key2 bob @@ -234,6 +237,7 @@ Retention: 0 Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 comment this is the third view numFiles 1 numRows 5 @@ -323,6 +327,7 @@ Retention: 0 Table Type: MATERIALIZED_VIEW Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 comment this is the last view numFiles 1 numRows 5 diff --git a/ql/src/test/results/clientpositive/llap/metadataonly1.q.out b/ql/src/test/results/clientpositive/llap/metadataonly1.q.out index c705f6406f..52b35655d8 100644 --- a/ql/src/test/results/clientpositive/llap/metadataonly1.q.out +++ b/ql/src/test/results/clientpositive/llap/metadataonly1.q.out @@ -174,6 +174,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -315,6 +316,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -456,6 +458,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -604,6 +607,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -651,6 +655,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -728,6 +733,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -775,6 +781,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1012,6 +1019,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1059,6 +1067,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1106,6 +1115,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1257,6 +1267,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1305,6 +1316,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1353,6 +1365,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1500,6 +1513,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1546,6 +1560,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1631,8 +1646,8 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### -2008-04-08 2008-04-09 +2008-04-08 PREHOOK: query: select min(ds),max(ds) from srcpart PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -1745,6 +1760,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1792,6 +1808,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1839,6 +1856,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1886,6 +1904,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments @@ -1933,6 +1952,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns a,b columns.comments diff --git a/ql/src/test/results/clientpositive/llap/mm_exim.q.out b/ql/src/test/results/clientpositive/llap/mm_exim.q.out index 1f40754373..0f93037d30 100644 --- a/ql/src/test/results/clientpositive/llap/mm_exim.q.out +++ b/ql/src/test/results/clientpositive/llap/mm_exim.q.out @@ -214,7 +214,7 @@ POSTHOOK: Input: default@import0_mm #### A masked pattern was here #### 0 456 10 456 -97 455 +98 455 98 455 100 457 103 457 @@ -267,7 +267,7 @@ POSTHOOK: Input: default@import1_mm@p=457 #### A masked pattern was here #### 0 456 10 456 -97 455 +98 455 98 455 100 457 103 457 @@ -310,7 +310,7 @@ POSTHOOK: Input: default@import2_mm #### A masked pattern was here #### 0 456 10 456 -97 455 +98 455 98 455 100 457 103 457 @@ -366,7 +366,7 @@ POSTHOOK: Input: default@import3_mm@p=457 #### A masked pattern was here #### 0 456 10 456 -97 455 +98 455 98 455 100 457 103 457 @@ -408,7 +408,7 @@ POSTHOOK: Input: default@import4_mm #### A masked pattern was here #### 0 456 10 456 -97 455 +98 455 98 455 100 457 103 457 @@ -451,7 +451,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@import5_mm POSTHOOK: Input: default@import5_mm@p=455 #### A masked pattern was here #### -97 455 +98 455 98 455 PREHOOK: query: drop table import5_mm PREHOOK: type: DROPTABLE @@ -491,7 +491,7 @@ POSTHOOK: Input: default@import6_mm #### A masked pattern was here #### 0 456 10 456 -97 455 +98 455 98 455 100 457 103 457 @@ -542,7 +542,7 @@ POSTHOOK: Input: default@import7_mm@p=457 #### A masked pattern was here #### 0 456 10 456 -97 455 +98 455 98 455 100 457 103 457 diff --git a/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out b/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out index 559173c89e..3e267e3694 100644 --- a/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out +++ b/ql/src/test/results/clientpositive/llap/multiMapJoin1.q.out @@ -659,12 +659,12 @@ POSTHOOK: Input: default@smalltbl1 POSTHOOK: Input: default@smalltbl2 #### A masked pattern was here #### RUN: Stage-1:MAPRED +270 10 +270 10 10 10 -270 -270 PREHOOK: query: drop table bigTbl PREHOOK: type: DROPTABLE PREHOOK: Input: default@bigtbl diff --git a/ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out b/ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out index c210c4c20d..68a922a3f4 100644 --- a/ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out +++ b/ql/src/test/results/clientpositive/llap/multi_count_distinct_null.q.out @@ -291,24 +291,24 @@ group by department_id, gender, education_level grouping sets POSTHOOK: type: QUERY POSTHOOK: Input: default@employee POSTHOOK: Output: hdfs://### HDFS PATH ### -5 NULL NULL NULL -2 NULL NULL 1 -2 1 NULL 1 -2 1 NULL 3 -3 4 NULL NULL -2 4 NULL 1 3 NULL NULL NULL 6 NULL NULL NULL -6 NULL NULL 2 -5 NULL F NULL -3 1 NULL NULL -2 2 NULL 3 +6 NULL NULL 1 6 NULL NULL 3 3 2 NULL NULL 2 2 NULL 1 +5 NULL NULL NULL +5 NULL F NULL +3 1 NULL NULL +2 1 NULL 2 +2 4 NULL 1 2 NULL NULL NULL -6 NULL NULL 1 +2 NULL NULL 1 +6 NULL NULL 2 5 NULL M NULL -2 1 NULL 2 +2 1 NULL 1 +2 1 NULL 3 +2 2 NULL 3 3 3 NULL NULL 2 3 NULL 2 +3 4 NULL NULL diff --git a/ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out b/ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out index 39f1ea8495..d73d4d189a 100644 --- a/ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out +++ b/ql/src/test/results/clientpositive/llap/optimize_nullscan.q.out @@ -180,6 +180,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -201,6 +202,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -280,6 +282,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -327,6 +330,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -374,6 +378,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -421,6 +426,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -573,6 +579,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -594,6 +601,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -673,6 +681,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -721,6 +730,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -769,6 +779,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -817,6 +828,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -981,6 +993,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1002,6 +1015,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1081,6 +1095,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1128,6 +1143,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1175,6 +1191,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1222,6 +1239,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1389,6 +1407,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1410,6 +1429,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1459,6 +1479,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1480,6 +1501,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1622,6 +1644,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1643,6 +1666,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1693,6 +1717,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1714,6 +1739,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1839,6 +1865,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -1860,6 +1887,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' diff --git a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out index fd33bfd408..6e950ce41f 100644 --- a/ql/src/test/results/clientpositive/llap/orc_analyze.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_analyze.q.out @@ -98,6 +98,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 100 rawDataSize 52600 @@ -145,6 +146,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 100 rawDataSize 52600 @@ -231,6 +233,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"address\":\"true\",\"first_name\":\"true\",\"id\":\"true\",\"last_name\":\"true\",\"salary\":\"true\",\"start_date\":\"true\",\"state\":\"true\"}} + bucketing_version 2 numFiles 1 numRows 100 rawDataSize 52600 @@ -727,8 +730,8 @@ Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} numFiles 4 numRows 50 - rawDataSize 21975 - totalSize 5260 + rawDataSize 21963 + totalSize 5341 #### A masked pattern was here #### # Storage Information @@ -768,8 +771,8 @@ Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} numFiles 4 numRows 50 - rawDataSize 22043 - totalSize 5331 + rawDataSize 22069 + totalSize 5301 #### A masked pattern was here #### # Storage Information @@ -821,8 +824,8 @@ Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} numFiles 4 numRows 50 - rawDataSize 21975 - totalSize 5260 + rawDataSize 21963 + totalSize 5341 #### A masked pattern was here #### # Storage Information @@ -862,8 +865,8 @@ Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} numFiles 4 numRows 50 - rawDataSize 22043 - totalSize 5331 + rawDataSize 22069 + totalSize 5301 #### A masked pattern was here #### # Storage Information @@ -964,8 +967,8 @@ Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"address\":\"true\",\"first_name\":\"true\",\"id\":\"true\",\"last_name\":\"true\",\"salary\":\"true\",\"start_date\":\"true\"}} numFiles 4 numRows 50 - rawDataSize 21975 - totalSize 5260 + rawDataSize 21963 + totalSize 5341 #### A masked pattern was here #### # Storage Information @@ -1005,8 +1008,8 @@ Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"address\":\"true\",\"first_name\":\"true\",\"id\":\"true\",\"last_name\":\"true\",\"salary\":\"true\",\"start_date\":\"true\"}} numFiles 4 numRows 50 - rawDataSize 22043 - totalSize 5331 + rawDataSize 22069 + totalSize 5301 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/orc_create.q.out b/ql/src/test/results/clientpositive/llap/orc_create.q.out index 44373da600..0654fc75fa 100644 --- a/ql/src/test/results/clientpositive/llap/orc_create.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_create.q.out @@ -66,6 +66,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"lst\":\"true\",\"mp\":\"true\",\"str\":\"true\",\"strct\":\"true\"}} + bucketing_version 2 numFiles 0 numRows 0 rawDataSize 0 @@ -119,6 +120,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 0 numPartitions 0 numRows 0 @@ -176,6 +178,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 0 numPartitions 0 numRows 0 @@ -223,6 +226,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 #### A masked pattern was here #### numFiles 0 numPartitions 0 @@ -281,6 +285,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 0 numPartitions 0 numRows 0 @@ -336,6 +341,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"lst\":\"true\",\"mp\":\"true\",\"str\":\"true\",\"strct\":\"true\"}} + bucketing_version 2 numFiles 0 numRows 0 rawDataSize 0 diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index af237eab30..98d9e06ee0 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -227,6 +227,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 2100 orc.bloom.filter.columns * diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out index 01fb4afad9..c07832633d 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out @@ -227,6 +227,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 2100 orc.bloom.filter.columns * diff --git a/ql/src/test/results/clientpositive/llap/parquet_types.q.out b/ql/src/test/results/clientpositive/llap/parquet_types.q.out index 4d7c5fda28..c26401e16d 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_types.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_types.q.out @@ -369,7 +369,7 @@ POSTHOOK: query: SELECT hex(cbinary), count(*) FROM parquet_types GROUP BY cbina POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_types #### A masked pattern was here #### -68692CCAC0BDE7 10 - 1 68656C6C6F 1 +68692CCAC0BDE7 10 B4F3CAFDBEDD 10 + 1 diff --git a/ql/src/test/results/clientpositive/llap/parquet_types_vectorization.q.out b/ql/src/test/results/clientpositive/llap/parquet_types_vectorization.q.out index 1ccdff8aa2..34753882a8 100644 --- a/ql/src/test/results/clientpositive/llap/parquet_types_vectorization.q.out +++ b/ql/src/test/results/clientpositive/llap/parquet_types_vectorization.q.out @@ -843,7 +843,7 @@ POSTHOOK: query: SELECT hex(cbinary), count(*) FROM parquet_types GROUP BY cbina POSTHOOK: type: QUERY POSTHOOK: Input: default@parquet_types POSTHOOK: Output: hdfs://### HDFS PATH ### -68692CCAC0BDE7 10 +B4F3CAFDBEDD 10 1 68656C6C6F 1 -B4F3CAFDBEDD 10 +68692CCAC0BDE7 10 diff --git a/ql/src/test/results/clientpositive/llap/partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/partition_pruning.q.out index 520e7145f1..829ab15346 100644 --- a/ql/src/test/results/clientpositive/llap/partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/partition_pruning.q.out @@ -134,6 +134,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns customer columns.comments @@ -203,6 +204,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns customer columns.comments @@ -248,6 +250,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns customer columns.comments @@ -317,6 +320,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns customer columns.comments @@ -362,6 +366,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns customer columns.comments diff --git a/ql/src/test/results/clientpositive/llap/ppd_union_view.q.out b/ql/src/test/results/clientpositive/llap/ppd_union_view.q.out index 00b5e88efb..5a6ff13948 100644 --- a/ql/src/test/results/clientpositive/llap/ppd_union_view.q.out +++ b/ql/src/test/results/clientpositive/llap/ppd_union_view.q.out @@ -253,6 +253,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns keymap,value columns.comments @@ -334,6 +335,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,keymap columns.comments @@ -535,6 +537,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/quotedid_smb.q.out b/ql/src/test/results/clientpositive/llap/quotedid_smb.q.out index 67ab4fe105..1d6e002560 100644 --- a/ql/src/test/results/clientpositive/llap/quotedid_smb.q.out +++ b/ql/src/test/results/clientpositive/llap/quotedid_smb.q.out @@ -128,16 +128,16 @@ POSTHOOK: Input: default@src_b2 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 -103 val_103 103 val_103 -103 val_103 103 val_103 -103 val_103 103 val_103 -103 val_103 103 val_103 -105 val_105 105 val_105 -10 val_10 10 val_10 100 val_100 100 val_100 100 val_100 100 val_100 100 val_100 100 val_100 100 val_100 100 val_100 +105 val_105 105 val_105 +10 val_10 10 val_10 +103 val_103 103 val_103 +103 val_103 103 val_103 +103 val_103 103 val_103 +103 val_103 103 val_103 104 val_104 104 val_104 104 val_104 104 val_104 104 val_104 104 val_104 diff --git a/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out b/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out index 68c2af2b51..28b7aede1d 100644 --- a/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out +++ b/ql/src/test/results/clientpositive/llap/reduce_deduplicate.q.out @@ -60,6 +60,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -82,6 +83,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -124,6 +126,7 @@ STAGE PLANS: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -195,6 +198,7 @@ STAGE PLANS: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -385,6 +389,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns aid,bid,t,ctime,etime,l,et columns.comments @@ -474,6 +479,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns aid,bid,t,ctime,etime,l,et columns.comments diff --git a/ql/src/test/results/clientpositive/llap/sample1.q.out b/ql/src/test/results/clientpositive/llap/sample1.q.out index 3458ee28d0..583ac3571a 100644 --- a/ql/src/test/results/clientpositive/llap/sample1.q.out +++ b/ql/src/test/results/clientpositive/llap/sample1.q.out @@ -57,6 +57,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,dt,hr columns.comments @@ -130,6 +131,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -193,6 +195,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"dt":"true","hr":"true","key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value,dt,hr columns.comments diff --git a/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out b/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out index e76eaf862c..2aff794bcf 100644 --- a/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/schema_evol_stats.q.out @@ -78,6 +78,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 #### A masked pattern was here #### numFiles 2 numPartitions 2 @@ -283,6 +284,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 #### A masked pattern was here #### numFiles 2 numPartitions 2 diff --git a/ql/src/test/results/clientpositive/llap/skiphf_aggr.q.out b/ql/src/test/results/clientpositive/llap/skiphf_aggr.q.out index 74f98ed50d..c40aa54b06 100644 --- a/ql/src/test/results/clientpositive/llap/skiphf_aggr.q.out +++ b/ql/src/test/results/clientpositive/llap/skiphf_aggr.q.out @@ -91,8 +91,8 @@ POSTHOOK: Input: default@skiphtbl POSTHOOK: Input: default@skiphtbl@b=1 POSTHOOK: Input: default@skiphtbl@b=2 #### A masked pattern was here #### -4 2 +4 3 PREHOOK: query: INSERT OVERWRITE TABLE skipHTbl PARTITION (b = 1) VALUES (1) PREHOOK: type: QUERY @@ -150,8 +150,8 @@ POSTHOOK: Input: default@skiphtbl POSTHOOK: Input: default@skiphtbl@b=1 POSTHOOK: Input: default@skiphtbl@b=2 #### A masked pattern was here #### -4 2 +4 3 PREHOOK: query: DROP TABLE IF EXISTS skipFTbl PREHOOK: type: DROPTABLE diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out index 490afb69a9..ddb594255c 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_15.q.out @@ -89,6 +89,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -112,6 +113,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -169,6 +171,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -192,6 +195,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -403,6 +407,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -426,6 +431,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -483,6 +489,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -506,6 +513,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -665,6 +673,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -688,6 +697,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -745,6 +755,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -768,6 +779,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -927,6 +939,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -950,6 +963,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -1007,6 +1021,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments @@ -1030,6 +1045,7 @@ STAGE PLANS: SORTBUCKETCOLSPREFIX TRUE bucket_count 16 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,key2,value columns.comments diff --git a/ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out b/ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out index 97e74517ee..4be922d382 100644 --- a/ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out +++ b/ql/src/test/results/clientpositive/llap/smb_mapjoin_19.q.out @@ -158,7 +158,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table1 POSTHOOK: Input: default@test_table1@ds=1 #### A masked pattern was here #### -36 +26 PREHOOK: query: select count(*) from test_table1 tablesample (bucket 6 out of 16) s where ds = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table1 @@ -169,7 +169,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table1 POSTHOOK: Input: default@test_table1@ds=1 #### A masked pattern was here #### -40 +30 PREHOOK: query: select count(*) from test_table1 tablesample (bucket 13 out of 16) s where ds = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table1 @@ -180,7 +180,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table1 POSTHOOK: Input: default@test_table1@ds=1 #### A masked pattern was here #### -29 +50 PREHOOK: query: select count(*) from test_table2 where ds = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table2 @@ -233,7 +233,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table2 POSTHOOK: Input: default@test_table2@ds=1 #### A masked pattern was here #### -36 +26 PREHOOK: query: select count(*) from test_table2 tablesample (bucket 6 out of 16) s where ds = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table2 @@ -244,7 +244,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table2 POSTHOOK: Input: default@test_table2@ds=1 #### A masked pattern was here #### -40 +30 PREHOOK: query: select count(*) from test_table2 tablesample (bucket 13 out of 16) s where ds = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table2 @@ -255,4 +255,4 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table2 POSTHOOK: Input: default@test_table2@ds=1 #### A masked pattern was here #### -29 +50 diff --git a/ql/src/test/results/clientpositive/llap/stats11.q.out b/ql/src/test/results/clientpositive/llap/stats11.q.out index 22be814c9e..a53b296341 100644 --- a/ql/src/test/results/clientpositive/llap/stats11.q.out +++ b/ql/src/test/results/clientpositive/llap/stats11.q.out @@ -346,6 +346,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -367,6 +368,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -447,6 +449,7 @@ STAGE PLANS: properties: bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -494,6 +497,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -575,6 +579,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -766,6 +771,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -787,6 +793,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -867,6 +874,7 @@ STAGE PLANS: properties: bucket_count 4 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -914,6 +922,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments @@ -995,6 +1004,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value1":"true","value2":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value1,value2 columns.comments diff --git a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out index b63b25fb13..f4ca55d7c5 100644 --- a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_1.q.out @@ -404,10 +404,10 @@ order by c1 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### --8915 1 -3799 1 -10782 1 -13036 1 +10782 1 +-8915 1 NULL 6 PREHOOK: query: explain select diff --git a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out index 4f557d3cc3..44788ebba8 100644 --- a/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_dynpart_hashjoin_3.q.out @@ -126,7 +126,8 @@ STAGE PLANS: Tez #### A masked pattern was here #### Edges: - Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE), Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) + Reducer 3 <- Map 4 (CUSTOM_SIMPLE_EDGE), Reducer 2 (CUSTOM_SIMPLE_EDGE) #### A masked pattern was here #### Vertices: Map 1 @@ -148,7 +149,7 @@ STAGE PLANS: value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col2 (type: int), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) Execution mode: llap LLAP IO: all inputs - Map 3 + Map 4 Map Operator Tree: TableScan alias: b @@ -174,29 +175,39 @@ STAGE PLANS: Limit Number of rows: 1 Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE - Map Join Operator - condition map: - Left Outer Join 0 to 1 - filter predicates: - 0 {(_col2 < 100)} - 1 - keys: - 0 _col2 (type: int) - 1 UDFToInteger(_col0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 - input vertices: - 1 Map 3 + Reduce Output Operator + key expressions: _col2 (type: int) + sort order: + + Map-reduce partition columns: _col2 (type: int) Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE - Limit - Number of rows: 1 - Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + value expressions: _col0 (type: tinyint), _col1 (type: smallint), _col3 (type: bigint), _col4 (type: float), _col5 (type: double), _col6 (type: string), _col7 (type: string), _col8 (type: timestamp), _col9 (type: timestamp), _col10 (type: boolean), _col11 (type: boolean) + Reducer 3 + Execution mode: llap + Reduce Operator Tree: + Map Join Operator + condition map: + Left Outer Join 0 to 1 + filter predicates: + 0 {(_col2 < 100)} + 1 + keys: + 0 KEY.reducesinkkey0 (type: int) + 1 KEY.reducesinkkey0 (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11 + input vertices: + 1 Map 4 + Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE + HybridGraceHashJoin: true + Limit + Number of rows: 1 + Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 310 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe Stage: Stage-0 Fetch Operator diff --git a/ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out b/ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out index eee2ab6100..8b5f73ff29 100644 --- a/ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_join_result_complex.q.out @@ -198,6 +198,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns contact_event_id,ce_create_dt,ce_end_dt,contact_type,cnctevs_cd,contact_mode,cntvnst_stts_cd,total_transfers,ce_notes columns.comments @@ -222,6 +223,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns contact_event_id,ce_create_dt,ce_end_dt,contact_type,cnctevs_cd,contact_mode,cntvnst_stts_cd,total_transfers,ce_notes columns.comments @@ -308,6 +310,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns cnctevn_id,svcrqst_id,svcrqst_crt_dts,subject_seq_no,plan_component,cust_segment,cnctyp_cd,cnctmd_cd,cnctevs_cd,svcrtyp_cd,svrstyp_cd,cmpltyp_cd,catsrsn_cd,apealvl_cd,cnstnty_cd,svcrqst_asrqst_ind,svcrqst_rtnorig_in,svcrqst_vwasof_dt,sum_reason_cd,sum_reason,crsr_master_claim_index,svcrqct_cds,svcrqst_lupdt,crsr_lupdt,cntevsds_lupdt,ignore_me,notes columns.comments @@ -332,6 +335,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns cnctevn_id,svcrqst_id,svcrqst_crt_dts,subject_seq_no,plan_component,cust_segment,cnctyp_cd,cnctmd_cd,cnctevs_cd,svcrtyp_cd,svrstyp_cd,cmpltyp_cd,catsrsn_cd,apealvl_cd,cnstnty_cd,svcrqst_asrqst_ind,svcrqst_rtnorig_in,svcrqst_vwasof_dt,sum_reason_cd,sum_reason,crsr_master_claim_index,svcrqct_cds,svcrqst_lupdt,crsr_lupdt,cntevsds_lupdt,ignore_me,notes columns.comments @@ -1183,6 +1187,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns contact_event_id,ce_create_dt,ce_end_dt,contact_type,cnctevs_cd,contact_mode,cntvnst_stts_cd,total_transfers,ce_notes columns.comments @@ -1207,6 +1212,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns contact_event_id,ce_create_dt,ce_end_dt,contact_type,cnctevs_cd,contact_mode,cntvnst_stts_cd,total_transfers,ce_notes columns.comments @@ -1293,6 +1299,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns cnctevn_id,svcrqst_id,svcrqst_crt_dts,subject_seq_no,plan_component,cust_segment,cnctyp_cd,cnctmd_cd,cnctevs_cd,svcrtyp_cd,svrstyp_cd,cmpltyp_cd,catsrsn_cd,apealvl_cd,cnstnty_cd,svcrqst_asrqst_ind,svcrqst_rtnorig_in,svcrqst_vwasof_dt,sum_reason_cd,sum_reason,crsr_master_claim_index,svcrqct_cds,svcrqst_lupdt,crsr_lupdt,cntevsds_lupdt,ignore_me,notes columns.comments @@ -1317,6 +1324,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns cnctevn_id,svcrqst_id,svcrqst_crt_dts,subject_seq_no,plan_component,cust_segment,cnctyp_cd,cnctmd_cd,cnctevs_cd,svcrtyp_cd,svrstyp_cd,cmpltyp_cd,catsrsn_cd,apealvl_cd,cnstnty_cd,svcrqst_asrqst_ind,svcrqst_rtnorig_in,svcrqst_vwasof_dt,sum_reason_cd,sum_reason,crsr_master_claim_index,svcrqct_cds,svcrqst_lupdt,crsr_lupdt,cntevsds_lupdt,ignore_me,notes columns.comments diff --git a/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out b/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out index d0a5e62d72..45ce27c717 100644 --- a/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out +++ b/ql/src/test/results/clientpositive/llap/tez_vector_dynpart_hashjoin_1.q.out @@ -404,10 +404,10 @@ order by c1 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### +-13036 1 -8915 1 --3799 1 10782 1 --13036 1 +-3799 1 NULL 6 PREHOOK: query: explain select @@ -824,8 +824,8 @@ order by c1 POSTHOOK: type: QUERY POSTHOOK: Input: default@alltypesorc #### A masked pattern was here #### +-13036 1 -8915 1 -3799 1 10782 1 --13036 1 NULL 6 diff --git a/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out b/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out index 5cd5d41b09..4ca5f47a85 100644 --- a/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/union_fast_stats.q.out @@ -176,6 +176,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 3 numRows 15 rawDataSize 3483 @@ -228,6 +229,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 3 numRows 15 rawDataSize 3483 @@ -292,6 +294,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 4 numRows 20 rawDataSize 4552 @@ -506,10 +509,11 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 5 rawDataSize 1069 - totalSize 3243 + totalSize 3245 #### A masked pattern was here #### # Storage Information @@ -558,10 +562,11 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 1 numRows 15 rawDataSize 3320 - totalSize 3243 + totalSize 3245 #### A masked pattern was here #### # Storage Information @@ -622,10 +627,11 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numRows 20 rawDataSize 4389 - totalSize 4616 + totalSize 4618 #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/union_stats.q.out b/ql/src/test/results/clientpositive/llap/union_stats.q.out index ba2700c80d..45c98db7b4 100644 --- a/ql/src/test/results/clientpositive/llap/union_stats.q.out +++ b/ql/src/test/results/clientpositive/llap/union_stats.q.out @@ -62,6 +62,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -83,6 +84,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -146,6 +148,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -167,6 +170,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -251,6 +255,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numRows 1000 rawDataSize 10624 @@ -299,6 +304,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numRows 1500 rawDataSize 15936 @@ -355,6 +361,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 2 numRows 1500 rawDataSize 15936 diff --git a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out b/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out index e8bb722080..451a6e12e5 100644 --- a/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_coalesce_2.q.out @@ -168,9 +168,9 @@ GROUP BY str2 POSTHOOK: type: QUERY POSTHOOK: Input: default@str_str_orc #### A masked pattern was here #### -y 0.0 -NULL 0.0 X 0.02 +NULL 0.0 +y 0.0 PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT COALESCE(str1, 0) as result from str_str_orc @@ -393,9 +393,9 @@ GROUP BY str2 POSTHOOK: type: QUERY POSTHOOK: Input: default@str_str_orc #### A masked pattern was here #### -y 0.0 -NULL 0.0 X 0.02 +NULL 0.0 +y 0.0 PREHOOK: query: EXPLAIN VECTORIZATION EXPRESSION SELECT COALESCE(str1, 0) as result from str_str_orc diff --git a/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out b/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out index f2277c1521..a123a33fb8 100644 --- a/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_complex_all.q.out @@ -1112,9 +1112,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_create_complex #### A masked pattern was here #### lst strct +["g","h","i"] {"a":"five","b":"six"} ["a","b","c"] {"a":"one","b":"two"} ["d","e","f"] {"a":"three","b":"four"} -["g","h","i"] {"a":"five","b":"six"} PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT str, count(val) FROM orc_create_complex GROUP BY str PREHOOK: type: QUERY @@ -1253,9 +1253,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_create_complex #### A masked pattern was here #### str _c1 +line2 4501 line3 4501 line1 4501 -line2 4501 PREHOOK: query: EXPLAIN VECTORIZATION DETAIL SELECT strct.B, count(val) FROM orc_create_complex GROUP BY strct.B PREHOOK: type: QUERY @@ -1492,6 +1492,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@orc_create_complex #### A masked pattern was here #### strct mp lst str _c4 -{"a":"three","b":"four"} {"key21":"value21","key22":"value22","key23":"value23"} ["d","e","f"] line2 4501 {"a":"one","b":"two"} {"key11":"value11","key12":"value12","key13":"value13"} ["a","b","c"] line1 4501 +{"a":"three","b":"four"} {"key21":"value21","key22":"value22","key23":"value23"} ["d","e","f"] line2 4501 {"a":"five","b":"six"} {"key31":"value31","key32":"value32","key33":"value33"} ["g","h","i"] line3 4501 diff --git a/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out b/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out index ec3e2b8631..e31de7d5dd 100644 --- a/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_grouping_sets.q.out @@ -247,13 +247,13 @@ POSTHOOK: query: select s_store_id POSTHOOK: type: QUERY POSTHOOK: Input: default@store #### A masked pattern was here #### +AAAAAAAACAAAAAAA +NULL AAAAAAAABAAAAAAA AAAAAAAAEAAAAAAA AAAAAAAAHAAAAAAA -NULL AAAAAAAAIAAAAAAA AAAAAAAAKAAAAAAA -AAAAAAAACAAAAAAA PREHOOK: query: explain vectorization expression select s_store_id, GROUPING__ID from store @@ -385,13 +385,13 @@ POSTHOOK: query: select s_store_id, GROUPING__ID POSTHOOK: type: QUERY POSTHOOK: Input: default@store #### A masked pattern was here #### +AAAAAAAACAAAAAAA 0 +NULL 1 AAAAAAAABAAAAAAA 0 AAAAAAAAEAAAAAAA 0 AAAAAAAAHAAAAAAA 0 -NULL 1 AAAAAAAAIAAAAAAA 0 AAAAAAAAKAAAAAAA 0 -AAAAAAAACAAAAAAA 0 PREHOOK: query: explain select s_store_id, GROUPING__ID from store diff --git a/ql/src/test/results/clientpositive/llap/vector_windowing_multipartitioning.q.out b/ql/src/test/results/clientpositive/llap/vector_windowing_multipartitioning.q.out index 595b0a8d36..09ecb7750e 100644 --- a/ql/src/test/results/clientpositive/llap/vector_windowing_multipartitioning.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_windowing_multipartitioning.q.out @@ -10724,36 +10724,36 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@over10k #### A masked pattern was here #### s _c1 sum_window_1 -bob steinbeck 722083 38.33000183105469 -tom allen 1248023 89.88999938964844 -tom allen 1248023 83.47000122070312 -bob steinbeck 722083 47.810001373291016 -bob steinbeck 722083 68.46999740600586 -tom allen 1248023 68.46999740600586 -bob steinbeck 722083 28.479999542236328 -tom allen 1248023 2.8499999046325684 -bob steinbeck 722083 26.290000915527344 -bob steinbeck 722083 36.209999084472656 +tom allen 1248023 47.16999816894531 +tom allen 1248023 77.77999877929688 bob steinbeck 722083 83.52999877929688 +tom allen 1248023 83.47000122070312 +tom allen 1248023 15.84000015258789 tom allen 1248023 39.4900016784668 bob steinbeck 722083 80.7300033569336 -tom allen 1248023 77.77999877929688 -tom allen 1248023 26.239999771118164 -tom allen 1248023 95.41000366210938 -tom allen 1248023 81.8499984741211 tom allen 1248023 11.300000190734863 +tom allen 1248023 2.8499999046325684 +tom allen 1248023 19.459999084472656 +bob steinbeck 722083 26.290000915527344 +tom allen 1248023 14.510000228881836 +bob steinbeck 722083 83.52999877929688 +tom allen 1248023 52.779998779296875 +bob steinbeck 722083 9.699999809265137 tom allen 1248023 55.38999938964844 tom allen 1248023 132.82000350952148 bob steinbeck 722083 132.82000350952148 -tom allen 1248023 47.16999816894531 tom allen 1248023 11.069999694824219 -bob steinbeck 722083 83.52999877929688 -tom allen 1248023 19.459999084472656 -tom allen 1248023 14.510000228881836 +bob steinbeck 722083 38.33000183105469 +tom allen 1248023 26.239999771118164 +bob steinbeck 722083 47.810001373291016 +bob steinbeck 722083 68.46999740600586 +tom allen 1248023 68.46999740600586 +bob steinbeck 722083 28.479999542236328 +bob steinbeck 722083 36.209999084472656 +tom allen 1248023 95.41000366210938 +tom allen 1248023 89.88999938964844 tom allen 1248023 38.93000030517578 -tom allen 1248023 15.84000015258789 -tom allen 1248023 52.779998779296875 -bob steinbeck 722083 9.699999809265137 +tom allen 1248023 81.8499984741211 PREHOOK: query: explain vectorization detail select s, rank() over (partition by s order by bo), rank() over (partition by si order by bin desc) from over10k where s = 'tom allen' or s = 'bob steinbeck' diff --git a/ql/src/test/results/clientpositive/llap/vectorization_0.q.out b/ql/src/test/results/clientpositive/llap/vectorization_0.q.out index 1a846ab595..d3d982d70a 100644 --- a/ql/src/test/results/clientpositive/llap/vectorization_0.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorization_0.q.out @@ -1800,6 +1800,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2 columns.comments @@ -1821,6 +1822,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2 columns.comments @@ -30621,6 +30623,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2 columns.comments @@ -30642,6 +30645,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2 columns.comments @@ -30736,6 +30740,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2 columns.comments @@ -30757,6 +30762,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2 columns.comments @@ -30851,6 +30857,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2 columns.comments @@ -30872,6 +30879,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2 columns.comments @@ -30957,6 +30965,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2 columns.comments @@ -30978,6 +30987,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"cbigint":"true","cboolean1":"true","cboolean2":"true","cdouble":"true","cfloat":"true","cint":"true","csmallint":"true","cstring1":"true","cstring2":"true","ctimestamp1":"true","ctimestamp2":"true","ctinyint":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns ctinyint,csmallint,cint,cbigint,cfloat,cdouble,cstring1,cstring2,ctimestamp1,ctimestamp2,cboolean1,cboolean2 columns.comments diff --git a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out index 15b62c9781..e6b1b557c7 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_dynamic_partition_pruning.q.out @@ -14,8 +14,8 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### -2008-04-08 2008-04-09 +2008-04-08 PREHOOK: query: select distinct hr from srcpart PREHOOK: type: QUERY PREHOOK: Input: default@srcpart @@ -4199,8 +4199,8 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### -2008-04-08 2008-04-09 +2008-04-08 PREHOOK: query: EXPLAIN VECTORIZATION select ds from (select distinct(ds) as ds from srcpart union all select distinct(ds) as ds from srcpart) s where s.ds in (select max(srcpart.ds) from srcpart union all select min(srcpart.ds) from srcpart) PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN VECTORIZATION select ds from (select distinct(ds) as ds from srcpart union all select distinct(ds) as ds from srcpart) s where s.ds in (select max(srcpart.ds) from srcpart union all select min(srcpart.ds) from srcpart) @@ -6729,8 +6729,8 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12 #### A masked pattern was here #### -2008-04-08 2008-04-09 +2008-04-08 PREHOOK: query: create table srcpart_orc (key int, value string) partitioned by (ds string, hr int) stored as orc PREHOOK: type: CREATETABLE PREHOOK: Output: database:default diff --git a/ql/src/test/results/clientpositive/llap/vectorized_insert_into_bucketed_table.q.out b/ql/src/test/results/clientpositive/llap/vectorized_insert_into_bucketed_table.q.out index cd2f6e9c2e..5afd8ec1be 100644 --- a/ql/src/test/results/clientpositive/llap/vectorized_insert_into_bucketed_table.q.out +++ b/ql/src/test/results/clientpositive/llap/vectorized_insert_into_bucketed_table.q.out @@ -126,9 +126,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@foo #### A masked pattern was here #### 4 000000_0 -9 000001_0 +2 000001_0 1 000001_0 -2 000002_0 +9 000003_0 3 000003_0 PREHOOK: query: drop table foo PREHOOK: type: DROPTABLE diff --git a/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out b/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out index d7967d1d10..041d28b257 100644 --- a/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out +++ b/ql/src/test/results/clientpositive/spark/bucketmapjoin7.q.out @@ -135,6 +135,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -241,6 +242,7 @@ STAGE PLANS: properties: bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/spark/infer_bucket_sort_bucketed_table.q.out b/ql/src/test/results/clientpositive/spark/infer_bucket_sort_bucketed_table.q.out index fbb1731956..231194c1cc 100644 --- a/ql/src/test/results/clientpositive/spark/infer_bucket_sort_bucketed_table.q.out +++ b/ql/src/test/results/clientpositive/spark/infer_bucket_sort_bucketed_table.q.out @@ -67,7 +67,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table_bucketed POSTHOOK: Input: default@test_table_bucketed@part=1 POSTHOOK: Output: hdfs://### HDFS PATH ### -31 +0 PREHOOK: query: SELECT COUNT(*) FROM test_table_bucketed TABLESAMPLE (BUCKET 2 OUT OF 3) WHERE part = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table_bucketed @@ -78,7 +78,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table_bucketed POSTHOOK: Input: default@test_table_bucketed@part=1 POSTHOOK: Output: hdfs://### HDFS PATH ### -179 +0 PREHOOK: query: SELECT COUNT(*) FROM test_table_bucketed TABLESAMPLE (BUCKET 3 OUT OF 3) WHERE part = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table_bucketed @@ -89,7 +89,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table_bucketed POSTHOOK: Input: default@test_table_bucketed@part=1 POSTHOOK: Output: hdfs://### HDFS PATH ### -99 +0 PREHOOK: query: SELECT cnt FROM (SELECT INPUT__FILE__NAME, COUNT(*) cnt FROM test_table_bucketed WHERE part = '1' GROUP BY INPUT__FILE__NAME ORDER BY INPUT__FILE__NAME ASC LIMIT 3) a PREHOOK: type: QUERY @@ -102,6 +102,6 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table_bucketed POSTHOOK: Input: default@test_table_bucketed@part=1 POSTHOOK: Output: hdfs://### HDFS PATH ### -31 -179 -99 +204 +101 +4 diff --git a/ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out b/ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out index 1bdf64158b..9f69620a41 100644 --- a/ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out +++ b/ql/src/test/results/clientpositive/spark/infer_bucket_sort_num_buckets.q.out @@ -129,7 +129,7 @@ Table: test_table #### A masked pattern was here #### Partition Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - numFiles 2 + numFiles 1 numRows 670 rawDataSize 5888 totalSize 6558 diff --git a/ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out b/ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out index 4bde96aa65..57425752e3 100644 --- a/ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out +++ b/ql/src/test/results/clientpositive/spark/list_bucket_dml_10.q.out @@ -55,6 +55,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -84,6 +85,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -106,6 +108,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -140,6 +143,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments diff --git a/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out b/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out index 8e9d2bb20b..9806647548 100644 --- a/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out +++ b/ql/src/test/results/clientpositive/spark/parallel_orderby.q.out @@ -118,6 +118,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 4 numRows 48 rawDataSize 512 @@ -228,6 +229,7 @@ Retention: 0 Table Type: MANAGED_TABLE Table Parameters: COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + bucketing_version 2 numFiles 4 numRows 48 rawDataSize 512 diff --git a/ql/src/test/results/clientpositive/spark/quotedid_smb.q.out b/ql/src/test/results/clientpositive/spark/quotedid_smb.q.out index c8b2e465d7..caeac14385 100644 --- a/ql/src/test/results/clientpositive/spark/quotedid_smb.q.out +++ b/ql/src/test/results/clientpositive/spark/quotedid_smb.q.out @@ -116,16 +116,16 @@ POSTHOOK: Output: hdfs://### HDFS PATH ### 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 0 val_0 -103 val_103 103 val_103 -103 val_103 103 val_103 -103 val_103 103 val_103 -103 val_103 103 val_103 -105 val_105 105 val_105 -10 val_10 10 val_10 100 val_100 100 val_100 100 val_100 100 val_100 100 val_100 100 val_100 100 val_100 100 val_100 +105 val_105 105 val_105 +10 val_10 10 val_10 +103 val_103 103 val_103 +103 val_103 103 val_103 +103 val_103 103 val_103 +103 val_103 103 val_103 104 val_104 104 val_104 104 val_104 104 val_104 104 val_104 104 val_104 diff --git a/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out b/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out index 4475c6e5be..d00cdafdca 100644 --- a/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out +++ b/ql/src/test/results/clientpositive/spark/reduce_deduplicate.q.out @@ -56,6 +56,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -78,6 +79,7 @@ STAGE PLANS: properties: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count -1 + bucketing_version 2 column.name.delimiter , columns key,value columns.comments 'default','default' @@ -119,6 +121,7 @@ STAGE PLANS: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -152,6 +155,7 @@ STAGE PLANS: COLUMN_STATS_ACCURATE {"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} bucket_count 2 bucket_field_name key + bucketing_version 2 column.name.delimiter , columns key,value columns.comments @@ -334,6 +338,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns aid,bid,t,ctime,etime,l,et columns.comments @@ -365,6 +370,7 @@ STAGE PLANS: output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 + bucketing_version 2 column.name.delimiter , columns aid,bid,t,ctime,etime,l,et columns.comments diff --git a/ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out b/ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out index 71267d6ac4..41f625283b 100644 --- a/ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out +++ b/ql/src/test/results/clientpositive/spark/smb_mapjoin_19.q.out @@ -148,7 +148,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table1 POSTHOOK: Input: default@test_table1@ds=1 #### A masked pattern was here #### -36 +26 PREHOOK: query: select count(*) from test_table1 tablesample (bucket 6 out of 16) s where ds = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table1 @@ -159,7 +159,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table1 POSTHOOK: Input: default@test_table1@ds=1 #### A masked pattern was here #### -40 +30 PREHOOK: query: select count(*) from test_table1 tablesample (bucket 13 out of 16) s where ds = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table1 @@ -170,7 +170,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table1 POSTHOOK: Input: default@test_table1@ds=1 #### A masked pattern was here #### -29 +50 PREHOOK: query: select count(*) from test_table2 where ds = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table2 @@ -223,7 +223,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table2 POSTHOOK: Input: default@test_table2@ds=1 #### A masked pattern was here #### -36 +26 PREHOOK: query: select count(*) from test_table2 tablesample (bucket 6 out of 16) s where ds = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table2 @@ -234,7 +234,7 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table2 POSTHOOK: Input: default@test_table2@ds=1 #### A masked pattern was here #### -40 +30 PREHOOK: query: select count(*) from test_table2 tablesample (bucket 13 out of 16) s where ds = '1' PREHOOK: type: QUERY PREHOOK: Input: default@test_table2 @@ -245,4 +245,4 @@ POSTHOOK: type: QUERY POSTHOOK: Input: default@test_table2 POSTHOOK: Input: default@test_table2@ds=1 #### A masked pattern was here #### -29 +50 diff --git a/ql/src/test/results/clientpositive/spark/truncate_column_buckets.q.out b/ql/src/test/results/clientpositive/spark/truncate_column_buckets.q.out index edeab0b9b1..858f780c10 100644 --- a/ql/src/test/results/clientpositive/spark/truncate_column_buckets.q.out +++ b/ql/src/test/results/clientpositive/spark/truncate_column_buckets.q.out @@ -30,8 +30,8 @@ test_tab GROUP BY INPUT__FILE__NAME POSTHOOK: type: QUERY POSTHOOK: Input: default@test_tab POSTHOOK: Output: hdfs://### HDFS PATH ### -258 -242 +267 +233 PREHOOK: query: TRUNCATE TABLE test_tab COLUMNS (value) PREHOOK: type: TRUNCATETABLE PREHOOK: Input: default@test_tab @@ -54,5 +54,5 @@ test_tab GROUP BY INPUT__FILE__NAME POSTHOOK: type: QUERY POSTHOOK: Input: default@test_tab POSTHOOK: Output: hdfs://### HDFS PATH ### -258 -242 +267 +233 diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java index e5b90c25e9..1dc9de7273 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/binarysortable/BinarySortableSerDe.java @@ -700,7 +700,7 @@ public static void writeByte(RandomAccessOutput buffer, byte b, boolean invert) buffer.write(b); } - static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, + public static void serialize(ByteStream.Output buffer, Object o, ObjectInspector oi, boolean invert, byte nullMarker, byte notNullMarker) throws SerDeException { // Is this field a null? if (o == null) { diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java index 8823d41274..1e0d1f3904 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java @@ -21,6 +21,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.lang.reflect.Type; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -28,9 +29,12 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.hive.serde2.ByteStream; +import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe; import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable; import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestampLocalTZObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector; +import org.apache.hive.common.util.Murmur3; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.serde.serdeConstants; @@ -154,7 +158,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return ObjectInspectorUtils.getBucketHashCode(objects, oi); + return ObjectInspectorUtils.getBucketHashCodeOld(objects, oi); } } @@ -618,12 +622,24 @@ public static String getObjectInspectorName(ObjectInspector oi) { * @param bucketFields the bucketed fields of the row * @param bucketFieldInspectors the ObjectInpsectors for each of the bucketed fields * @param totalBuckets the number of buckets in the table - * @return the bucket number + * @return the bucket number using Murmur hash */ public static int getBucketNumber(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors, int totalBuckets) { return getBucketNumber(getBucketHashCode(bucketFields, bucketFieldInspectors), totalBuckets); } + /** + * Computes the bucket number to which the bucketFields belong to + * @param bucketFields the bucketed fields of the row + * @param bucketFieldInspectors the ObjectInpsectors for each of the bucketed fields + * @param totalBuckets the number of buckets in the table + * @return the bucket number + */ + @Deprecated + public static int getBucketNumberOld(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors, int totalBuckets) { + return getBucketNumber(getBucketHashCodeOld(bucketFields, bucketFieldInspectors), totalBuckets); + } + /** * https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL+BucketedTables * @param hashCode as produced by {@link #getBucketHashCode(Object[], ObjectInspector[])} @@ -636,13 +652,15 @@ public static int getBucketNumber(int hashCode, int numberOfBuckets) { } return (hashCode & Integer.MAX_VALUE) % numberOfBuckets; } + /** * Computes the hash code for the given bucketed fields * @param bucketFields * @param bucketFieldInspectors * @return */ - public static int getBucketHashCode(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors) { + @Deprecated + public static int getBucketHashCodeOld(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors) { int hashCode = 0; for (int i = 0; i < bucketFields.length; i++) { int fieldHash = ObjectInspectorUtils.hashCode(bucketFields[i], bucketFieldInspectors[i]); @@ -651,7 +669,6 @@ public static int getBucketHashCode(Object[] bucketFields, ObjectInspector[] buc return hashCode; } - public static int hashCode(Object o, ObjectInspector objIns) { if (o == null) { return 0; @@ -769,6 +786,139 @@ public static int hashCode(Object o, ObjectInspector objIns) { } } + public static int getBucketHashCode(Object[] bucketFields, ObjectInspector[] bucketFieldInspectors) { + int hashCode = 0; + for (int i = 0; i < bucketFields.length; i++) { + int fieldHash = ObjectInspectorUtils.hashCodeMurmur(bucketFields[i], bucketFieldInspectors[i]); + hashCode = 31 * hashCode + fieldHash; + } + return hashCode; + } + + public static int hashCodeMurmur(Object o, ObjectInspector objIns) { + if (o == null) { + return 0; + } + switch (objIns.getCategory()) { + case PRIMITIVE: { + PrimitiveObjectInspector poi = ((PrimitiveObjectInspector) objIns); + switch (poi.getPrimitiveCategory()) { + case VOID: + return 0; + case BOOLEAN: + return (((BooleanObjectInspector) poi).get(o) ? 1 : 0); + case BYTE: + return ((ByteObjectInspector) poi).get(o); + case SHORT: { + ByteBuffer b = ByteBuffer.allocate(2); + b.putShort(((ShortObjectInspector) poi).get(o)); + return Murmur3.hash32(b.array()); + } + case INT: { + ByteBuffer b = ByteBuffer.allocate(4); + b.putInt(((IntObjectInspector) poi).get(o)); + return Murmur3.hash32(b.array()); + } + case LONG: { + ByteBuffer b = ByteBuffer.allocate(8); + b.putLong(((LongObjectInspector) poi).get(o)); + return Murmur3.hash32(b.array()); + } + case FLOAT: { + ByteBuffer b = ByteBuffer.allocate(4); + b.putFloat(Float.floatToIntBits(((FloatObjectInspector) poi).get(o))); + return Murmur3.hash32(b.array()); + } + case DOUBLE: { + // This hash function returns the same result as Double.hashCode() + // while DoubleWritable.hashCode returns a different result. + ByteBuffer b = ByteBuffer.allocate(8); + b.putDouble(Double.doubleToLongBits(((DoubleObjectInspector) poi).get(o))); + return Murmur3.hash32(b.array()); + } + case STRING: { + // This hash function returns the same result as String.hashCode() when + // all characters are ASCII, while Text.hashCode() always returns a + // different result. + return Murmur3.hash32((((StringObjectInspector) poi).getPrimitiveWritableObject(o)).getBytes()); + } + case CHAR: + return Murmur3.hash32(((HiveCharObjectInspector) poi).getPrimitiveWritableObject(o).getStrippedValue().getBytes()); + case VARCHAR: + return Murmur3.hash32(((HiveVarcharObjectInspector)poi).getPrimitiveWritableObject(o).getTextValue().getBytes()); + case BINARY: + return Murmur3.hash32(((BinaryObjectInspector) poi).getPrimitiveWritableObject(o).getBytes()); + + case DATE: + return Murmur3.hash32(((DateObjectInspector) poi).getPrimitiveWritableObject(o).toString().getBytes()); + case TIMESTAMP: { + TimestampWritable t = ((TimestampObjectInspector) poi) + .getPrimitiveWritableObject(o); + return Murmur3.hash32(t.getBytes()); + } + case TIMESTAMPLOCALTZ: + return Murmur3.hash32((((TimestampLocalTZObjectInspector) poi).getPrimitiveWritableObject(o)).getBytes()); + case INTERVAL_YEAR_MONTH: + HiveIntervalYearMonthWritable intervalYearMonth = ((HiveIntervalYearMonthObjectInspector) poi) + .getPrimitiveWritableObject(o); + return Murmur3.hash32(intervalYearMonth.toString().getBytes()); + case INTERVAL_DAY_TIME: + HiveIntervalDayTimeWritable intervalDayTime = ((HiveIntervalDayTimeObjectInspector) poi) + .getPrimitiveWritableObject(o); + return Murmur3.hash32(intervalDayTime.toString().getBytes()); + case DECIMAL: + // Since getBucketHashCode uses this, HiveDecimal return the old (much slower) but + // compatible hash code. + return Murmur3.hash32(((HiveDecimalObjectInspector) poi).getPrimitiveWritableObject(o).getInternalStorage()); + + default: { + throw new RuntimeException("Unknown type: " + + poi.getPrimitiveCategory()); + } + } + } + case LIST: { + int r = 0; + ListObjectInspector listOI = (ListObjectInspector)objIns; + ObjectInspector elemOI = listOI.getListElementObjectInspector(); + for (int ii = 0; ii < listOI.getListLength(o); ++ii) { + //r = 31 * r + hashCode(listOI.getListElement(o, ii), elemOI); + r = 31 * r + hashCodeMurmur(listOI.getListElement(o, ii), elemOI); + } + return r; + } + case MAP: { + int r = 0; + MapObjectInspector mapOI = (MapObjectInspector)objIns; + ObjectInspector keyOI = mapOI.getMapKeyObjectInspector(); + ObjectInspector valueOI = mapOI.getMapValueObjectInspector(); + Map map = mapOI.getMap(o); + for (Map.Entry entry : map.entrySet()) { + r += hashCodeMurmur(entry.getKey(), keyOI) ^ + hashCode(entry.getValue(), valueOI); + } + return r; + } + case STRUCT: + int r = 0; + StructObjectInspector structOI = (StructObjectInspector)objIns; + List fields = structOI.getAllStructFieldRefs(); + for (StructField field : fields) { + r = 31 * r + hashCodeMurmur(structOI.getStructFieldData(o, field), + field.getFieldObjectInspector()); + } + return r; + + case UNION: + UnionObjectInspector uOI = (UnionObjectInspector)objIns; + byte tag = uOI.getTag(o); + return hashCodeMurmur(uOI.getField(o), uOI.getObjectInspectors().get(tag)); + + default: + throw new RuntimeException("Unknown type: "+ objIns.getTypeName()); + } + } + /** * Compare two arrays of objects with their respective arrays of * ObjectInspectors. diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java index eca74057a3..23ca194b6f 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorUtils.java @@ -150,9 +150,9 @@ public void testBucketIdGeneration() { struct.add("two"); struct.add(true); - int hashCode = ObjectInspectorUtils.getBucketHashCode(struct.toArray(), fieldObjectInspectors.toArray(new ObjectInspector[fieldObjectInspectors.size()])); + int hashCode = ObjectInspectorUtils.getBucketHashCodeOld(struct.toArray(), fieldObjectInspectors.toArray(new ObjectInspector[fieldObjectInspectors.size()])); assertEquals("", 3574518, hashCode); - int bucketId = ObjectInspectorUtils.getBucketNumber(struct.toArray(), fieldObjectInspectors.toArray(new ObjectInspector[fieldObjectInspectors.size()]), 16); + int bucketId = ObjectInspectorUtils.getBucketNumberOld(struct.toArray(), fieldObjectInspectors.toArray(new ObjectInspector[fieldObjectInspectors.size()]), 16); assertEquals("", 6, bucketId); assertEquals("", bucketId, ObjectInspectorUtils.getBucketNumber(hashCode, 16)); } diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java index cb1d40a4a8..6d4457945b 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/hive_metastoreConstants.java @@ -84,4 +84,7 @@ public static final String TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties"; + public static final String TABLE_BUCKETING_VERSION = "bucketing_version"; + + public static final String BUCKETED_TABLE_LOADED_IN_EXPERT_MODE = "bucketed_table_loaded_in_expert_mode"; } diff --git a/storage-api/src/java/org/apache/hive/common/util/Murmur3.java b/storage-api/src/java/org/apache/hive/common/util/Murmur3.java index 6848a59563..1ceeeb3868 100644 --- a/storage-api/src/java/org/apache/hive/common/util/Murmur3.java +++ b/storage-api/src/java/org/apache/hive/common/util/Murmur3.java @@ -61,7 +61,7 @@ * @return - hashcode */ public static int hash32(byte[] data) { - return hash32(data, data.length, DEFAULT_SEED); + return hash32(data, 0, data.length, DEFAULT_SEED); } /** @@ -73,16 +73,29 @@ public static int hash32(byte[] data) { * @return - hashcode */ public static int hash32(byte[] data, int length, int seed) { + return hash32(data, 0, length, seed); + } + + /** + * Murmur3 32-bit variant. + * + * @param data - input byte array + * @param offset - offset of data + * @param length - length of array + * @param seed - seed. (default 0) + * @return - hashcode + */ + public static int hash32(byte[] data, int offset, int length, int seed) { int hash = seed; final int nblocks = length >> 2; // body for (int i = 0; i < nblocks; i++) { int i_4 = i << 2; - int k = (data[i_4] & 0xff) - | ((data[i_4 + 1] & 0xff) << 8) - | ((data[i_4 + 2] & 0xff) << 16) - | ((data[i_4 + 3] & 0xff) << 24); + int k = (data[offset + i_4] & 0xff) + | ((data[offset + i_4 + 1] & 0xff) << 8) + | ((data[offset + i_4 + 2] & 0xff) << 16) + | ((data[offset + i_4 + 3] & 0xff) << 24); // mix functions k *= C1_32; @@ -97,11 +110,11 @@ public static int hash32(byte[] data, int length, int seed) { int k1 = 0; switch (length - idx) { case 3: - k1 ^= data[idx + 2] << 16; + k1 ^= data[offset + idx + 2] << 16; case 2: - k1 ^= data[idx + 1] << 8; + k1 ^= data[offset + idx + 1] << 8; case 1: - k1 ^= data[idx]; + k1 ^= data[offset + idx]; // mix functions k1 *= C1_32;