diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index c9924333d7fe84ceb35d1d1249d8e8fca31b9352..98c63728db610c10f19a2689dd0033d8bab25597 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1213,6 +1213,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_ORC_MS_FOOTER_CACHE_ENABLED("hive.orc.splits.ms.footer.cache.enabled", false, "Whether to enable using file metadata cache in metastore for ORC file footers."), + HIVE_ORC_MS_FOOTER_CACHE_PPD("hive.orc.splits.ms.footer.cache.ppd.enabled", true, + "Whether to enable file footer cache PPD (hive.orc.splits.ms.footer.cache.enabled\n" + + "must also be set to true for this to work)."), HIVE_ORC_INCLUDE_FILE_FOOTER_IN_SPLITS("hive.orc.splits.include.file.footer", false, "If turned on splits generated by orc will include metadata about the stripes in the file. This\n" + @@ -1222,7 +1225,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "generation. 0 means process directories individually. This can increase the number of\n" + "metastore calls if metastore metadata cache is used."), HIVE_ORC_INCLUDE_FILE_ID_IN_SPLITS("hive.orc.splits.include.fileid", true, - "Include file ID in splits on file systems thaty support it."), + "Include file ID in splits on file systems that support it."), + HIVE_ORC_ALLOW_SYNTHETIC_FILE_ID_IN_SPLITS("hive.orc.splits.allow.synthetic.fileid", true, + "Allow synthetic file ID in splits on file systems that don't have a native one."), HIVE_ORC_CACHE_STRIPE_DETAILS_SIZE("hive.orc.cache.stripe.details.size", 10000, "Max cache size for keeping meta info about orc splits cached in the client."), HIVE_ORC_COMPUTE_SPLITS_NUM_THREADS("hive.orc.compute.splits.num.threads", 10, diff --git a/common/src/java/org/apache/hive/common/util/StreamPrinter.java b/common/src/java/org/apache/hive/common/util/StreamPrinter.java index 72638fdb966a63c64e44f2d952ecc49b4b0b5121..1517751347490a9e3ed4b11c2981cfed24745102 100644 --- a/common/src/java/org/apache/hive/common/util/StreamPrinter.java +++ b/common/src/java/org/apache/hive/common/util/StreamPrinter.java @@ -33,12 +33,12 @@ public class StreamPrinter extends Thread { InputStream is; String type; - PrintStream os; + PrintStream[] outputStreams; - public StreamPrinter(InputStream is, String type, PrintStream os) { + public StreamPrinter(InputStream is, String type, PrintStream... outputStreams) { this.is = is; this.type = type; - this.os = os; + this.outputStreams = outputStreams; } @Override @@ -50,18 +50,22 @@ public void run() { String line = null; if (type != null) { while ((line = br.readLine()) != null) { - os.println(type + ">" + line); + for (PrintStream os: outputStreams) { + os.println(type + ">" + line); + } } } else { while ((line = br.readLine()) != null) { - os.println(line); + for (PrintStream os: outputStreams) { + os.println(line); + } } } br.close(); - br=null; + br = null; } catch (IOException ioe) { ioe.printStackTrace(); - }finally{ + } finally { IOUtils.closeStream(br); } } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java index d33f7242c1407c3c37feb60d1e31fb5b22df6c55..7d6829476ff4a4f3856ec1d4b2490c1024cef680 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/IncrementalObjectSizeEstimator.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.llap; +import com.google.common.collect.Lists; +import com.google.protobuf.UnknownFieldSet; import java.lang.reflect.AccessibleObject; import java.lang.reflect.Array; import java.lang.reflect.Field; @@ -35,13 +37,11 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator.ObjectEstimator; import org.apache.hadoop.hive.llap.cache.LlapCacheableBuffer; import org.apache.hadoop.hive.llap.io.api.impl.LlapIoImpl; import org.apache.hadoop.hive.ql.util.JavaDataModel; -import com.google.common.collect.Lists; -import com.google.protobuf.UnknownFieldSet; - /** * Creates size estimators for java objects. The estimators attempt to do most of the reflection * work at initialization time, and also take some shortcuts, to minimize the amount of work done @@ -622,6 +622,7 @@ public static void addEstimator(String className, } catch (ClassNotFoundException e) { // Ignore and hope for the best. LlapIoImpl.LOG.warn("Cannot find " + className); + return; } IncrementalObjectSizeEstimator.createEstimators(clazz, sizeEstimators); } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/Cache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/Cache.java deleted file mode 100644 index cee23a928bdc0e7046dd1cae26f613e4426aeb1c..0000000000000000000000000000000000000000 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/Cache.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.llap.cache; - -import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData; - -/** Dummy interface for now, might be different. */ -public interface Cache { - public ColumnStreamData[] cacheOrGet(CacheKey key, ColumnStreamData[] value); - public ColumnStreamData[] get(CacheKey key); -} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java index 17d9fdf264129d8789fea2ed44799d105ba79b6d..1b61a6e9e48f1cb13869121f293c612e961fd6f0 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCache.java @@ -49,7 +49,7 @@ * Some sort of InvalidCacheChunk could be placed to avoid them. TODO * @param base base offset for the ranges (stripe/stream offset in case of ORC). */ - DiskRangeList getFileData(long fileId, DiskRangeList range, long baseOffset, + DiskRangeList getFileData(Object fileKey, DiskRangeList range, long baseOffset, DiskRangeListFactory factory, LowLevelCacheCounters qfCounters, BooleanRef gotAllData); /** @@ -57,6 +57,6 @@ DiskRangeList getFileData(long fileId, DiskRangeList range, long baseOffset, * @return null if all data was put; bitmask indicating which chunks were not put otherwise; * the replacement chunks from cache are updated directly in the array. */ - long[] putFileData(long fileId, DiskRange[] ranges, MemoryBuffer[] chunks, + long[] putFileData(Object fileKey, DiskRange[] ranges, MemoryBuffer[] chunks, long baseOffset, Priority priority, LowLevelCacheCounters qfCounters); } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java index 1132171b91d61447f58df627713f830f8572ccbf..a60fed3be7d1d64eeec97d249f08a2dfa150ba47 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/LowLevelCacheImpl.java @@ -43,8 +43,8 @@ private final EvictionAwareAllocator allocator; private final AtomicInteger newEvictions = new AtomicInteger(0); private Thread cleanupThread = null; - private final ConcurrentHashMap cache = - new ConcurrentHashMap(); + private final ConcurrentHashMap cache = + new ConcurrentHashMap(); private final LowLevelCachePolicy cachePolicy; private final long cleanupInterval; private final LlapDaemonCacheMetrics metrics; @@ -75,11 +75,11 @@ public void init() { } @Override - public DiskRangeList getFileData(long fileId, DiskRangeList ranges, long baseOffset, + public DiskRangeList getFileData(Object fileKey, DiskRangeList ranges, long baseOffset, DiskRangeListFactory factory, LowLevelCacheCounters qfCounters, BooleanRef gotAllData) { if (ranges == null) return null; DiskRangeList prev = ranges.prev; - FileCache subCache = cache.get(fileId); + FileCache subCache = cache.get(fileKey); if (subCache == null || !subCache.incRef()) { long totalMissed = ranges.getTotalLength(); metrics.incrCacheRequestedBytes(totalMissed); @@ -232,11 +232,11 @@ private boolean lockBuffer(LlapDataBuffer buffer, boolean doNotifyPolicy) { } @Override - public long[] putFileData(long fileId, DiskRange[] ranges, MemoryBuffer[] buffers, + public long[] putFileData(Object fileKey, DiskRange[] ranges, MemoryBuffer[] buffers, long baseOffset, Priority priority, LowLevelCacheCounters qfCounters) { long[] result = null; assert buffers.length == ranges.length; - FileCache subCache = getOrAddFileSubCache(fileId); + FileCache subCache = getOrAddFileSubCache(fileKey); try { for (int i = 0; i < ranges.length; ++i) { LlapDataBuffer buffer = (LlapDataBuffer)buffers[i]; @@ -260,7 +260,7 @@ private boolean lockBuffer(LlapDataBuffer buffer, boolean doNotifyPolicy) { } if (DebugUtils.isTraceCachingEnabled()) { LlapIoImpl.LOG.info("Trying to cache when the chunk is already cached for " - + fileId + "@" + offset + " (base " + baseOffset + "); old " + oldVal + + fileKey + "@" + offset + " (base " + baseOffset + "); old " + oldVal + ", new " + buffer); } if (DebugUtils.isTraceLockingEnabled()) { @@ -301,10 +301,10 @@ private boolean lockBuffer(LlapDataBuffer buffer, boolean doNotifyPolicy) { * All this mess is necessary because we want to be able to remove sub-caches for fully * evicted files. It may actually be better to have non-nested map with object keys? */ - private FileCache getOrAddFileSubCache(long fileId) { + private FileCache getOrAddFileSubCache(Object fileKey) { FileCache newSubCache = null; while (true) { // Overwhelmingly executes once. - FileCache subCache = cache.get(fileId); + FileCache subCache = cache.get(fileKey); if (subCache != null) { if (subCache.incRef()) return subCache; // Main path - found it, incRef-ed it. if (newSubCache == null) { @@ -312,7 +312,7 @@ private FileCache getOrAddFileSubCache(long fileId) { newSubCache.incRef(); } // Found a stale value we cannot incRef; try to replace it with new value. - if (cache.replace(fileId, subCache, newSubCache)) return newSubCache; + if (cache.replace(fileKey, subCache, newSubCache)) return newSubCache; continue; // Someone else replaced/removed a stale value, try again. } // No value found. @@ -320,11 +320,11 @@ private FileCache getOrAddFileSubCache(long fileId) { newSubCache = new FileCache(); newSubCache.incRef(); } - FileCache oldSubCache = cache.putIfAbsent(fileId, newSubCache); + FileCache oldSubCache = cache.putIfAbsent(fileKey, newSubCache); if (oldSubCache == null) return newSubCache; // Main path 2 - created a new file cache. if (oldSubCache.incRef()) return oldSubCache; // Someone created one in parallel. // Someone created one in parallel and then it went stale. - if (cache.replace(fileId, oldSubCache, newSubCache)) return newSubCache; + if (cache.replace(fileKey, oldSubCache, newSubCache)) return newSubCache; // Someone else replaced/removed a parallel-added stale value, try again. Max confusion. } } @@ -463,7 +463,7 @@ private void doOneCleanupRound() throws InterruptedException { // Iterate thru all the filecaches. This is best-effort. // If these super-long-lived iterators affect the map in some bad way, // we'd need to sleep once per round instead. - Iterator> iter = cache.entrySet().iterator(); + Iterator> iter = cache.entrySet().iterator(); boolean isPastEndTime = false; while (iter.hasNext()) { FileCache fc = iter.next().getValue(); @@ -516,7 +516,7 @@ public Allocator getAllocator() { @Override public String debugDumpForOom() { StringBuilder sb = new StringBuilder("File cache state "); - for (Map.Entry e : cache.entrySet()) { + for (Map.Entry e : cache.entrySet()) { if (!e.getValue().incRef()) continue; try { sb.append("\n file " + e.getKey()); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/NoopCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/NoopCache.java deleted file mode 100644 index d0461e831727b252464a3f4ed371e798fabc694d..0000000000000000000000000000000000000000 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/NoopCache.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.llap.cache; - -import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch.ColumnStreamData; - -public class NoopCache implements Cache { - @Override - public ColumnStreamData[] cacheOrGet(CacheKey key, ColumnStreamData[] value) { - return value; - } - - @Override - public ColumnStreamData[] get(CacheKey key) { - return null; // TODO: ensure real implementation increases refcount - } -} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java index d2c19079c40ebf798fbecf1b3b0169e313ca9f50..dbee82398969d513ee9137f2b5e4bca1705ba316 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.llap.cache.BuddyAllocator; import org.apache.hadoop.hive.llap.cache.BufferUsageManager; -import org.apache.hadoop.hive.llap.cache.Cache; import org.apache.hadoop.hive.llap.cache.EvictionAwareAllocator; import org.apache.hadoop.hive.llap.cache.EvictionDispatcher; import org.apache.hadoop.hive.llap.cache.LowLevelCacheImpl; @@ -50,7 +49,6 @@ import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics; import org.apache.hadoop.hive.llap.metrics.MetricsUtils; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; -import org.apache.hadoop.hive.ql.io.orc.encoded.OrcCacheKey; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.metrics2.util.MBeans; @@ -91,8 +89,6 @@ private LlapIoImpl(Configuration conf) throws IOException { LOG.info("Started llap daemon metrics with displayName: " + displayName + " sessionId: " + sessionId); - Cache cache = null; // High-level cache is not implemented or supported. - OrcMetadataCache metadataCache = null; LowLevelCacheImpl orcCache = null; BufferUsageManager bufferManager = null; @@ -131,7 +127,7 @@ private LlapIoImpl(Configuration conf) throws IOException { // TODO: this should depends on input format and be in a map, or something. this.cvp = new OrcColumnVectorProducer( - metadataCache, orcCache, bufferManager, cache, conf, cacheMetrics, queueMetrics); + metadataCache, orcCache, bufferManager, conf, cacheMetrics, queueMetrics); if (LOGL.isInfoEnabled()) { LOG.info("LLAP IO initialized"); } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java index b81e97d0c859035ccf02da28f0ac4b0a9fd443d5..137acb0f3b5e9b471dcad61f3d14fec3f87fcbf8 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java @@ -17,9 +17,6 @@ */ package org.apache.hadoop.hive.llap.io.decode; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; import java.util.concurrent.Callable; import org.apache.hadoop.hive.common.Pool; @@ -30,14 +27,9 @@ import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; import org.apache.hive.common.util.FixedSizedObjectPool; -/** - * - */ public abstract class EncodedDataConsumer> implements Consumer, ReadPipeline { private volatile boolean isStopped = false; - // TODO: use array, precreate array based on metadata first? Works for ORC. For now keep dumb. - private final HashMap pendingData = new HashMap<>(); private ConsumerFeedback upstreamFeedback; private final Consumer downstreamConsumer; private Callable readCallable; @@ -76,50 +68,15 @@ public void init(ConsumerFeedback upstreamFeedback, @Override public void consumeData(BatchType data) { - // TODO: data arrives in whole batches now, not in columns. We could greatly simplify this. - BatchType targetBatch = null; - boolean localIsStopped = false; - Integer targetBatchVersion = null; - synchronized (pendingData) { - localIsStopped = isStopped; - if (!localIsStopped) { - targetBatch = pendingData.get(data.getBatchKey()); - if (targetBatch == null) { - targetBatch = data; - pendingData.put(data.getBatchKey(), data); - } - // We have the map locked; the code the throws things away from map only bumps the version - // under the same map lock; code the throws things away here only bumps the version when - // the batch was taken out of the map. - targetBatchVersion = targetBatch.version; - } - queueMetrics.setQueueSize(pendingData.size()); - } - if (localIsStopped) { - returnSourceData(data); - return; - } - assert targetBatchVersion != null; - synchronized (targetBatch) { - if (targetBatch != data) { - throw new UnsupportedOperationException("Merging is not supported"); - } - synchronized (pendingData) { - targetBatch = isStopped ? null : pendingData.remove(data.getBatchKey()); - // Check if someone already threw this away and changed the version. - localIsStopped = (targetBatchVersion != targetBatch.version); - } - // We took the batch out of the map. No more contention with stop possible. - } - if (localIsStopped && (targetBatch != data)) { + if (isStopped) { returnSourceData(data); return; } long start = System.currentTimeMillis(); - decodeBatch(targetBatch, downstreamConsumer); + decodeBatch(data, downstreamConsumer); long end = System.currentTimeMillis(); queueMetrics.addProcessingTime(end - start); - returnSourceData(targetBatch); + returnSourceData(data); } /** @@ -127,7 +84,6 @@ public void consumeData(BatchType data) { * of the ECB in question; or, if ECB is still in pendingData, pendingData must be locked. */ private void returnSourceData(BatchType data) { - ++data.version; upstreamFeedback.returnData(data); } @@ -136,19 +92,12 @@ protected abstract void decodeBatch(BatchType batch, @Override public void setDone() { - synchronized (pendingData) { - if (!pendingData.isEmpty()) { - throw new AssertionError("Not all data has been sent downstream: " + pendingData.size()); - } - } downstreamConsumer.setDone(); } - @Override public void setError(Throwable t) { downstreamConsumer.setError(t); - dicardPendingData(false); } @Override @@ -156,28 +105,10 @@ public void returnData(ColumnVectorBatch data) { cvbPool.offer(data); } - private void dicardPendingData(boolean isStopped) { - List batches = new ArrayList( - pendingData.size()); - synchronized (pendingData) { - if (isStopped) { - this.isStopped = true; - } - for (BatchType ecb : pendingData.values()) { - ++ecb.version; - batches.add(ecb); - } - pendingData.clear(); - } - for (BatchType batch : batches) { - upstreamFeedback.returnData(batch); - } - } - @Override public void stop() { upstreamFeedback.stop(); - dicardPendingData(true); + this.isStopped = true; } @Override diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java index 18191da6e06fd61e8e12d4f32220c1c17de70528..37fc8d0d94cf68a235ca1e02da51fe8dadd15e24 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java @@ -23,7 +23,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.llap.cache.BufferUsageManager; -import org.apache.hadoop.hive.llap.cache.Cache; import org.apache.hadoop.hive.llap.cache.LowLevelCache; import org.apache.hadoop.hive.llap.cache.LowLevelCacheImpl; import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters; @@ -34,14 +33,12 @@ import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics; import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics; import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; -import org.apache.hadoop.hive.ql.io.orc.encoded.OrcCacheKey; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.mapred.FileSplit; public class OrcColumnVectorProducer implements ColumnVectorProducer { private final OrcMetadataCache metadataCache; - private final Cache cache; private final LowLevelCache lowLevelCache; private final BufferUsageManager bufferManager; private final Configuration conf; @@ -50,7 +47,7 @@ private LlapDaemonQueueMetrics queueMetrics; public OrcColumnVectorProducer(OrcMetadataCache metadataCache, - LowLevelCacheImpl lowLevelCache, BufferUsageManager bufferManager, Cache cache, + LowLevelCacheImpl lowLevelCache, BufferUsageManager bufferManager, Configuration conf, LlapDaemonCacheMetrics metrics, LlapDaemonQueueMetrics queueMetrics) { if (LlapIoImpl.LOGL.isInfoEnabled()) { LlapIoImpl.LOG.info("Initializing ORC column vector producer"); @@ -59,7 +56,6 @@ public OrcColumnVectorProducer(OrcMetadataCache metadataCache, this.metadataCache = metadataCache; this.lowLevelCache = lowLevelCache; this.bufferManager = bufferManager; - this.cache = cache; this.conf = conf; this._skipCorrupt = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_SKIP_CORRUPT_DATA); this.cacheMetrics = metrics; @@ -74,7 +70,7 @@ public ReadPipeline createReadPipeline( cacheMetrics.incrCacheReadRequests(); OrcEncodedDataConsumer edc = new OrcEncodedDataConsumer(consumer, columnIds.size(), _skipCorrupt, counters, queueMetrics); - OrcEncodedDataReader reader = new OrcEncodedDataReader(lowLevelCache, bufferManager, cache, + OrcEncodedDataReader reader = new OrcEncodedDataReader(lowLevelCache, bufferManager, metadataCache, conf, split, columnIds, sarg, columnNames, edc, counters); edc.init(reader, reader); return edc; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java index 28cae87146b07fadf8fcaaf558594fd89367935c..7ee263dbe2f9f5703842b96790a41c5090e2d59e 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java @@ -54,6 +54,7 @@ public OrcEncodedDataConsumer( Consumer consumer, int colCount, boolean skipCorrupt, QueryFragmentCounters counters, LlapDaemonQueueMetrics queueMetrics) { super(consumer, colCount, queueMetrics); + // TODO: get rid of this this.skipCorrupt = skipCorrupt; this.counters = counters; } @@ -62,7 +63,6 @@ public void setFileMetadata(OrcFileMetadata f) { assert fileMetadata == null; fileMetadata = f; stripes = new OrcStripeMetadata[f.getStripes().size()]; - // TODO: get rid of this codec = WriterImpl.createCodec(fileMetadata.getCompressionKind()); } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index bcee56b0a082d5543e0f8fbe1378b90f89ea4946..eb251a8a92da6f5524a60d39f275a0750102c939 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hive.llap.ConsumerFeedback; import org.apache.hadoop.hive.llap.DebugUtils; import org.apache.hadoop.hive.llap.cache.BufferUsageManager; -import org.apache.hadoop.hive.llap.cache.Cache; import org.apache.hadoop.hive.llap.cache.LowLevelCache; import org.apache.hadoop.hive.llap.cache.LowLevelCache.Priority; import org.apache.hadoop.hive.llap.counters.QueryFragmentCounters; @@ -71,7 +70,6 @@ import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedOrcFile; import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReader; import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey; -import org.apache.hadoop.hive.ql.io.orc.encoded.OrcCacheKey; import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.OrcEncodedColumnBatch; import org.apache.hadoop.hive.ql.io.orc.encoded.Reader.PoolFactory; import org.apache.hadoop.hive.ql.io.orc.RecordReaderUtils; @@ -90,7 +88,7 @@ * consumer. It also serves as ConsumerFeedback that receives processed EncodedColumnBatch-es. */ public class OrcEncodedDataReader extends CallableWithNdc - implements ConsumerFeedback, Consumer { + implements ConsumerFeedback { private static final Logger LOG = LoggerFactory.getLogger(OrcEncodedDataReader.class); public static final FixedSizedObjectPool CSD_POOL = new FixedSizedObjectPool<>(8192, new PoolObjectHelper() { @@ -135,7 +133,6 @@ public void resetBeforeOffer(OrcEncodedColumnBatch t) { private final LowLevelCache lowLevelCache; private final BufferUsageManager bufferManager; private final Configuration conf; - private final Cache cache; private final FileSplit split; private List columnIds; private final SearchArgument sarg; @@ -150,7 +147,7 @@ public void resetBeforeOffer(OrcEncodedColumnBatch t) { private Reader orcReader; private MetadataReader metadataReader; private EncodedReader stripeReader; - private Long fileId; + private Object fileKey; private FileSystem fs; /** * readState[stripeIx'][colIx'] => boolean array (could be a bitmask) of rg-s that need to be @@ -162,13 +159,12 @@ public void resetBeforeOffer(OrcEncodedColumnBatch t) { private volatile boolean isPaused = false; public OrcEncodedDataReader(LowLevelCache lowLevelCache, BufferUsageManager bufferManager, - Cache cache, OrcMetadataCache metadataCache, Configuration conf, - FileSplit split, List columnIds, SearchArgument sarg, String[] columnNames, - OrcEncodedDataConsumer consumer, QueryFragmentCounters counters) { + OrcMetadataCache metadataCache, Configuration conf, FileSplit split, List columnIds, + SearchArgument sarg, String[] columnNames, OrcEncodedDataConsumer consumer, + QueryFragmentCounters counters) { this.lowLevelCache = lowLevelCache; this.metadataCache = metadataCache; this.bufferManager = bufferManager; - this.cache = cache; this.conf = conf; this.split = split; this.columnIds = columnIds; @@ -230,10 +226,10 @@ protected Void performDataRead() throws IOException { // 1. Get file metadata from cache, or create the reader and read it. // Don't cache the filesystem object for now; Tez closes it and FS cache will fix all that fs = split.getPath().getFileSystem(conf); - fileId = determineFileId(fs, split, + fileKey = determineFileId(fs, split, HiveConf.getBoolVar(conf, ConfVars.LLAP_CACHE_ALLOW_SYNTHETIC_FILEID)); counters.setDesc(QueryFragmentCounters.Desc.FILE, split.getPath() - + (fileId == null ? "" : " (" + fileId + ")")); + + (fileKey == null ? "" : " (" + fileKey + ")")); try { fileMetadata = getOrReadFileMetadata(); @@ -307,27 +303,13 @@ protected Void performDataRead() throws IOException { // read for every stripe (null means read all of them - the usual path). In any case, // readState will be modified for column x rgs that were fetched from high-level cache. List[] stripeColsToRead = null; - if (cache != null) { - try { - stripeColsToRead = produceDataFromCache(stride); - } catch (Throwable t) { - // produceDataFromCache handles its own cleanup. - consumer.setError(t); - cleanupReaders(); - recordReaderTime(startTime); - return null; - } - } // 5. Create encoded data reader. - // In case if we have high-level cache, we will intercept the data and add it there; - // otherwise just pass the data directly to the consumer. - Consumer dataConsumer = (cache == null) ? this.consumer : this; try { ensureOrcReader(); // Reader creating updates HDFS counters, don't do it here. DataWrapperForOrc dw = new DataWrapperForOrc(); - stripeReader = orcReader.encodedReader(fileId, dw, dw, POOL_FACTORY); + stripeReader = orcReader.encodedReader(fileKey, dw, dw, POOL_FACTORY); stripeReader.setDebugTracing(DebugUtils.isTraceOrcEnabled()); } catch (Throwable t) { consumer.setError(t); @@ -338,9 +320,8 @@ protected Void performDataRead() throws IOException { // 6. Read data. // TODO: I/O threadpool could be here - one thread per stripe; for now, linear. - boolean hasFileId = this.fileId != null; - long fileId = hasFileId ? this.fileId : 0; - OrcBatchKey stripeKey = hasFileId ? new OrcBatchKey(fileId, -1, 0) : null; + boolean hasFileId = this.fileKey != null; + OrcBatchKey stripeKey = hasFileId ? new OrcBatchKey(fileKey, -1, 0) : null; for (int stripeIxMod = 0; stripeIxMod < readState.length; ++stripeIxMod) { if (processStop()) { cleanupReaders(); @@ -369,7 +350,7 @@ protected Void performDataRead() throws IOException { if (colRgs.length > 0 && colRgs[0] == SargApplier.READ_NO_RGS) continue; // 6.1. Determine the columns to read (usually the same as requested). - if (cache == null || cols == null || cols.size() == colRgs.length) { + if (cols == null || cols.size() == colRgs.length) { cols = columnIds; stripeIncludes = globalIncludes; } else { @@ -393,7 +374,7 @@ protected Void performDataRead() throws IOException { counters.incrCounter(LlapIOCounters.METADATA_CACHE_MISS); ensureMetadataReader(); long startTimeHdfs = counters.startTimeCounter(); - stripeMetadata = new OrcStripeMetadata(new OrcBatchKey(fileId, stripeIx, 0), + stripeMetadata = new OrcStripeMetadata(new OrcBatchKey(fileKey, stripeIx, 0), metadataReader, stripe, stripeIncludes, sargColumns); counters.incrTimeCounter(LlapIOCounters.HDFS_TIME_NS, startTimeHdfs); if (hasFileId && metadataCache != null) { @@ -439,7 +420,7 @@ protected Void performDataRead() throws IOException { // data it receives for one stripe. We could probably interrupt it, if it checked that. stripeReader.readEncodedColumns(stripeIx, stripe, stripeMetadata.getRowIndexes(), stripeMetadata.getEncodings(), stripeMetadata.getStreams(), stripeIncludes, - colRgs, dataConsumer); + colRgs, consumer); } catch (Throwable t) { consumer.setError(t); cleanupReaders(); @@ -450,7 +431,7 @@ protected Void performDataRead() throws IOException { // Done with all the things. recordReaderTime(startTime); - dataConsumer.setDone(); + consumer.setDone(); if (DebugUtils.isTraceMttEnabled()) { LlapIoImpl.LOG.info("done processing " + split); } @@ -525,12 +506,12 @@ private boolean processStop() { return true; } - private static Long determineFileId(FileSystem fs, FileSplit split, + private static Object determineFileId(FileSystem fs, FileSplit split, boolean allowSynthetic) throws IOException { if (split instanceof OrcSplit) { - Long fileId = ((OrcSplit)split).getFileId(); - if (fileId != null) { - return fileId; + Object fileKey = ((OrcSplit)split).getFileKey(); + if (fileKey != null) { + return fileKey; } } LOG.warn("Split for " + split.getPath() + " (" + split.getClass() + ") does not have file ID"); @@ -600,8 +581,8 @@ private void cleanupReaders() { private void ensureOrcReader() throws IOException { if (orcReader != null) return; Path path = split.getPath(); - if (fileId != null && HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_USE_FILEID_PATH)) { - path = HdfsUtils.getFileIdPath(fs, path, fileId); + if (fileKey instanceof Long && HiveConf.getBoolVar(conf, ConfVars.LLAP_IO_USE_FILEID_PATH)) { + path = HdfsUtils.getFileIdPath(fs, path, (long)fileKey); } if (DebugUtils.isTraceOrcEnabled()) { LOG.info("Creating reader for " + path + " (" + split.getPath() + ")"); @@ -617,8 +598,8 @@ private void ensureOrcReader() throws IOException { */ private OrcFileMetadata getOrReadFileMetadata() throws IOException { OrcFileMetadata metadata = null; - if (fileId != null && metadataCache != null) { - metadata = metadataCache.getFileMetadata(fileId); + if (fileKey != null && metadataCache != null) { + metadata = metadataCache.getFileMetadata(fileKey); if (metadata != null) { counters.incrCounter(LlapIOCounters.METADATA_CACHE_HIT); return metadata; @@ -628,8 +609,8 @@ private OrcFileMetadata getOrReadFileMetadata() throws IOException { } ensureOrcReader(); // We assume this call doesn't touch HDFS because everything is already read; don't add time. - metadata = new OrcFileMetadata(fileId != null ? fileId : 0, orcReader); - if (fileId == null || metadataCache == null) return metadata; + metadata = new OrcFileMetadata(fileKey, orcReader); + if (fileKey == null || metadataCache == null) return metadata; return metadataCache.putFileMetadata(metadata); } @@ -639,9 +620,8 @@ private OrcFileMetadata getOrReadFileMetadata() throws IOException { private ArrayList readStripesMetadata( boolean[] globalInc, boolean[] sargColumns) throws IOException { ArrayList result = new ArrayList(readState.length); - boolean hasFileId = this.fileId != null; - long fileId = hasFileId ? this.fileId : 0; - OrcBatchKey stripeKey = hasFileId ? new OrcBatchKey(fileId, 0, 0) : null; + boolean hasFileId = this.fileKey != null; + OrcBatchKey stripeKey = hasFileId ? new OrcBatchKey(fileKey, 0, 0) : null; for (int stripeIxMod = 0; stripeIxMod < readState.length; ++stripeIxMod) { OrcStripeMetadata value = null; int stripeIx = stripeIxMod + stripeIxFrom; @@ -655,7 +635,7 @@ private OrcFileMetadata getOrReadFileMetadata() throws IOException { StripeInformation si = fileMetadata.getStripes().get(stripeIx); if (value == null) { long startTime = counters.startTimeCounter(); - value = new OrcStripeMetadata(new OrcBatchKey(fileId, stripeIx, 0), + value = new OrcStripeMetadata(new OrcBatchKey(fileKey, stripeIx, 0), metadataReader, si, globalInc, sargColumns); counters.incrTimeCounter(LlapIOCounters.HDFS_TIME_NS, startTime); if (hasFileId && metadataCache != null) { @@ -771,7 +751,7 @@ private void adjustRgMetric(int rgCount, boolean[] rgsToRead, boolean isNone, } else if (!isNone) { count = rgCount; } - counters.setCounter(LlapIOCounters.SELECTED_ROWGROUPS, count); + counters.incrCounter(LlapIOCounters.SELECTED_ROWGROUPS, count); } @@ -836,105 +816,6 @@ public void determineStripesToRead() { readState = new boolean[stripeIxTo - stripeIxFrom][][]; } - // TODO: split by stripe? we do everything by stripe, and it might be faster - /** - * Takes the data from high-level cache for all stripes and returns to consumer. - * @return List of columns to read per stripe, if any columns were fully eliminated by cache. - */ - private List[] produceDataFromCache(int rowIndexStride) throws IOException { - OrcCacheKey key = new OrcCacheKey(fileId, -1, -1, -1); - // For each stripe, keep a list of columns that are not fully in cache (null => all of them). - @SuppressWarnings("unchecked") - List[] stripeColsNotInCache = new List[readState.length]; - for (int stripeIxMod = 0; stripeIxMod < readState.length; ++stripeIxMod) { - key.stripeIx = stripeIxFrom + stripeIxMod; - boolean[][] cols = readState[stripeIxMod]; - boolean[] isMissingAnyRgs = new boolean[cols.length]; - int totalRgCount = getRgCount(fileMetadata.getStripes().get(key.stripeIx), rowIndexStride); - for (int rgIx = 0; rgIx < totalRgCount; ++rgIx) { - OrcEncodedColumnBatch col = ECB_POOL.take(); - col.init(fileId, key.stripeIx, rgIx, cols.length); - boolean hasAnyCached = false; - try { - key.rgIx = rgIx; - for (int colIxMod = 0; colIxMod < cols.length; ++colIxMod) { - boolean[] readMask = cols[colIxMod]; - // Check if RG is eliminated by SARG - if ((readMask == SargApplier.READ_NO_RGS) || (readMask != SargApplier.READ_ALL_RGS - && (readMask.length <= rgIx || !readMask[rgIx]))) continue; - key.colIx = columnIds.get(colIxMod); - ColumnStreamData[] cached = cache.get(key); - if (cached == null) { - isMissingAnyRgs[colIxMod] = true; - continue; - } - assert cached.length == OrcEncodedColumnBatch.MAX_DATA_STREAMS; - col.setAllStreamsData(colIxMod, key.colIx, cached); - hasAnyCached = true; - if (readMask == SargApplier.READ_ALL_RGS) { - // We were going to read all RGs, but some were in cache, allocate the mask. - cols[colIxMod] = readMask = new boolean[totalRgCount]; - Arrays.fill(readMask, true); - } - readMask[rgIx] = false; // Got from cache, don't read from disk. - } - } catch (Throwable t) { - // TODO: Any cleanup needed to release data in col back to cache should be here. - throw (t instanceof IOException) ? (IOException)t : new IOException(t); - } - if (hasAnyCached) { - consumer.consumeData(col); - } - } - boolean makeStripeColList = false; // By default assume we'll fetch all original columns. - for (int colIxMod = 0; colIxMod < cols.length; ++colIxMod) { - if (isMissingAnyRgs[colIxMod]) { - if (makeStripeColList) { - stripeColsNotInCache[stripeIxMod].add(columnIds.get(colIxMod)); - } - } else if (!makeStripeColList) { - // Some columns were fully in cache. Make a per-stripe col list, add previous columns. - makeStripeColList = true; - stripeColsNotInCache[stripeIxMod] = new ArrayList(cols.length - 1); - for (int i = 0; i < colIxMod; ++i) { - stripeColsNotInCache[stripeIxMod].add(columnIds.get(i)); - } - } - } - } - return stripeColsNotInCache; - } - - @Override - public void setDone() { - consumer.setDone(); - } - - @Override - public void consumeData(OrcEncodedColumnBatch data) { - // Store object in cache; create new key object - cannot be reused. - assert cache != null; - throw new UnsupportedOperationException("not implemented"); - /*for (int i = 0; i < data.getColumnData().length; ++i) { - OrcCacheKey key = new OrcCacheKey(data.getBatchKey(), data.getColumnIxs()[i]); - ColumnStreamData[] toCache = data.getColumnData()[i]; - ColumnStreamData[] cached = cache.cacheOrGet(key, toCache); - if (toCache != cached) { - for (ColumnStreamData sb : toCache) { - if (sb.decRef() != 0) continue; - lowLevelCache.releaseBuffers(sb.getCacheBuffers()); - } - data.getColumnData()[i] = cached; - } - } - consumer.consumeData(data);*/ - } - - @Override - public void setError(Throwable t) { - consumer.setError(t); - } - private class DataWrapperForOrc implements DataReader, DataCache { private final DataReader orcDataReader; @@ -948,17 +829,17 @@ public DataWrapperForOrc() { } @Override - public DiskRangeList getFileData(long fileId, DiskRangeList range, + public DiskRangeList getFileData(Object fileKey, DiskRangeList range, long baseOffset, DiskRangeListFactory factory, BooleanRef gotAllData) { return (lowLevelCache == null) ? range : lowLevelCache.getFileData( - fileId, range, baseOffset, factory, counters, gotAllData); + fileKey, range, baseOffset, factory, counters, gotAllData); } @Override - public long[] putFileData(long fileId, DiskRange[] ranges, + public long[] putFileData(Object fileKey, DiskRange[] ranges, MemoryBuffer[] data, long baseOffset) { return (lowLevelCache == null) ? null : lowLevelCache.putFileData( - fileId, ranges, data, baseOffset, Priority.NORMAL, counters); + fileKey, ranges, data, baseOffset, Priority.NORMAL, counters); } @Override @@ -989,7 +870,7 @@ public DiskRangeList readFileData(DiskRangeList range, long baseOffset, DiskRangeList result = orcDataReader.readFileData(range, baseOffset, doForceDirect); counters.recordHdfsTime(startTime); if (DebugUtils.isTraceOrcEnabled() && LOG.isInfoEnabled()) { - LOG.info("Disk ranges after disk read (file " + fileId + ", base offset " + baseOffset + LOG.info("Disk ranges after disk read (file " + fileKey + ", base offset " + baseOffset + "): " + RecordReaderUtils.stringifyDiskRanges(result)); } return result; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java index 2e4e0c557caf9aa57debbfa51adc1b40d2017a1e..4e42a0f58e23d039bca310d9b2ea7cd24f07f3ae 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcFileMetadata.java @@ -18,23 +18,22 @@ package org.apache.hadoop.hive.llap.io.metadata; +import com.google.common.annotations.VisibleForTesting; import java.util.ArrayList; import java.util.HashMap; import java.util.List; - import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator; import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator.ObjectEstimator; import org.apache.hadoop.hive.llap.cache.EvictionDispatcher; import org.apache.hadoop.hive.llap.cache.LlapCacheableBuffer; -import org.apache.orc.CompressionKind; -import org.apache.orc.FileMetadata; +import org.apache.hadoop.hive.ql.io.SyntheticFileId; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.io.orc.Reader; import org.apache.hadoop.hive.ql.io.orc.ReaderImpl.StripeInformationImpl; -import org.apache.orc.StripeInformation; +import org.apache.orc.CompressionKind; +import org.apache.orc.FileMetadata; import org.apache.orc.OrcProto; - -import com.google.common.annotations.VisibleForTesting; +import org.apache.orc.StripeInformation; /** ORC file metadata. Currently contains some duplicate info due to how different parts * of ORC use different info. Ideally we would get rid of protobuf structs in code beyond reading, @@ -46,7 +45,7 @@ private final List stripeStats; private final List types; private final List fileStats; - private final long fileId; + private final Object fileKey; private final CompressionKind compressionKind; private final int rowIndexStride; private final int compressionBufferSize; @@ -61,16 +60,18 @@ private final static HashMap, ObjectEstimator> SIZE_ESTIMATORS; private final static ObjectEstimator SIZE_ESTIMATOR; static { - OrcFileMetadata ofm = createDummy(0); + OrcFileMetadata ofm = createDummy(new SyntheticFileId()); SIZE_ESTIMATORS = IncrementalObjectSizeEstimator.createEstimators(ofm); IncrementalObjectSizeEstimator.addEstimator( "com.google.protobuf.LiteralByteString", SIZE_ESTIMATORS); + // Add long for the regular file ID estimation. + IncrementalObjectSizeEstimator.createEstimators(Long.class, SIZE_ESTIMATORS); SIZE_ESTIMATOR = SIZE_ESTIMATORS.get(OrcFileMetadata.class); } @VisibleForTesting - public static OrcFileMetadata createDummy(int fileId) { - OrcFileMetadata ofm = new OrcFileMetadata(fileId); + public static OrcFileMetadata createDummy(Object fileKey) { + OrcFileMetadata ofm = new OrcFileMetadata(fileKey); ofm.stripes.add(new StripeInformationImpl( OrcProto.StripeInformation.getDefaultInstance())); ofm.fileStats.add(OrcProto.ColumnStatistics.getDefaultInstance()); @@ -87,8 +88,8 @@ public static OrcFileMetadata createDummy(int fileId) { } // Ctor for memory estimation and tests - private OrcFileMetadata(int fileId) { - this.fileId = fileId; + private OrcFileMetadata(Object fileKey) { + this.fileKey = fileKey; stripes = new ArrayList(); versionList = new ArrayList(); fileStats = new ArrayList<>(); @@ -101,8 +102,8 @@ private OrcFileMetadata(int fileId) { compressionKind = CompressionKind.NONE; } - public OrcFileMetadata(long fileId, Reader reader) { - this.fileId = fileId; + public OrcFileMetadata(Object fileKey, Reader reader) { + this.fileKey = fileKey; this.stripeStats = reader.getOrcProtoStripeStatistics(); this.compressionKind = reader.getCompressionKind(); this.compressionBufferSize = reader.getCompressionSize(); @@ -183,8 +184,8 @@ public int getFlattenedColumnCount() { } @Override - public long getFileId() { - return fileId; + public Object getFileKey() { + return fileKey; } @Override diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java index 43c8fb368f8e67a7944b42de3e81e9eaddd51c13..e970137619151459a3a135713c883e71a8a08386 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcMetadataCache.java @@ -27,8 +27,8 @@ import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey; public class OrcMetadataCache { - private final ConcurrentHashMap metadata = - new ConcurrentHashMap(); + private final ConcurrentHashMap metadata = + new ConcurrentHashMap(); private final ConcurrentHashMap stripeMetadata = new ConcurrentHashMap(); private final MemoryManager memoryManager; @@ -42,7 +42,7 @@ public OrcMetadataCache(MemoryManager memoryManager, LowLevelCachePolicy policy) public OrcFileMetadata putFileMetadata(OrcFileMetadata metaData) { long memUsage = metaData.getMemoryUsage(); memoryManager.reserveMemory(memUsage, false); - OrcFileMetadata val = metadata.putIfAbsent(metaData.getFileId(), metaData); + OrcFileMetadata val = metadata.putIfAbsent(metaData.getFileKey(), metaData); // See OrcFileMetadata; it is always unlocked, so we just "touch" it here to simulate use. if (val == null) { val = metaData; @@ -75,12 +75,12 @@ public OrcStripeMetadata getStripeMetadata(OrcBatchKey stripeKey) throws IOExcep return stripeMetadata.get(stripeKey); } - public OrcFileMetadata getFileMetadata(long fileId) throws IOException { - return metadata.get(fileId); + public OrcFileMetadata getFileMetadata(Object fileKey) throws IOException { + return metadata.get(fileKey); } public void notifyEvicted(OrcFileMetadata buffer) { - metadata.remove(buffer.getFileId()); + metadata.remove(buffer.getFileKey()); // See OrcFileMetadata - we don't clear the object, it will be GCed when released by users. } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java index 8479d225ddf6599790f9f57ff4401b9ac3dbac0b..82187bd4c91bf6b18d1aa301ba23ea9d97d47123 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java @@ -17,22 +17,21 @@ */ package org.apache.hadoop.hive.llap.io.metadata; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; - import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator; import org.apache.hadoop.hive.llap.IncrementalObjectSizeEstimator.ObjectEstimator; import org.apache.hadoop.hive.llap.cache.EvictionDispatcher; import org.apache.hadoop.hive.llap.cache.LlapCacheableBuffer; -import org.apache.orc.impl.MetadataReader; -import org.apache.orc.impl.OrcIndex; -import org.apache.orc.StripeInformation; +import org.apache.hadoop.hive.ql.io.SyntheticFileId; import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey; import org.apache.orc.OrcProto; - -import com.google.common.annotations.VisibleForTesting; +import org.apache.orc.StripeInformation; +import org.apache.orc.impl.MetadataReader; +import org.apache.orc.impl.OrcIndex; public class OrcStripeMetadata extends LlapCacheableBuffer { private final OrcBatchKey stripeKey; @@ -46,10 +45,12 @@ private final static HashMap, ObjectEstimator> SIZE_ESTIMATORS; private final static ObjectEstimator SIZE_ESTIMATOR; static { - OrcStripeMetadata osm = createDummy(0); + OrcStripeMetadata osm = createDummy(new SyntheticFileId()); SIZE_ESTIMATORS = IncrementalObjectSizeEstimator.createEstimators(osm); IncrementalObjectSizeEstimator.addEstimator( "com.google.protobuf.LiteralByteString", SIZE_ESTIMATORS); + // Add long for the regular file ID estimation. + IncrementalObjectSizeEstimator.createEstimators(Long.class, SIZE_ESTIMATORS); SIZE_ESTIMATOR = SIZE_ESTIMATORS.get(OrcStripeMetadata.class); } @@ -65,7 +66,7 @@ public OrcStripeMetadata(OrcBatchKey stripeKey, MetadataReader mr, StripeInforma estimatedMemUsage = SIZE_ESTIMATOR.estimate(this, SIZE_ESTIMATORS); } - private OrcStripeMetadata(long id) { + private OrcStripeMetadata(Object id) { stripeKey = new OrcBatchKey(id, 0, 0); encodings = new ArrayList<>(); streams = new ArrayList<>(); @@ -73,7 +74,7 @@ private OrcStripeMetadata(long id) { } @VisibleForTesting - public static OrcStripeMetadata createDummy(long id) { + public static OrcStripeMetadata createDummy(Object id) { OrcStripeMetadata dummy = new OrcStripeMetadata(id); dummy.encodings.add(OrcProto.ColumnEncoding.getDefaultInstance()); dummy.streams.add(OrcProto.Stream.getDefaultInstance()); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java index ec0be2bb18efdf5f4d72de2d3cb805d973dcc8b3..14ff18777396ee2d00d656a828ed026c9b211f06 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/FileFormatProxy.java @@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.metastore.Metastore.SplitInfos; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; /** @@ -33,11 +34,10 @@ /** * Applies SARG to file metadata, and produces some result for this file. * @param sarg SARG - * @param byteBuffer File metadata from metastore cache. + * @param fileMetadata File metadata from metastore cache. * @return The result to return to client for this file, or null if file is eliminated. - * @throws IOException */ - ByteBuffer applySargToMetadata(SearchArgument sarg, ByteBuffer byteBuffer) throws IOException; + SplitInfos applySargToMetadata(SearchArgument sarg, ByteBuffer fileMetadata) throws IOException; /** * @param fs The filesystem of the file. diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java index bd4e188f582149a819352a30d2bb3271f6ab0e39..832daec736e1cfef9607603e199ea1aeab54aea1 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/FileMetadataHandler.java @@ -37,7 +37,7 @@ * contains the actual implementation that depends on some stuff in QL (for ORC). */ public abstract class FileMetadataHandler { - static final Log LOG = LogFactory.getLog(FileMetadataHandler.class); + protected static final Log LOG = LogFactory.getLog(FileMetadataHandler.class); private Configuration conf; private PartitionExpressionProxy expressionProxy; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index f0bc560f0bc0d43aaeb67f318e38a0b40fde2c67..c9fadad498aa7b24aae0791f23ad93bb9b4a0783 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -1508,8 +1508,7 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St } } - // tblPath will be null when tbl is a view. We skip the following if block in that case. - checkTrashPurgeCombination(tblPath, dbname + "." + name, ifPurge); + checkTrashPurgeCombination(tblPath, dbname + "." + name, ifPurge, deleteData && !isExternal); // Drop the partitions and get a list of locations which need to be deleted partPaths = dropPartitionsAndGetLocations(ms, dbname, name, tblPath, tbl.getPartitionKeys(), deleteData && !isExternal); @@ -1546,15 +1545,20 @@ private boolean drop_table_core(final RawStore ms, final String dbname, final St * @param objectName db.table, or db.table.part * @param ifPurge if PURGE options is specified */ - private void checkTrashPurgeCombination(Path pathToData, String objectName, boolean ifPurge) - throws MetaException { - if (!(pathToData != null && !ifPurge)) {//pathToData may be NULL for a view + private void checkTrashPurgeCombination(Path pathToData, String objectName, boolean ifPurge, + boolean deleteData) throws MetaException { + // There is no need to check TrashPurgeCombination in following cases since Purge/Trash + // is not applicable: + // a) deleteData is false -- drop an external table + // b) pathToData is null -- a view + // c) ifPurge is true -- force delete without Trash + if (!deleteData || pathToData == null || ifPurge) { return; } boolean trashEnabled = false; try { - trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1); + trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1); } catch(NumberFormatException ex) { // nothing to do } @@ -2644,11 +2648,13 @@ private boolean drop_partition_common(RawStore ms, String db_name, String tbl_na boolean isArchived = false; Path archiveParentDir = null; boolean mustPurge = false; + boolean isExternalTbl = false; try { ms.openTransaction(); part = ms.getPartition(db_name, tbl_name, part_vals); tbl = get_table_core(db_name, tbl_name); + isExternalTbl = isExternal(tbl); firePreEvent(new PreDropPartitionEvent(tbl, part, deleteData, this)); mustPurge = isMustPurge(envContext, tbl); @@ -2661,7 +2667,8 @@ private boolean drop_partition_common(RawStore ms, String db_name, String tbl_na if (isArchived) { archiveParentDir = MetaStoreUtils.getOriginalLocation(part); verifyIsWritablePath(archiveParentDir); - checkTrashPurgeCombination(archiveParentDir, db_name + "." + tbl_name + "." + part_vals, mustPurge); + checkTrashPurgeCombination(archiveParentDir, db_name + "." + tbl_name + "." + part_vals, + mustPurge, deleteData && !isExternalTbl); } if (!ms.dropPartition(db_name, tbl_name, part_vals)) { throw new MetaException("Unable to drop partition"); @@ -2670,13 +2677,14 @@ private boolean drop_partition_common(RawStore ms, String db_name, String tbl_na if ((part.getSd() != null) && (part.getSd().getLocation() != null)) { partPath = new Path(part.getSd().getLocation()); verifyIsWritablePath(partPath); - checkTrashPurgeCombination(partPath, db_name + "." + tbl_name + "." + part_vals, mustPurge); + checkTrashPurgeCombination(partPath, db_name + "." + tbl_name + "." + part_vals, + mustPurge, deleteData && !isExternalTbl); } } finally { if (!success) { ms.rollbackTransaction(); } else if (deleteData && ((partPath != null) || (archiveParentDir != null))) { - if (tbl != null && !isExternal(tbl)) { + if (!isExternalTbl) { if (mustPurge) { LOG.info("dropPartition() will purge " + partPath + " directly, skipping trash."); } @@ -2761,10 +2769,12 @@ public DropPartitionsResult drop_partitions_req( Table tbl = null; List parts = null; boolean mustPurge = false; + boolean isExternalTbl = false; try { // We need Partition-s for firing events and for result; DN needs MPartition-s to drop. // Great... Maybe we could bypass fetching MPartitions by issuing direct SQL deletes. tbl = get_table_core(dbName, tblName); + isExternalTbl = isExternal(tbl); mustPurge = isMustPurge(envContext, tbl); int minCount = 0; RequestPartsSpec spec = request.getParts(); @@ -2829,13 +2839,15 @@ public DropPartitionsResult drop_partitions_req( if (MetaStoreUtils.isArchived(part)) { Path archiveParentDir = MetaStoreUtils.getOriginalLocation(part); verifyIsWritablePath(archiveParentDir); - checkTrashPurgeCombination(archiveParentDir, dbName + "." + tblName + "." + part.getValues(), mustPurge); + checkTrashPurgeCombination(archiveParentDir, dbName + "." + tblName + "." + + part.getValues(), mustPurge, deleteData && !isExternalTbl); archToDelete.add(archiveParentDir); } if ((part.getSd() != null) && (part.getSd().getLocation() != null)) { Path partPath = new Path(part.getSd().getLocation()); verifyIsWritablePath(partPath); - checkTrashPurgeCombination(partPath, dbName + "." + tblName + "." + part.getValues(), mustPurge); + checkTrashPurgeCombination(partPath, dbName + "." + tblName + "." + part.getValues(), + mustPurge, deleteData && !isExternalTbl); dirsToDelete.add(new PathAndPartValSize(partPath, part.getValues().size())); } } @@ -5935,17 +5947,16 @@ public GetFileMetadataByExprResult get_file_metadata_by_expr(GetFileMetadataByEx boolean[] eliminated = new boolean[fileIds.size()]; getMS().getFileMetadataByExpr(fileIds, type, req.getExpr(), metadatas, ppdResults, eliminated); - for (int i = 0; i < metadatas.length; ++i) { - long fileId = fileIds.get(i); - ByteBuffer metadata = metadatas[i]; - if (metadata == null) continue; - metadata = (eliminated[i] || !needMetadata) ? null - : handleReadOnlyBufferForThrift(metadata); + for (int i = 0; i < fileIds.size(); ++i) { + if (!eliminated[i] && ppdResults[i] == null) continue; // No metadata => no ppd. MetadataPpdResult mpr = new MetadataPpdResult(); - ByteBuffer bitset = eliminated[i] ? null : handleReadOnlyBufferForThrift(ppdResults[i]); - mpr.setMetadata(metadata); - mpr.setIncludeBitset(bitset); - result.putToMetadata(fileId, mpr); + ByteBuffer ppdResult = eliminated[i] ? null : handleReadOnlyBufferForThrift(ppdResults[i]); + mpr.setIncludeBitset(ppdResult); + if (needMetadata) { + ByteBuffer metadata = eliminated[i] ? null : handleReadOnlyBufferForThrift(metadatas[i]); + mpr.setMetadata(metadata); + } + result.putToMetadata(fileIds.get(i), mpr); } if (!result.isSetMetadata()) { result.setMetadata(EMPTY_MAP_FM2); // Set the required field. diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 9048d45423fee465ee03181d7ff8b5f4c1df9b3f..cdd12aba9fb4284bbb9989d7fcbe3c591ef17d98 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.hive.common.ObjectPair; -import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.classification.InterfaceAudience; import org.apache.hadoop.hive.common.classification.InterfaceAudience.Public; @@ -54,6 +53,8 @@ import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.GetAllFunctionsResponse; import org.apache.hadoop.hive.metastore.api.GetChangeVersionRequest; +import org.apache.hadoop.hive.metastore.api.GetFileMetadataByExprRequest; +import org.apache.hadoop.hive.metastore.api.GetFileMetadataByExprResult; import org.apache.hadoop.hive.metastore.api.GetFileMetadataRequest; import org.apache.hadoop.hive.metastore.api.GetFileMetadataResult; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; @@ -79,6 +80,7 @@ import org.apache.hadoop.hive.metastore.api.LockRequest; import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.MetadataPpdResult; import org.apache.hadoop.hive.metastore.api.NoSuchLockException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.NoSuchTxnException; @@ -2247,15 +2249,48 @@ public AggrStats getAggrColStatsFor(String dbName, String tblName, if (listIndex == fileIds.size()) return null; int endIndex = Math.min(listIndex + fileMetadataBatchSize, fileIds.size()); List subList = fileIds.subList(listIndex, endIndex); - GetFileMetadataRequest req = new GetFileMetadataRequest(); - req.setFileIds(subList); - GetFileMetadataResult resp = client.get_file_metadata(req); + GetFileMetadataResult resp = sendGetFileMetadataReq(subList); + // TODO: we could remember if it's unsupported and stop sending calls; although, it might + // be a bad idea for HS2+standalone metastore that could be updated with support. + // Maybe we should just remember this for some time. + if (!resp.isIsSupported()) return null; listIndex = endIndex; return resp.getMetadata(); } }; } + private GetFileMetadataResult sendGetFileMetadataReq(List fileIds) throws TException { + return client.get_file_metadata(new GetFileMetadataRequest(fileIds)); + } + + @Override + public Iterable> getFileMetadataBySarg( + final List fileIds, final ByteBuffer sarg, final boolean doGetFooters) + throws TException { + return new MetastoreMapIterable() { + private int listIndex = 0; + @Override + protected Map fetchNextBatch() throws TException { + if (listIndex == fileIds.size()) return null; + int endIndex = Math.min(listIndex + fileMetadataBatchSize, fileIds.size()); + List subList = fileIds.subList(listIndex, endIndex); + GetFileMetadataByExprResult resp = sendGetFileMetadataBySargReq( + sarg, subList, doGetFooters); + if (!resp.isIsSupported()) return null; + listIndex = endIndex; + return resp.getMetadata(); + } + }; + } + + private GetFileMetadataByExprResult sendGetFileMetadataBySargReq( + ByteBuffer sarg, List fileIds, boolean doGetFooters) throws TException { + GetFileMetadataByExprRequest req = new GetFileMetadataByExprRequest(fileIds, sarg); + req.setDoGetFooters(doGetFooters); // No need to get footers + return client.get_file_metadata_by_expr(req); + } + public static abstract class MetastoreMapIterable implements Iterable>, Iterator> { private Iterator> currentIter; diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 62677d127f8d306eeb4706e213ac3c64c638fd66..39cf9277cb16bd4711de8061f9b70ea3cd852f1e 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hive.metastore.api.LockRequest; import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.MetadataPpdResult; import org.apache.hadoop.hive.metastore.api.NoSuchLockException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.NoSuchTxnException; @@ -1534,6 +1535,9 @@ public AggrStats getAggrColStatsFor(String dbName, String tblName, */ Iterable> getFileMetadata(List fileIds) throws TException; + Iterable> getFileMetadataBySarg( + List fileIds, ByteBuffer sarg, boolean doGetFooters) throws TException; + /** * Cleares the file metadata cache for respective file IDs. */ diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/filemeta/OrcFileMetadataHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/filemeta/OrcFileMetadataHandler.java index 1b388aab7247648ccdc04f6b080e3c5e71641a90..3bca85d17bcf58124dbfb45d5d2c175aa74d96eb 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/filemeta/OrcFileMetadataHandler.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/filemeta/OrcFileMetadataHandler.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.hadoop.hive.metastore.FileMetadataHandler; +import org.apache.hadoop.hive.metastore.Metastore.SplitInfos; import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; @@ -44,11 +45,21 @@ public void getFileMetadataByExpr(List fileIds, byte[] expr, } getStore().getFileMetadata(fileIds, metadatas); for (int i = 0; i < metadatas.length; ++i) { + eliminated[i] = false; + results[i] = null; if (metadatas[i] == null) continue; - ByteBuffer result = getFileFormatProxy().applySargToMetadata(sarg, metadatas[i]); + ByteBuffer metadata = metadatas[i].duplicate(); // Duplicate to avoid modification. + SplitInfos result = null; + try { + result = getFileFormatProxy().applySargToMetadata(sarg, metadata); + } catch (IOException ex) { + LOG.error("Failed to apply SARG to metadata", ex); + metadatas[i] = null; + continue; + } eliminated[i] = (result == null); if (!eliminated[i]) { - results[i] = result; + results[i] = ByteBuffer.wrap(result.toByteArray()); } } } diff --git a/orc/src/java/org/apache/orc/FileMetadata.java b/orc/src/java/org/apache/orc/FileMetadata.java index d63bdccb20eda4a4b49ea83718340cf9e6d61422..807e696ad2e1165213190553e99d1e2d4409ef4a 100644 --- a/orc/src/java/org/apache/orc/FileMetadata.java +++ b/orc/src/java/org/apache/orc/FileMetadata.java @@ -44,7 +44,7 @@ int getFlattenedColumnCount(); - long getFileId(); + Object getFileKey(); List getVersionList(); diff --git a/orc/src/java/org/apache/orc/impl/InStream.java b/orc/src/java/org/apache/orc/impl/InStream.java index b1c6de59e7e96bb648584bb81563d53b8956b4c0..1893afeb38c3477a3e14a9920e4feb5b1a204c18 100644 --- a/orc/src/java/org/apache/orc/impl/InStream.java +++ b/orc/src/java/org/apache/orc/impl/InStream.java @@ -35,7 +35,7 @@ public abstract class InStream extends InputStream { private static final Logger LOG = LoggerFactory.getLogger(InStream.class); - private static final int PROTOBUF_MESSAGE_MAX_LIMIT = 1024 << 20; // 1GB + public static final int PROTOBUF_MESSAGE_MAX_LIMIT = 1024 << 20; // 1GB protected final String name; protected long length; diff --git a/orc/src/java/org/apache/orc/impl/OutStream.java b/orc/src/java/org/apache/orc/impl/OutStream.java index 68ef7d47dffc44fd23d0800fcacd79b1fe089cf6..81662cc31f9249c2f11cefc54ce9445e9065e1e6 100644 --- a/orc/src/java/org/apache/orc/impl/OutStream.java +++ b/orc/src/java/org/apache/orc/impl/OutStream.java @@ -243,8 +243,8 @@ public void flush() throws IOException { if (compressed != null && compressed.position() != 0) { compressed.flip(); receiver.output(compressed); - compressed = null; } + compressed = null; uncompressedBytes = 0; compressedBytes = 0; overflow = null; diff --git a/orc/src/test/org/apache/orc/impl/TestOutStream.java b/orc/src/test/org/apache/orc/impl/TestOutStream.java new file mode 100644 index 0000000000000000000000000000000000000000..e9614d51f24ffb27b3b9f86dd5877ab46a1e2487 --- /dev/null +++ b/orc/src/test/org/apache/orc/impl/TestOutStream.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.orc.impl; + +import org.apache.orc.CompressionCodec; +import org.junit.Test; +import org.mockito.Mockito; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.assertEquals; + +public class TestOutStream { + + @Test + public void testFlush() throws Exception { + OutStream.OutputReceiver receiver = + Mockito.mock(OutStream.OutputReceiver.class); + CompressionCodec codec = new ZlibCodec(); + OutStream stream = new OutStream("test", 128*1024, codec, receiver); + assertEquals(0L, stream.getBufferSize()); + stream.write(new byte[]{0, 1, 2}); + stream.flush(); + Mockito.verify(receiver).output(Mockito.any(ByteBuffer.class)); + assertEquals(0L, stream.getBufferSize()); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java index f6fd08104f6d5a8c93bca1f5d77468096e823144..81f6db0ed494fa8dcd78afed5c604cb080234017 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java @@ -32,7 +32,6 @@ **/ public class TaskRunner extends Thread { - protected Task tsk; protected TaskResult result; protected SessionState ss; @@ -103,7 +102,7 @@ public void runSequential() { if (tsk.getException() == null) { tsk.setException(t); } - t.printStackTrace(); + LOG.error("Error in executeTask", t); } result.setExitVal(exitVal, tsk.getException()); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java index 310356c526149088b2c1fc5dc4c1da3cbf166595..a42c2e99fe62aac96c21e4c5590c1891c1ea529c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java @@ -301,8 +301,7 @@ public int execute(DriverContext driverContext) { return exitVal; } catch (Exception e) { - e.printStackTrace(); - LOG.error("Exception: " + e.getMessage()); + LOG.error("Got exception", e); return (1); } finally { try { @@ -313,7 +312,7 @@ public int execute(DriverContext driverContext) { } } catch (Exception e) { - LOG.error("Exception: " + e.getMessage()); + LOG.error("Exception: ", e); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java index f5500a4c4ece56fcd756c9c2159169228e542a6e..c81b14c660015ddb247cea578dc8bddd53f2c838 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java @@ -65,6 +65,7 @@ import org.apache.hadoop.hive.ql.plan.MapredLocalWork; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.api.StageType; +import org.apache.hadoop.hive.ql.session.OperationLog; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; @@ -317,8 +318,10 @@ public int executeInChildVM(DriverContext driverContext) { CachingPrintStream errPrintStream = new CachingPrintStream(System.err); - StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out); - StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, errPrintStream); + StreamPrinter outPrinter = new StreamPrinter(executor.getInputStream(), null, System.out, + OperationLog.getCurrentOperationLog().getPrintStream()); + StreamPrinter errPrinter = new StreamPrinter(executor.getErrorStream(), null, errPrintStream, + OperationLog.getCurrentOperationLog().getPrintStream()); outPrinter.start(); errPrinter.start(); @@ -340,7 +343,7 @@ public int executeInChildVM(DriverContext driverContext) { return exitVal; } catch (Exception e) { - LOG.error("Exception: " + e, e); + LOG.error("Exception: ", e); return (1); } finally { if (secureDoAs != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java index 418a03e174358c6a125e84dda35a5c80e9199db7..67f9da8f8d7b7da7cadd486d340e23df58a2ab02 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java @@ -73,7 +73,7 @@ private static final int COLUMN_1_WIDTH = 16; private static final int SEPARATOR_WIDTH = InPlaceUpdates.MIN_TERMINAL_WIDTH; private static final String SEPARATOR = new String(new char[SEPARATOR_WIDTH]).replace("\0", "-"); - private static final String PREP_SUMMARY_HEADER = "DAG Preparation Summary"; + private static final String QUERY_EXEC_SUMMARY_HEADER = "Query Execution Summary"; private static final String TASK_SUMMARY_HEADER = "Task Execution Summary"; private static final String LLAP_IO_SUMMARY_HEADER = "LLAP IO Summary"; @@ -95,9 +95,11 @@ private static final String LLAP_SUMMARY_HEADER = String.format(LLAP_SUMMARY_HEADER_FORMAT, "VERTICES", "ROWGROUPS", "META_HIT", "META_MISS", "DATA_HIT", "DATA_MISS", "ALLOCATION", "USED", "TOTAL_IO"); - private static final String TOTAL_PREP_TIME = "TotalPrepTime"; - private static final String METHOD = "METHOD"; - private static final String DURATION = "DURATION(ms)"; + + // Methods summary + private static final String OPERATION_SUMMARY = "%-35s %9s"; + private static final String OPERATION = "OPERATION"; + private static final String DURATION = "DURATION"; // in-place progress update related variables private int lines; @@ -214,6 +216,7 @@ public int monitorExecution(final DAGClient dagClient, HiveConf conf, boolean running = false; boolean done = false; + boolean success = false; int failedCounter = 0; int rc = 0; DAGStatus.State lastState = null; @@ -231,12 +234,12 @@ public int monitorExecution(final DAGClient dagClient, HiveConf conf, console.printInfo("\n"); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_RUN_DAG); perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.TEZ_SUBMIT_TO_RUNNING); - + Map progressMap = null; while (true) { try { status = dagClient.getDAGStatus(opts, checkInterval); - Map progressMap = status.getVertexProgress(); + progressMap = status.getVertexProgress(); DAGStatus.State state = status.getState(); if (state != lastState || state == RUNNING) { @@ -277,35 +280,7 @@ public int monitorExecution(final DAGClient dagClient, HiveConf conf, } else { lastReport = printStatus(progressMap, lastReport, console); } - - /* Profile info is collected anyways, isProfileEnabled - * decides if it gets printed or not - */ - if (isProfileEnabled) { - - double duration = (System.currentTimeMillis() - startTime) / 1000.0; - console.printInfo("Status: DAG finished successfully in " - + String.format("%.2f seconds", duration)); - console.printInfo("\n"); - - console.printInfo(PREP_SUMMARY_HEADER); - printMethodsSummary(); - console.printInfo(SEPARATOR); - console.printInfo(""); - - console.printInfo(TASK_SUMMARY_HEADER); - printDagSummary(progressMap, console, dagClient, conf, dag); - console.printInfo(SEPARATOR); - console.printInfo(""); - - if (llapIoEnabled) { - console.printInfo(LLAP_IO_SUMMARY_HEADER); - printLlapIOSummary(progressMap, console, dagClient); - console.printInfo(SEPARATOR); - } - - console.printInfo("\n"); - } + success = true; running = false; done = true; break; @@ -376,6 +351,33 @@ public int monitorExecution(final DAGClient dagClient, HiveConf conf, } perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.TEZ_RUN_DAG); + + if (isProfileEnabled && success && progressMap != null) { + + double duration = (System.currentTimeMillis() - startTime) / 1000.0; + console.printInfo("Status: DAG finished successfully in " + + String.format("%.2f seconds", duration)); + console.printInfo("\n"); + + console.printInfo(QUERY_EXEC_SUMMARY_HEADER); + printQueryExecutionBreakDown(); + console.printInfo(SEPARATOR); + console.printInfo(""); + + console.printInfo(TASK_SUMMARY_HEADER); + printDagSummary(progressMap, console, dagClient, conf, dag); + console.printInfo(SEPARATOR); + console.printInfo(""); + + if (llapIoEnabled) { + console.printInfo(LLAP_IO_SUMMARY_HEADER); + printLlapIOSummary(progressMap, console, dagClient); + console.printInfo(SEPARATOR); + } + + console.printInfo("\n"); + } + return rc; } @@ -414,37 +416,44 @@ private static long getCounterValueByGroupName(TezCounters vertexCounters, return (tezCounter == null) ? 0 : tezCounter.getValue(); } - private void printMethodsSummary() { - long totalInPrepTime = 0; - - String[] perfLoggerReportMethods = { - (PerfLogger.PARSE), - (PerfLogger.ANALYZE), - (PerfLogger.TEZ_BUILD_DAG), - (PerfLogger.TEZ_SUBMIT_TO_RUNNING) - }; + private void printQueryExecutionBreakDown() { /* Build the method summary header */ - String methodBreakdownHeader = String.format("%-30s %-13s", METHOD, DURATION); + String execBreakdownHeader = String.format(OPERATION_SUMMARY, OPERATION, DURATION); console.printInfo(SEPARATOR); - reprintLineWithColorAsBold(methodBreakdownHeader, Ansi.Color.CYAN); + reprintLineWithColorAsBold(execBreakdownHeader, Ansi.Color.CYAN); console.printInfo(SEPARATOR); - for (String method : perfLoggerReportMethods) { - long duration = perfLogger.getDuration(method); - totalInPrepTime += duration; - console.printInfo(String.format("%-30s %11s", method, commaFormat.format(duration))); - } - - /* - * The counters list above don't capture the total time from TimeToSubmit.startTime till - * TezRunDag.startTime, so calculate the duration and print it. - */ - totalInPrepTime = perfLogger.getStartTime(PerfLogger.TEZ_RUN_DAG) - - perfLogger.getStartTime(PerfLogger.TIME_TO_SUBMIT); + // parse, analyze, optimize and compile + long compile = perfLogger.getEndTime(PerfLogger.COMPILE) - + perfLogger.getStartTime(PerfLogger.DRIVER_RUN); + console.printInfo(String.format(OPERATION_SUMMARY, "Compile Query", + secondsFormat.format(compile / 1000.0) + "s")); + + // prepare plan for submission (building DAG, adding resources, creating scratch dirs etc.) + long totalDAGPrep = perfLogger.getStartTime(PerfLogger.TEZ_SUBMIT_DAG) - + perfLogger.getEndTime(PerfLogger.COMPILE); + console.printInfo(String.format(OPERATION_SUMMARY, "Prepare Plan", + secondsFormat.format(totalDAGPrep / 1000.0) + "s")); + + // submit to accept dag (if session is closed, this will include re-opening of session time, + // localizing files for AM, submitting DAG) + long submitToAccept = perfLogger.getStartTime(PerfLogger.TEZ_RUN_DAG) - + perfLogger.getStartTime(PerfLogger.TEZ_SUBMIT_DAG); + console.printInfo(String.format(OPERATION_SUMMARY, "Submit Plan", + secondsFormat.format(submitToAccept / 1000.0) + "s")); + + // accept to start dag (schedule wait time, resource wait time etc.) + long acceptToStart = perfLogger.getDuration(PerfLogger.TEZ_SUBMIT_TO_RUNNING); + console.printInfo(String.format(OPERATION_SUMMARY, "Start", + secondsFormat.format(acceptToStart / 1000.0) + "s")); + + // time to actually run the dag (actual dag runtime) + long startToEnd = perfLogger.getEndTime(PerfLogger.TEZ_RUN_DAG) - + perfLogger.getEndTime(PerfLogger.TEZ_SUBMIT_TO_RUNNING); + console.printInfo(String.format(OPERATION_SUMMARY, "Finish", + secondsFormat.format(startToEnd / 1000.0) + "s")); - console.printInfo(String.format("%-30s %11s", TOTAL_PREP_TIME, commaFormat.format( - totalInPrepTime))); } private void printDagSummary(Map progressMap, LogHelper console, @@ -582,7 +591,7 @@ private String humanReadableByteCount(long bytes) { } private void printLlapIOSummary(Map progressMap, LogHelper console, - DAGClient dagClient) throws Exception { + DAGClient dagClient) { SortedSet keys = new TreeSet<>(progressMap.keySet()); Set statusOptions = new HashSet<>(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); @@ -593,8 +602,15 @@ private void printLlapIOSummary(Map progressMap, LogHelper con if (vertexName.startsWith("Reducer")) { continue; } - TezCounters vertexCounters = dagClient.getVertexStatus(vertexName, statusOptions) - .getVertexCounters(); + TezCounters vertexCounters = null; + try { + vertexCounters = dagClient.getVertexStatus(vertexName, statusOptions) + .getVertexCounters(); + } catch (IOException e) { + // best attempt, shouldn't really kill DAG for this + } catch (TezException e) { + // best attempt, shouldn't really kill DAG for this + } if (vertexCounters != null) { final long selectedRowgroups = getCounterValueByGroupName(vertexCounters, counterGroup, LlapIOCounters.SELECTED_ROWGROUPS.name()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index 2b50a2a464d1cb809971500ee204e7e1c5ba261b..94468769330585477dc86d0557c84ac1ab76667b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -481,7 +481,7 @@ public static Directory getAcidState(Path directory, try { childrenWithId = SHIMS.listLocatedHdfsStatus(fs, directory, hiddenFileFilter); } catch (Throwable t) { - LOG.error("Failed to get files with ID; using regular API", t); + LOG.error("Failed to get files with ID; using regular API: " + t.getMessage()); useFileIds = false; } } @@ -648,7 +648,7 @@ private static void findOriginals(FileSystem fs, FileStatus stat, try { childrenWithId = SHIMS.listLocatedHdfsStatus(fs, stat.getPath(), hiddenFileFilter); } catch (Throwable t) { - LOG.error("Failed to get files with ID; using regular API", t); + LOG.error("Failed to get files with ID; using regular API: " + t.getMessage()); useFileIds = false; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java index af64fc8d9a5ebc00d159a56735b89597a270e22f..b71ca09c558263b37fc7e95a4490d09f08a7f602 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HdfsUtils.java @@ -33,32 +33,37 @@ private static final HadoopShims SHIMS = ShimLoader.getHadoopShims(); private static final Logger LOG = LoggerFactory.getLogger(HdfsUtils.class); - public static Long getFileId( + public static Object getFileId( FileSystem fileSystem, Path path, boolean allowSynthetic) throws IOException { - String pathStr = path.toUri().getPath(); if (fileSystem instanceof DistributedFileSystem) { - return SHIMS.getFileId(fileSystem, pathStr); + return SHIMS.getFileId(fileSystem, path.toUri().getPath()); } if (!allowSynthetic) { LOG.warn("Cannot get unique file ID from " + fileSystem.getClass().getSimpleName() + "; returning null"); return null; } - // If we are not on DFS, we just hash the file name + size and hope for the best. - // TODO: we assume it only happens in tests. Fix? - int nameHash = pathStr.hashCode(); FileStatus fs = fileSystem.getFileStatus(path); + return new SyntheticFileId(path, fs.getLen(), fs.getModificationTime()); + } + + public static long createFileId(String pathStr, FileStatus fs, boolean doLog, String fsName) { + int nameHash = pathStr.hashCode(); long fileSize = fs.getLen(), modTime = fs.getModificationTime(); int fileSizeHash = (int)(fileSize ^ (fileSize >>> 32)), modTimeHash = (int)(modTime ^ (modTime >>> 32)), combinedHash = modTimeHash ^ fileSizeHash; long id = (((long)nameHash & 0xffffffffL) << 32) | ((long)combinedHash & 0xffffffffL); - LOG.warn("Cannot get unique file ID from " - + fileSystem.getClass().getSimpleName() + "; using " + id + " (" + pathStr - + "," + nameHash + "," + fileSize + ")"); + if (doLog) { + LOG.warn("Cannot get unique file ID from " + fsName + "; using " + id + + " (" + pathStr + "," + nameHash + "," + fileSize + ")"); + } return id; } + + + // TODO: this relies on HDFS not changing the format; we assume if we could get inode ID, this // is still going to work. Otherwise, file IDs can be turned off. Later, we should use // as public utility method in HDFS to obtain the inode-based path. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/SyntheticFileId.java b/ql/src/java/org/apache/hadoop/hive/ql/io/SyntheticFileId.java new file mode 100644 index 0000000000000000000000000000000000000000..905bbb983409e1a1b6a4341355e8c1cfb897d359 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/SyntheticFileId.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Writable; + +public final class SyntheticFileId implements Writable { + private long pathHash; + private long modTime; + private long length; + + /** Writable ctor. */ + public SyntheticFileId() { + } + + public SyntheticFileId(Path path, long len, long modificationTime) { + this.pathHash = hashCode(path.toUri().getPath()); + this.modTime = modificationTime; + this.length = len; + } + + public SyntheticFileId(FileStatus file) { + this(file.getPath(), file.getLen(), file.getModificationTime()); + } + + @Override + public String toString() { + return "[" + pathHash + ", " + modTime + ", " + length + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = prime + (int) (length ^ (length >>> 32)); + result = prime * result + (int) (modTime ^ (modTime >>> 32)); + return prime * result + (int) (pathHash ^ (pathHash >>> 32)); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!(obj instanceof SyntheticFileId)) return false; + SyntheticFileId other = (SyntheticFileId)obj; + return length == other.length && modTime == other.modTime && pathHash == other.pathHash; + } + + private long hashCode(String path) { + long h = 0; + for (int i = 0; i < path.length(); ++i) { + h = 1223 * h + path.charAt(i); + } + return h; + } + + /** Length allows for some backward compatibility wrt field addition. */ + private static final short THREE_LONGS = 24; + + @Override + public void write(DataOutput out) throws IOException { + out.writeShort(THREE_LONGS); + out.writeLong(pathHash); + out.writeLong(modTime); + out.writeLong(length); + } + + @Override + public void readFields(DataInput in) throws IOException { + short len = in.readShort(); + if (len < THREE_LONGS) throw new IOException("Need at least " + THREE_LONGS + " bytes"); + pathHash = in.readLong(); + modTime = in.readLong(); + length = in.readLong(); + int extraBytes = len - THREE_LONGS; + if (extraBytes > 0) { + in.skipBytes(extraBytes); + } + } +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ExternalCache.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ExternalCache.java new file mode 100644 index 0000000000000000000000000000000000000000..6556fbfccb7a8b6efde1427d5d2ab58e8e8fe579 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ExternalCache.java @@ -0,0 +1,338 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.commons.codec.binary.Hex; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.api.MetadataPpdResult; +import org.apache.hadoop.hive.ql.exec.SerializationUtilities; +import org.apache.hadoop.hive.ql.io.HdfsUtils; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.FileInfo; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.FooterCache; +import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; +import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId; +import org.apache.orc.FileMetaInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.io.Output; +import com.google.common.collect.Lists; + +/** Metastore-based footer cache storing serialized footers. Also has a local cache. */ +public class ExternalCache implements FooterCache { + private static final Logger LOG = LoggerFactory.getLogger(ExternalCache.class); + private static boolean isDebugEnabled = LOG.isDebugEnabled(); + + private final LocalCache localCache; + private final ExternalFooterCachesByConf externalCacheSrc; + private boolean isWarnLogged = false; + + // Configuration and things set from it. + private HiveConf conf; + private boolean isInTest; + private SearchArgument sarg; + private ByteBuffer sargIsOriginal, sargNotIsOriginal; + private boolean isPpdEnabled; + + public ExternalCache(LocalCache lc, ExternalFooterCachesByConf efcf) { + localCache = lc; + externalCacheSrc = efcf; + } + + @Override + public void put(Long fileId, FileStatus file, FileMetaInfo fileMetaInfo, Reader orcReader) + throws IOException { + localCache.put(fileId, file, fileMetaInfo, orcReader); + if (fileId != null) { + try { + externalCacheSrc.getCache(conf).putFileMetadata(Lists.newArrayList(fileId), + Lists.newArrayList(((ReaderImpl)orcReader).getSerializedFileFooter())); + } catch (HiveException e) { + throw new IOException(e); + } + } + } + + @Override + public boolean isBlocking() { + return true; + } + + @Override + public boolean hasPpd() { + return isPpdEnabled; + } + + public void configure(HiveConf queryConfig) { + this.conf = queryConfig; + this.sarg = ConvertAstToSearchArg.createFromConf(conf); + this.isPpdEnabled = HiveConf.getBoolVar(conf, ConfVars.HIVEOPTINDEXFILTER) + && HiveConf.getBoolVar(conf, ConfVars.HIVE_ORC_MS_FOOTER_CACHE_PPD); + this.isInTest = HiveConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST); + this.sargIsOriginal = this.sargNotIsOriginal = null; + } + + @Override + public void getAndValidate(List files, boolean isOriginal, + FileInfo[] result, ByteBuffer[] ppdResult) throws IOException, HiveException { + assert result.length == files.size(); + assert ppdResult == null || ppdResult.length == files.size(); + // First, check the local cache. + localCache.getAndValidate(files, isOriginal, result, ppdResult); + + // posMap is an unfortunate consequence of batching/iterating thru MS results. + HashMap posMap = new HashMap(); + // We won't do metastore-side PPD for the things we have locally. + List fileIds = determineFileIdsToQuery(files, result, posMap); + // Need to get a new one, see the comment wrt threadlocals. + ExternalFooterCachesByConf.Cache cache = externalCacheSrc.getCache(conf); + ByteBuffer serializedSarg = null; + if (isPpdEnabled) { + serializedSarg = getSerializedSargForMetastore(isOriginal); + } + if (serializedSarg != null) { + Iterator> iter = cache.getFileMetadataByExpr( + fileIds, serializedSarg, false); // don't fetch the footer, PPD happens in MS. + while (iter.hasNext()) { + Entry e = iter.next(); + int ix = getAndVerifyIndex(posMap, files, result, e.getKey()); + processPpdResult(e.getValue(), files.get(ix), ix, result, ppdResult); + } + } else { + // Only populate corrupt IDs for the things we couldn't deserialize if we are not using + // ppd. We assume that PPD makes sure the cached values are correct (or fails otherwise); + // also, we don't use the footers in PPD case. + List corruptIds = null; + Iterator> iter = cache.getFileMetadata(fileIds); + while (iter.hasNext()) { + Entry e = iter.next(); + int ix = getAndVerifyIndex(posMap, files, result, e.getKey()); + if (!processBbResult(e.getValue(), ix, files.get(ix), result)) { + if (corruptIds == null) { + corruptIds = new ArrayList<>(); + } + corruptIds.add(e.getKey()); + } + } + if (corruptIds != null) { + cache.clearFileMetadata(corruptIds); + } + } + } + + private int getAndVerifyIndex(HashMap posMap, + List files, FileInfo[] result, Long fileId) { + int ix = posMap.get(fileId); + assert result[ix] == null; + assert fileId != null && fileId.equals(files.get(ix).getFileId()); + return ix; + } + + private boolean processBbResult( + ByteBuffer bb, int ix, HdfsFileStatusWithId file, FileInfo[] result) throws IOException { + if (bb == null) return true; + result[ix] = createFileInfoFromMs(file, bb); + if (result[ix] == null) { + return false; + } + + localCache.put(file.getFileStatus().getPath(), result[ix]); + return true; + } + + private void processPpdResult(MetadataPpdResult mpr, HdfsFileStatusWithId file, + int ix, FileInfo[] result, ByteBuffer[] ppdResult) throws IOException { + if (mpr == null) return; // This file is unknown to metastore. + + ppdResult[ix] = mpr.isSetIncludeBitset() ? mpr.bufferForIncludeBitset() : NO_SPLIT_AFTER_PPD; + if (mpr.isSetMetadata()) { + result[ix] = createFileInfoFromMs(file, mpr.bufferForMetadata()); + if (result[ix] != null) { + localCache.put(file.getFileStatus().getPath(), result[ix]); + } + } + } + + private List determineFileIdsToQuery( + List files, FileInfo[] result, HashMap posMap) { + for (int i = 0; i < result.length; ++i) { + if (result[i] != null) continue; + HdfsFileStatusWithId file = files.get(i); + final FileStatus fs = file.getFileStatus(); + Long fileId = file.getFileId(); + if (fileId == null) { + if (!isInTest) { + if (!isWarnLogged || isDebugEnabled) { + LOG.warn("Not using metastore cache because fileId is missing: " + fs.getPath()); + isWarnLogged = true; + } + continue; + } + fileId = generateTestFileId(fs, files, i); + LOG.info("Generated file ID " + fileId + " at " + i); + } + posMap.put(fileId, i); + } + return Lists.newArrayList(posMap.keySet()); + } + + private Long generateTestFileId(final FileStatus fs, List files, int i) { + final Long fileId = HdfsUtils.createFileId(fs.getPath().toUri().getPath(), fs, false, null); + files.set(i, new HdfsFileStatusWithId() { + @Override + public FileStatus getFileStatus() { + return fs; + } + + @Override + public Long getFileId() { + return fileId; + } + }); + return fileId; + } + + private ByteBuffer getSerializedSargForMetastore(boolean isOriginal) { + if (sarg == null) return null; + ByteBuffer serializedSarg = isOriginal ? sargIsOriginal : sargNotIsOriginal; + if (serializedSarg != null) return serializedSarg; + SearchArgument sarg2 = sarg; + Kryo kryo = SerializationUtilities.borrowKryo(); + try { + if ((isOriginal ? sargNotIsOriginal : sargIsOriginal) == null) { + sarg2 = kryo.copy(sarg2); // In case we need it for the other case. + } + translateSargToTableColIndexes(sarg2, conf, OrcInputFormat.getRootColumn(isOriginal)); + ExternalCache.Baos baos = new Baos(); + Output output = new Output(baos); + kryo.writeObject(output, sarg2); + output.flush(); + serializedSarg = baos.get(); + if (isOriginal) { + sargIsOriginal = serializedSarg; + } else { + sargNotIsOriginal = serializedSarg; + } + } finally { + SerializationUtilities.releaseKryo(kryo); + } + return serializedSarg; + } + + /** + * Modifies the SARG, replacing column names with column indexes in target table schema. This + * basically does the same thing as all the shennannigans with included columns, except for the + * last step where ORC gets direct subtypes of root column and uses the ordered match to map + * table columns to file columns. The numbers put into predicate leaf should allow to go into + * said subtypes directly by index to get the proper index in the file. + * This won't work with schema evolution, although it's probably much easier to reason about + * if schema evolution was to be supported, because this is a clear boundary between table + * schema columns and all things ORC. None of the ORC stuff is used here and none of the + * table schema stuff is used after that - ORC doesn't need a bunch of extra crap to apply + * the SARG thus modified. + */ + public static void translateSargToTableColIndexes( + SearchArgument sarg, Configuration conf, int rootColumn) { + String nameStr = OrcInputFormat.getNeededColumnNamesString(conf), + idStr = OrcInputFormat.getSargColumnIDsString(conf); + String[] knownNames = nameStr.split(","); + String[] idStrs = (idStr == null) ? null : idStr.split(","); + assert idStrs == null || knownNames.length == idStrs.length; + HashMap nameIdMap = new HashMap<>(); + for (int i = 0; i < knownNames.length; ++i) { + Integer newId = (idStrs != null) ? Integer.parseInt(idStrs[i]) : i; + Integer oldId = nameIdMap.put(knownNames[i], newId); + if (oldId != null && oldId.intValue() != newId.intValue()) { + throw new RuntimeException("Multiple IDs for " + knownNames[i] + " in column strings: [" + + idStr + "], [" + nameStr + "]"); + } + } + List leaves = sarg.getLeaves(); + for (int i = 0; i < leaves.size(); ++i) { + PredicateLeaf pl = leaves.get(i); + Integer colId = nameIdMap.get(pl.getColumnName()); + String newColName = RecordReaderImpl.encodeTranslatedSargColumn(rootColumn, colId); + SearchArgumentFactory.setPredicateLeafColumn(pl, newColName); + } + if (LOG.isDebugEnabled()) { + LOG.debug("SARG translated into " + sarg); + } + } + + private static FileInfo createFileInfoFromMs( + HdfsFileStatusWithId file, ByteBuffer bb) throws IOException { + if (bb == null) return null; + FileStatus fs = file.getFileStatus(); + ReaderImpl.FooterInfo fi = null; + ByteBuffer copy = bb.duplicate(); + try { + fi = ReaderImpl.extractMetaInfoFromFooter(copy, fs.getPath()); + } catch (Exception ex) { + byte[] data = new byte[bb.remaining()]; + System.arraycopy(bb.array(), bb.arrayOffset() + bb.position(), data, 0, data.length); + String msg = "Failed to parse the footer stored in cache for file ID " + + file.getFileId() + " " + bb + " [ " + Hex.encodeHexString(data) + " ]"; + LOG.error(msg, ex); + return null; + } + return new FileInfo(fs.getModificationTime(), fs.getLen(), fi.getStripes(), fi.getMetadata(), + fi.getFooter().getTypesList(), fi.getFooter().getStatisticsList(), fi.getFileMetaInfo(), + fi.getFileMetaInfo().writerVersion, file.getFileId()); + } + + private static final class Baos extends ByteArrayOutputStream { + public ByteBuffer get() { + return ByteBuffer.wrap(buf, 0, count); + } + } + + + /** An abstraction for testing ExternalCache in OrcInputFormat. */ + public interface ExternalFooterCachesByConf { + public interface Cache { + Iterator> getFileMetadataByExpr(List fileIds, + ByteBuffer serializedSarg, boolean doGetFooters) throws HiveException; + void clearFileMetadata(List fileIds) throws HiveException; + Iterator> getFileMetadata(List fileIds) + throws HiveException; + void putFileMetadata( + ArrayList keys, ArrayList values) throws HiveException; + } + + public Cache getCache(HiveConf conf) throws IOException; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java new file mode 100644 index 0000000000000000000000000000000000000000..8151e52c63e6b299213ef7b962028699e853de8f --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/LocalCache.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.FileInfo; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.FooterCache; +import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId; +import org.apache.orc.FileMetaInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; + +/** Local footer cache using Guava. Stores convoluted Java objects. */ +class LocalCache implements FooterCache { + private static final Logger LOG = LoggerFactory.getLogger(LocalCache.class); + private static boolean isDebugEnabled = LOG.isDebugEnabled(); + + private final Cache cache; + + public LocalCache(int numThreads, int cacheStripeDetailsSize) { + cache = CacheBuilder.newBuilder() + .concurrencyLevel(numThreads) + .initialCapacity(cacheStripeDetailsSize) + .maximumSize(cacheStripeDetailsSize) + .softValues() + .build(); + } + + public void clear() { + cache.invalidateAll(); + cache.cleanUp(); + } + + public void getAndValidate(List files, boolean isOriginal, + FileInfo[] result, ByteBuffer[] ppdResult) throws IOException { + // TODO: should local cache also be by fileId? Preserve the original logic for now. + assert result.length == files.size(); + int i = -1; + for (HdfsFileStatusWithId fileWithId : files) { + ++i; + FileStatus file = fileWithId.getFileStatus(); + Path path = file.getPath(); + Long fileId = fileWithId.getFileId(); + FileInfo fileInfo = cache.getIfPresent(path); + if (isDebugEnabled) { + LOG.debug("Info " + (fileInfo == null ? "not " : "") + "cached for path: " + path); + } + if (fileInfo == null) continue; + if ((fileId != null && fileInfo.fileId != null && fileId == fileInfo.fileId) + || (fileInfo.modificationTime == file.getModificationTime() && + fileInfo.size == file.getLen())) { + result[i] = fileInfo; + continue; + } + // Invalidate + cache.invalidate(path); + if (isDebugEnabled) { + LOG.debug("Meta-Info for : " + path + " changed. CachedModificationTime: " + + fileInfo.modificationTime + ", CurrentModificationTime: " + + file.getModificationTime() + ", CachedLength: " + fileInfo.size + + ", CurrentLength: " + file.getLen()); + } + } + } + + public void put(Path path, FileInfo fileInfo) { + cache.put(path, fileInfo); + } + + @Override + public void put(Long fileId, FileStatus file, FileMetaInfo fileMetaInfo, Reader orcReader) + throws IOException { + cache.put(file.getPath(), new FileInfo(file.getModificationTime(), file.getLen(), + orcReader.getStripes(), orcReader.getStripeStatistics(), orcReader.getTypes(), + orcReader.getOrcProtoFileStatistics(), fileMetaInfo, orcReader.getWriterVersion(), + fileId)); + } + + @Override + public boolean isBlocking() { + return false; + } + + @Override + public boolean hasPpd() { + return false; + } +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetastoreExternalCachesByConf.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetastoreExternalCachesByConf.java new file mode 100644 index 0000000000000000000000000000000000000000..ad8f4efd3589d782084ff71c8bbcaa94cd15d964 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetastoreExternalCachesByConf.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.io.orc; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.MetadataPpdResult; +import org.apache.hadoop.hive.ql.io.orc.ExternalCache.ExternalFooterCachesByConf; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; + +/** + * An implementation of external cache and factory based on metastore. + */ +public class MetastoreExternalCachesByConf implements ExternalFooterCachesByConf { + public static class HBaseCache implements ExternalFooterCachesByConf.Cache { + private Hive hive; + + public HBaseCache(Hive hive) { + this.hive = hive; + } + + @Override + public Iterator> getFileMetadataByExpr( + List fileIds, ByteBuffer sarg, boolean doGetFooters) throws HiveException { + return hive.getFileMetadataByExpr(fileIds, sarg, doGetFooters).iterator(); + } + + @Override + public void clearFileMetadata(List fileIds) throws HiveException { + hive.clearFileMetadata(fileIds); + } + + @Override + public Iterator> getFileMetadata( + List fileIds) throws HiveException { + return hive.getFileMetadata(fileIds).iterator(); + } + + @Override + public void putFileMetadata( + ArrayList fileIds, ArrayList metadata) throws HiveException { + hive.putFileMetadata(fileIds, metadata); + } + } + + @Override + public ExternalFooterCachesByConf.Cache getCache(HiveConf conf) throws IOException { + // TODO: we wish we could cache the Hive object, but it's not thread safe, and each + // threadlocal we "cache" would need to be reinitialized for every query. This is + // a huge PITA. Hive object will be cached internally, but the compat check will be + // done every time inside get(). + try { + return new HBaseCache(Hive.getWithFastCheck(conf)); + } catch (HiveException e) { + throw new IOException(e); + } + } +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileFormatProxy.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileFormatProxy.java index ef76723950bad579808b8e3e74b77a4f0c334987..c9c7b5a0782c72c7bd68ab6526fe7ed7f2d4d2b5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileFormatProxy.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileFormatProxy.java @@ -29,16 +29,19 @@ import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.orc.OrcProto; import org.apache.orc.StripeInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** File format proxy for ORC. */ public class OrcFileFormatProxy implements FileFormatProxy { + private static final Logger LOG = LoggerFactory.getLogger(OrcFileFormatProxy.class); @Override - public ByteBuffer applySargToMetadata( - SearchArgument sarg, ByteBuffer byteBuffer) throws IOException { + public SplitInfos applySargToMetadata( + SearchArgument sarg, ByteBuffer fileMetadata) throws IOException { // TODO: ideally we should store shortened representation of only the necessary fields // in HBase; it will probably require custom SARG application code. - ReaderImpl.FooterInfo fi = ReaderImpl.extractMetaInfoFromFooter(byteBuffer, null); + ReaderImpl.FooterInfo fi = ReaderImpl.extractMetaInfoFromFooter(fileMetadata, null); OrcProto.Footer footer = fi.getFooter(); int stripeCount = footer.getStripesCount(); boolean[] result = OrcInputFormat.pickStripesViaTranslatedSarg( @@ -52,10 +55,13 @@ public ByteBuffer applySargToMetadata( if (result != null && !result[i]) continue; isEliminated = false; StripeInformation si = stripes.get(i); + if (LOG.isDebugEnabled()) { + LOG.debug("PPD is adding a split " + i + ": " + si.getOffset() + ", " + si.getLength()); + } sb.addInfos(SplitInfo.newBuilder().setIndex(i) .setOffset(si.getOffset()).setLength(si.getLength())); } - return isEliminated ? null : ByteBuffer.wrap(sb.build().toByteArray()); + return isEliminated ? null : sb.build(); } public ByteBuffer[] getAddedColumnsToCache() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 0ebcd2aeaf2cb1204a4798f9f4762487135b0b5a..8b611bb0098105b25edd07a3340969a2c48e4368 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -18,16 +18,17 @@ package org.apache.hadoop.hive.ql.io.orc; +import org.apache.orc.impl.InStream; + + import java.io.IOException; import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.NavigableMap; import java.util.TreeMap; import java.util.concurrent.Callable; @@ -40,7 +41,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.codec.binary.Hex; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.serde.serdeConstants; @@ -70,6 +70,8 @@ import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.Metastore; +import org.apache.hadoop.hive.metastore.Metastore.SplitInfos; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface; @@ -84,14 +86,14 @@ import org.apache.hadoop.hive.ql.io.RecordIdentifier; import org.apache.hadoop.hive.ql.io.SelfDescribingInputFormatInterface; import org.apache.hadoop.hive.ql.io.StatsProvidingRecordReader; +import org.apache.hadoop.hive.ql.io.orc.ExternalCache.ExternalFooterCachesByConf; +import org.apache.hadoop.hive.ql.io.SyntheticFileId; import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.TruthValue; -import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.VirtualColumn; -import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.SerDeStats; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; @@ -110,10 +112,9 @@ import org.apache.orc.OrcProto; import com.google.common.annotations.VisibleForTesting; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.protobuf.CodedInputStream; /** * A MapReduce/Hive input format for ORC files. *

@@ -273,7 +274,7 @@ public SerDeStats getStats() { * @param isOriginal is the file in the original format? * @return the column number for the root of row. */ - private static int getRootColumn(boolean isOriginal) { + static int getRootColumn(boolean isOriginal) { return isOriginal ? 0 : (OrcRecordUpdater.ROW + 1); } @@ -334,45 +335,6 @@ private static void includeColumnRecursive(List types, } } - /** - * Modifies the SARG, replacing column names with column indexes in target table schema. This - * basically does the same thing as all the shennannigans with included columns, except for the - * last step where ORC gets direct subtypes of root column and uses the ordered match to map - * table columns to file columns. The numbers put into predicate leaf should allow to go into - * said subtypes directly by index to get the proper index in the file. - * This won't work with schema evolution, although it's probably much easier to reason about - * if schema evolution was to be supported, because this is a clear boundary between table - * schema columns and all things ORC. None of the ORC stuff is used here and none of the - * table schema stuff is used after that - ORC doesn't need a bunch of extra crap to apply - * the SARG thus modified. - */ - public static void translateSargToTableColIndexes( - SearchArgument sarg, Configuration conf, int rootColumn) { - String nameStr = getNeededColumnNamesString(conf), idStr = getSargColumnIDsString(conf); - String[] knownNames = nameStr.split(","); - String[] idStrs = (idStr == null) ? null : idStr.split(","); - assert idStrs == null || knownNames.length == idStrs.length; - HashMap nameIdMap = new HashMap<>(); - for (int i = 0; i < knownNames.length; ++i) { - Integer newId = (idStrs != null) ? Integer.parseInt(idStrs[i]) : i; - Integer oldId = nameIdMap.put(knownNames[i], newId); - if (oldId != null && oldId.intValue() != newId.intValue()) { - throw new RuntimeException("Multiple IDs for " + knownNames[i] + " in column strings: [" - + idStr + "], [" + nameStr + "]"); - } - } - List leaves = sarg.getLeaves(); - for (int i = 0; i < leaves.size(); ++i) { - PredicateLeaf pl = leaves.get(i); - Integer colId = nameIdMap.get(pl.getColumnName()); - String newColName = RecordReaderImpl.encodeTranslatedSargColumn(rootColumn, colId); - SearchArgumentFactory.setPredicateLeafColumn(pl, newColName); - } - if (LOG.isDebugEnabled()) { - LOG.debug("SARG translated into " + sarg); - } - } - public static boolean[] genIncludedColumns( List types, List included, boolean isOriginal) { int rootColumn = getRootColumn(isOriginal); @@ -476,14 +438,15 @@ static boolean canCreateSargFromConf(Configuration conf) { return getSargColumnNames(columnNamesString.split(","), types, include, isOriginal); } - private static String getNeededColumnNamesString(Configuration conf) { + static String getNeededColumnNamesString(Configuration conf) { return conf.get(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR); } - private static String getSargColumnIDsString(Configuration conf) { + static String getSargColumnIDsString(Configuration conf) { return conf.getBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, true) ? null : conf.get(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR); } + @Override public boolean validateInput(FileSystem fs, HiveConf conf, List files @@ -541,7 +504,7 @@ public boolean validateInput(FileSystem fs, HiveConf conf, // This is not thread safe between different split generations (and wasn't anyway). private FooterCache footerCache; private static LocalCache localCache; - private static MetastoreCache metaCache; + private static ExternalCache metaCache; static ExecutorService threadPool = null; private final int numBuckets; private final int splitStrategyBatchMs; @@ -558,10 +521,15 @@ public boolean validateInput(FileSystem fs, HiveConf conf, private final SearchArgument sarg; Context(Configuration conf) { - this(conf, 1); + this(conf, 1, null); } Context(Configuration conf, final int minSplits) { + this(conf, minSplits, null); + } + + @VisibleForTesting + Context(Configuration conf, final int minSplits, ExternalFooterCachesByConf efc) { this.conf = conf; this.forceThreadpool = HiveConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST); this.sarg = ConvertAstToSearchArg.createFromConf(conf); @@ -602,20 +570,22 @@ public boolean validateInput(FileSystem fs, HiveConf conf, // HDFS, because only HDFS would return fileIds for us. If fileId is extended using // size/mod time/etc. for other FSes, we might need to check FSes explicitly because // using such an aggregate fileId cache is not bulletproof and should be disable-able. - boolean useMetastoreCache = HiveConf.getBoolVar( + boolean useExternalCache = HiveConf.getBoolVar( conf, HiveConf.ConfVars.HIVE_ORC_MS_FOOTER_CACHE_ENABLED); if (localCache == null) { localCache = new LocalCache(numThreads, cacheStripeDetailsSize); } - if (useMetastoreCache) { + if (useExternalCache) { if (metaCache == null) { - metaCache = new MetastoreCache(localCache); + metaCache = new ExternalCache(localCache, + efc == null ? new MetastoreExternalCachesByConf() : efc); } assert conf instanceof HiveConf; metaCache.configure((HiveConf)conf); } // Set footer cache for current split generation. See field comment - not thread safe. - footerCache = useMetastoreCache ? metaCache : localCache; + // TODO: we should be able to enable caches separately + footerCache = useExternalCache ? metaCache : localCache; } } String value = conf.get(ValidTxnList.VALID_TXNS_KEY, @@ -637,6 +607,12 @@ public static void resetThreadPool() { threadPool = null; } } + + @VisibleForTesting + public static void clearLocalCache() { + if (localCache == null) return; + localCache.clear(); + } } /** @@ -675,12 +651,11 @@ public AcidDirInfo(FileSystem fs, Path splitPath, Directory acidInfo, private final boolean isOriginal; private final List deltas; private final boolean hasBase; + private final ByteBuffer ppdResult; - SplitInfo(Context context, FileSystem fs, - HdfsFileStatusWithId fileWithId, FileInfo fileInfo, - boolean isOriginal, - List deltas, - boolean hasBase, Path dir, boolean[] covered) throws IOException { + SplitInfo(Context context, FileSystem fs, HdfsFileStatusWithId fileWithId, FileInfo fileInfo, + boolean isOriginal, List deltas, boolean hasBase, Path dir, + boolean[] covered, ByteBuffer ppdResult) throws IOException { super(dir, context.numBuckets, deltas, covered); this.context = context; this.fs = fs; @@ -689,6 +664,7 @@ public AcidDirInfo(FileSystem fs, Path splitPath, Directory acidInfo, this.isOriginal = isOriginal; this.deltas = deltas; this.hasBase = hasBase; + this.ppdResult = ppdResult; } @VisibleForTesting @@ -696,7 +672,7 @@ public SplitInfo(Context context, FileSystem fs, FileStatus fileStatus, FileInfo boolean isOriginal, ArrayList deltas, boolean hasBase, Path dir, boolean[] covered) throws IOException { this(context, fs, AcidUtils.createOriginalObj(null, fileStatus), - fileInfo, isOriginal, deltas, hasBase, dir, covered); + fileInfo, isOriginal, deltas, hasBase, dir, covered, null); } } @@ -718,19 +694,21 @@ public ETLDir(Path dir, FileSystem fs, int fileCount) { private final FileSystem fs; } - Context context; - List dirs; + final List dirs; List files; - boolean isOriginal; - List deltas; - boolean[] covered; - private List>> splitFuturesRef; + private final List deltas; + private final boolean[] covered; + final boolean isOriginal; + // References to external fields for async SplitInfo generation. + private List>> splitFuturesRef = null; + private List splitsRef = null; private final UserGroupInformation ugi; + private final boolean allowSyntheticFileIds; public ETLSplitStrategy(Context context, FileSystem fs, Path dir, List children, boolean isOriginal, List deltas, - boolean[] covered, UserGroupInformation ugi) { + boolean[] covered, UserGroupInformation ugi, boolean allowSyntheticFileIds) { assert !children.isEmpty(); this.context = context; this.dirs = Lists.newArrayList(new ETLDir(dir, fs, children.size())); @@ -739,16 +717,26 @@ public ETLSplitStrategy(Context context, FileSystem fs, Path dir, this.deltas = deltas; this.covered = covered; this.ugi = ugi; + this.allowSyntheticFileIds = allowSyntheticFileIds; } @Override public List getSplits() throws IOException { List result = new ArrayList<>(files.size()); // Force local cache if we have deltas. - FooterCache cache = context.cacheStripeDetails ? - (deltas == null ? context.footerCache : Context.localCache) : null; + FooterCache cache = context.cacheStripeDetails ? ((deltas == null || deltas.isEmpty()) + ? context.footerCache : Context.localCache) : null; if (cache != null) { - FileInfo[] infos = cache.getAndValidate(files); + FileInfo[] infos = new FileInfo[files.size()]; + ByteBuffer[] ppdResults = null; + if (cache.hasPpd()) { + ppdResults = new ByteBuffer[files.size()]; + } + try { + cache.getAndValidate(files, isOriginal, infos, ppdResults); + } catch (HiveException e) { + throw new IOException(e); + } int dirIx = -1, fileInDirIx = -1, filesInDirCount = 0; ETLDir dir = null; for (int i = 0; i < files.size(); ++i) { @@ -757,15 +745,16 @@ public ETLSplitStrategy(Context context, FileSystem fs, Path dir, filesInDirCount = dir.fileCount; } FileInfo info = infos[i]; + ByteBuffer ppdResult = ppdResults == null ? null : ppdResults[i]; + HdfsFileStatusWithId file = files.get(i); if (info != null) { // Cached copy is valid context.cacheHitCounter.incrementAndGet(); } - HdfsFileStatusWithId file = files.get(i); - // ignore files of 0 length - if (file.getFileStatus().getLen() > 0) { - result.add(new SplitInfo( - context, dir.fs, file, info, isOriginal, deltas, true, dir.dir, covered)); + // Ignore files eliminated by PPD, or of 0 length. + if (ppdResult != FooterCache.NO_SPLIT_AFTER_PPD && file.getFileStatus().getLen() > 0) { + result.add(new SplitInfo(context, dir.fs, file, info, + isOriginal, deltas, true, dir.dir, covered, ppdResult)); } } } else { @@ -778,8 +767,8 @@ public ETLSplitStrategy(Context context, FileSystem fs, Path dir, } // ignore files of 0 length if (file.getFileStatus().getLen() > 0) { - result.add(new SplitInfo( - context, dir.fs, file, null, isOriginal, deltas, true, dir.dir, covered)); + result.add(new SplitInfo(context, dir.fs, file, null, + isOriginal, deltas, true, dir.dir, covered, null)); } } } @@ -823,14 +812,15 @@ public CombineResult combineWith(FileSystem fs, Path dir, return CombineResult.YES; } - public Future generateSplitWork( - Context context, List>> splitFutures) throws IOException { + public Future generateSplitWork(Context context, + List>> splitFutures, List splits) throws IOException { if ((context.cacheStripeDetails && context.footerCache.isBlocking()) || context.forceThreadpool) { this.splitFuturesRef = splitFutures; + this.splitsRef = splits; return Context.threadPool.submit(this); } else { - runGetSplitsSync(splitFutures, null); + runGetSplitsSync(splitFutures, splits, null); return null; } } @@ -838,14 +828,14 @@ public CombineResult combineWith(FileSystem fs, Path dir, @Override public Void call() throws IOException { if (ugi == null) { - runGetSplitsSync(splitFuturesRef, null); + runGetSplitsSync(splitFuturesRef, splitsRef, null); return null; } try { return ugi.doAs(new PrivilegedExceptionAction() { @Override public Void run() throws Exception { - runGetSplitsSync(splitFuturesRef, ugi); + runGetSplitsSync(splitFuturesRef, splitsRef, ugi); return null; } }); @@ -854,35 +844,60 @@ public Void run() throws Exception { } } + + + + private void runGetSplitsSync(List>> splitFutures, - UserGroupInformation ugi) throws IOException { - List splits = getSplits(); - List>> localList = new ArrayList<>(splits.size()); + List splits, UserGroupInformation ugi) throws IOException { UserGroupInformation tpUgi = ugi == null ? UserGroupInformation.getCurrentUser() : ugi; - for (SplitInfo splitInfo : splits) { - localList.add(Context.threadPool.submit(new SplitGenerator(splitInfo, tpUgi))); + List splitInfos = getSplits(); + List>> localListF = null; + List localListS = null; + for (SplitInfo splitInfo : splitInfos) { + SplitGenerator sg = new SplitGenerator(splitInfo, tpUgi, allowSyntheticFileIds); + if (!sg.isBlocking()) { + if (localListS == null) { + localListS = new ArrayList<>(splits.size()); + } + // Already called in doAs, so no need to doAs here. + localListS.addAll(sg.call()); + } else { + if (localListF == null) { + localListF = new ArrayList<>(splits.size()); + } + localListF.add(Context.threadPool.submit(sg)); + } } - synchronized (splitFutures) { - splitFutures.addAll(localList); + if (localListS != null) { + synchronized (splits) { + splits.addAll(localListS); + } } - } - } + if (localListF != null) { + synchronized (splitFutures) { + splitFutures.addAll(localListF); + } + } + } + } /** * BI strategy is used when the requirement is to spend less time in split generation * as opposed to query execution (split generation does not read or cache file footers). */ static final class BISplitStrategy extends ACIDSplitStrategy { - List fileStatuses; - boolean isOriginal; - List deltas; - FileSystem fs; - Context context; - Path dir; + private final List fileStatuses; + private final boolean isOriginal; + private final List deltas; + private final FileSystem fs; + private final Context context; + private final Path dir; + private final boolean allowSyntheticFileIds; public BISplitStrategy(Context context, FileSystem fs, Path dir, List fileStatuses, boolean isOriginal, - List deltas, boolean[] covered) { + List deltas, boolean[] covered, boolean allowSyntheticFileIds) { super(dir, context.numBuckets, deltas, covered); this.context = context; this.fileStatuses = fileStatuses; @@ -890,6 +905,7 @@ public BISplitStrategy(Context context, FileSystem fs, this.deltas = deltas; this.fs = fs; this.dir = dir; + this.allowSyntheticFileIds = allowSyntheticFileIds; } @Override @@ -900,7 +916,11 @@ public BISplitStrategy(Context context, FileSystem fs, if (fileStatus.getLen() != 0) { String[] hosts = SHIMS.getLocationsWithOffset(fs, fileStatus).firstEntry().getValue() .getHosts(); - OrcSplit orcSplit = new OrcSplit(fileStatus.getPath(), file.getFileId(), 0, + Object fileKey = file.getFileId(); + if (fileKey == null && allowSyntheticFileIds) { + fileKey = new SyntheticFileId(fileStatus); + } + OrcSplit orcSplit = new OrcSplit(fileStatus.getPath(), fileKey, 0, fileStatus.getLen(), hosts, null, isOriginal, true, deltas, -1); splits.add(orcSplit); } @@ -1008,7 +1028,7 @@ private AcidDirInfo callInternal() throws IOException { try { return SHIMS.listLocatedHdfsStatus(fs, base, AcidUtils.hiddenFileFilter); } catch (Throwable t) { - LOG.error("Failed to get files with ID; using regular API", t); + LOG.error("Failed to get files with ID; using regular API: " + t.getMessage()); } } @@ -1029,8 +1049,8 @@ private AcidDirInfo callInternal() throws IOException { static final class SplitGenerator implements Callable> { private final Context context; private final FileSystem fs; - private final HdfsFileStatusWithId fileWithId; private final FileStatus file; + private final Long fsFileId; private final long blockSize; private final TreeMap locations; private final FileInfo fileInfo; @@ -1045,32 +1065,41 @@ private AcidDirInfo callInternal() throws IOException { private OrcFile.WriterVersion writerVersion; private long projColsUncompressedSize; private final List deltaSplits; + private final ByteBuffer ppdResult; private final UserGroupInformation ugi; + private final boolean allowSyntheticFileIds; - public SplitGenerator(SplitInfo splitInfo, UserGroupInformation ugi) throws IOException { + public SplitGenerator(SplitInfo splitInfo, UserGroupInformation ugi, + boolean allowSyntheticFileIds) throws IOException { this.ugi = ugi; this.context = splitInfo.context; this.fs = splitInfo.fs; - this.fileWithId = splitInfo.fileWithId; - this.file = this.fileWithId.getFileStatus(); + this.file = splitInfo.fileWithId.getFileStatus(); + this.fsFileId = splitInfo.fileWithId.getFileId(); this.blockSize = this.file.getBlockSize(); this.fileInfo = splitInfo.fileInfo; // TODO: potential DFS call - this.locations = SHIMS.getLocationsWithOffset(fs, fileWithId.getFileStatus()); + this.locations = SHIMS.getLocationsWithOffset(fs, file); this.isOriginal = splitInfo.isOriginal; this.deltas = splitInfo.deltas; this.hasBase = splitInfo.hasBase; this.projColsUncompressedSize = -1; this.deltaSplits = splitInfo.getSplits(); + this.allowSyntheticFileIds = allowSyntheticFileIds; + this.ppdResult = splitInfo.ppdResult; + } + + public boolean isBlocking() { + return ppdResult != null; } Path getPath() { - return fileWithId.getFileStatus().getPath(); + return file.getPath(); } @Override public String toString() { - return "splitter(" + fileWithId.getFileStatus().getPath() + ")"; + return "splitter(" + file.getPath() + ")"; } /** @@ -1133,7 +1162,7 @@ OrcSplit createSplit(long offset, long length, maxSize = Math.max(maxSize, val.get()); } } else { - throw new IOException("File " + fileWithId.getFileStatus().getPath().toString() + + throw new IOException("File " + file.getPath().toString() + " should have had overlap on block starting at " + block.getOffset()); } } @@ -1161,10 +1190,28 @@ OrcSplit createSplit(long offset, long length, final double splitRatio = (double) length / (double) fileLen; final long scaledProjSize = projColsUncompressedSize > 0 ? (long) (splitRatio * projColsUncompressedSize) : fileLen; - return new OrcSplit(file.getPath(), fileWithId.getFileId(), offset, length, hosts, + Object fileKey = fsFileId; + if (fileKey == null && allowSyntheticFileIds) { + fileKey = new SyntheticFileId(file); + } + return new OrcSplit(file.getPath(), fileKey, offset, length, hosts, fileMetaInfo, isOriginal, hasBase, deltas, scaledProjSize); } + private static final class OffsetAndLength { // Java cruft; pair of long. + public OffsetAndLength() { + this.offset = -1; + this.length = 0; + } + + long offset, length; + + @Override + public String toString() { + return "[offset=" + offset + ", length=" + length + "]"; + } + } + /** * Divide the adjacent stripes in the file into input splits based on the * block size and the configured minimum and maximum sizes. @@ -1187,74 +1234,122 @@ OrcSplit createSplit(long offset, long length, } private List callInternal() throws IOException { - populateAndCacheStripeDetails(); - List splits = Lists.newArrayList(); - - // figure out which stripes we need to read - boolean[] includeStripe = null; + // Figure out which stripes we need to read. + if (ppdResult != null) { + assert deltaSplits.isEmpty(); + assert ppdResult.hasArray(); + + // TODO: when PB is upgraded to 2.6, newInstance(ByteBuffer) method should be used here. + CodedInputStream cis = CodedInputStream.newInstance( + ppdResult.array(), ppdResult.arrayOffset(), ppdResult.remaining()); + cis.setSizeLimit(InStream.PROTOBUF_MESSAGE_MAX_LIMIT); + return generateSplitsFromPpd(SplitInfos.parseFrom(cis)); + } else { + populateAndCacheStripeDetails(); + boolean[] includeStripe = null; + // We can't eliminate stripes if there are deltas because the + // deltas may change the rows making them match the predicate. + if ((deltas == null || deltas.isEmpty()) && context.sarg != null) { + String[] colNames = extractNeededColNames(types, context.conf, includedCols, isOriginal); + if (colNames == null) { + LOG.warn("Skipping split elimination for {} as column names is null", file.getPath()); + } else { + includeStripe = pickStripes(context.sarg, colNames, writerVersion, isOriginal, + stripeStats, stripes.size(), file.getPath()); + } + } + return generateSplitsFromStripes(includeStripe); + } + } - // we can't eliminate stripes if there are deltas because the - // deltas may change the rows making them match the predicate. - if ((deltas == null || deltas.isEmpty()) && context.sarg != null) { - String[] colNames = extractNeededColNames(types, context.conf, includedCols, isOriginal); - if (colNames == null) { - LOG.warn("Skipping split elimination for {} as column names is null", file.getPath()); - } else { - includeStripe = pickStripes(context.sarg, colNames, writerVersion, isOriginal, - stripeStats, stripes.size(), file.getPath()); + private List generateSplitsFromPpd(SplitInfos ppdResult) throws IOException { + OffsetAndLength current = new OffsetAndLength(); + List splits = new ArrayList<>(ppdResult.getInfosCount()); + int lastIdx = -1; + for (Metastore.SplitInfo si : ppdResult.getInfosList()) { + int index = si.getIndex(); + if (lastIdx >= 0 && lastIdx + 1 != index && current.offset != -1) { + // Create split for the previous unfinished stripe. + splits.add(createSplit(current.offset, current.length, null)); + current.offset = -1; + } + lastIdx = index; + String debugStr = null; + if (LOG.isDebugEnabled()) { + debugStr = current.toString(); + } + current = generateOrUpdateSplit(splits, current, si.getOffset(), si.getLength(), null); + if (LOG.isDebugEnabled()) { + LOG.debug("Updated split from {" + index + ": " + si.getOffset() + ", " + + si.getLength() + "} and "+ debugStr + " to " + current); } } + generateLastSplit(splits, current, null); + return splits; + } + private List generateSplitsFromStripes(boolean[] includeStripe) throws IOException { + List splits = new ArrayList<>(stripes.size()); // if we didn't have predicate pushdown, read everything if (includeStripe == null) { includeStripe = new boolean[stripes.size()]; Arrays.fill(includeStripe, true); } - long currentOffset = -1; - long currentLength = 0; + OffsetAndLength current = new OffsetAndLength(); int idx = -1; for (StripeInformation stripe : stripes) { idx++; if (!includeStripe[idx]) { // create split for the previous unfinished stripe - if (currentOffset != -1) { - splits.add(createSplit(currentOffset, currentLength, fileMetaInfo)); - currentOffset = -1; + if (current.offset != -1) { + splits.add(createSplit(current.offset, current.length, fileMetaInfo)); + current.offset = -1; } continue; } - // if we are working on a stripe, over the min stripe size, and - // crossed a block boundary, cut the input split here. - if (currentOffset != -1 && currentLength > context.minSize && - (currentOffset / blockSize != stripe.getOffset() / blockSize)) { - splits.add(createSplit(currentOffset, currentLength, fileMetaInfo)); - currentOffset = -1; - } - // if we aren't building a split, start a new one. - if (currentOffset == -1) { - currentOffset = stripe.getOffset(); - currentLength = stripe.getLength(); - } else { - currentLength = - (stripe.getOffset() + stripe.getLength()) - currentOffset; - } - if (currentLength >= context.maxSize) { - splits.add(createSplit(currentOffset, currentLength, fileMetaInfo)); - currentOffset = -1; - } - } - if (currentOffset != -1) { - splits.add(createSplit(currentOffset, currentLength, fileMetaInfo)); + current = generateOrUpdateSplit( + splits, current, stripe.getOffset(), stripe.getLength(), fileMetaInfo); } + generateLastSplit(splits, current, fileMetaInfo); - // add uncovered ACID delta splits + // Add uncovered ACID delta splits. splits.addAll(deltaSplits); return splits; } + private OffsetAndLength generateOrUpdateSplit( + List splits, OffsetAndLength current, long offset, + long length, FileMetaInfo fileMetaInfo) throws IOException { + // if we are working on a stripe, over the min stripe size, and + // crossed a block boundary, cut the input split here. + if (current.offset != -1 && current.length > context.minSize && + (current.offset / blockSize != offset / blockSize)) { + splits.add(createSplit(current.offset, current.length, fileMetaInfo)); + current.offset = -1; + } + // if we aren't building a split, start a new one. + if (current.offset == -1) { + current.offset = offset; + current.length = length; + } else { + current.length = (offset + length) - current.offset; + } + if (current.length >= context.maxSize) { + splits.add(createSplit(current.offset, current.length, fileMetaInfo)); + current.offset = -1; + } + return current; + } + + private void generateLastSplit(List splits, OffsetAndLength current, + FileMetaInfo fileMetaInfo) throws IOException { + if (current.offset == -1) return; + splits.add(createSplit(current.offset, current.length, fileMetaInfo)); + } + private void populateAndCacheStripeDetails() throws IOException { // Only create OrcReader if we are missing some information. List colStatsLocal; @@ -1273,8 +1368,8 @@ private void populateAndCacheStripeDetails() throws IOException { assert fileInfo.stripeStats != null && fileInfo.types != null && fileInfo.writerVersion != null; // We assume that if we needed to create a reader, we need to cache it to meta cache. - // TODO: This will also needlessly overwrite it in local cache for now. - context.footerCache.put(fileWithId.getFileId(), file, fileInfo.fileMetaInfo, orcReader); + // This will also needlessly overwrite it in local cache for now. + context.footerCache.put(fsFileId, file, fileInfo.fileMetaInfo, orcReader); } } else { Reader orcReader = createOrcReader(); @@ -1286,8 +1381,7 @@ private void populateAndCacheStripeDetails() throws IOException { fileMetaInfo = context.footerInSplits ? ((ReaderImpl) orcReader).getFileMetaInfo() : null; if (context.cacheStripeDetails) { - Long fileId = fileWithId.getFileId(); - context.footerCache.put(fileId, file, fileMetaInfo, orcReader); + context.footerCache.put(fsFileId, file, fileMetaInfo, orcReader); } } includedCols = genIncludedColumns(types, context.conf, isOriginal); @@ -1314,10 +1408,6 @@ private long computeProjectionSize(List types, } } - static List generateSplitsInfo(Configuration conf) - throws IOException { - return generateSplitsInfo(conf, -1); - } /** Class intended to update two values from methods... Java-related cruft. */ @VisibleForTesting @@ -1326,18 +1416,14 @@ private long computeProjectionSize(List types, long combineStartUs; } - static List generateSplitsInfo(Configuration conf, int numSplits) + static List generateSplitsInfo(Configuration conf, Context context) throws IOException { - // Use threads to resolve directories into splits. - if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_MS_FOOTER_CACHE_ENABLED)) { - // Create HiveConf once, since this is expensive. - conf = new HiveConf(conf, OrcInputFormat.class); - } - Context context = new Context(conf, numSplits); if (LOG.isInfoEnabled()) { LOG.info("ORC pushdown predicate: " + context.sarg); } boolean useFileIds = HiveConf.getBoolVar(conf, ConfVars.HIVE_ORC_INCLUDE_FILE_ID_IN_SPLITS); + boolean allowSyntheticFileIds = useFileIds && HiveConf.getBoolVar( + conf, ConfVars.HIVE_ORC_ALLOW_SYNTHETIC_FILE_ID_IN_SPLITS); List splits = Lists.newArrayList(); List> pathFutures = Lists.newArrayList(); List> strategyFutures = Lists.newArrayList(); @@ -1373,15 +1459,15 @@ private long computeProjectionSize(List types, if (adi == null) { // We were combining SS-es and the time has expired. assert combinedCtx.combined != null; - scheduleSplits(combinedCtx.combined, context, splitFutures, strategyFutures); + scheduleSplits(combinedCtx.combined, context, splitFutures, strategyFutures, splits); combinedCtx.combined = null; continue; } // We have received a new directory information, make a split strategy. --resultsLeft; - SplitStrategy splitStrategy = determineSplitStrategy(combinedCtx, context, - adi.fs, adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, ugi); + SplitStrategy splitStrategy = determineSplitStrategy(combinedCtx, context, adi.fs, + adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, ugi, allowSyntheticFileIds); if (splitStrategy == null) continue; // Combined. if (isDebugEnabled) { @@ -1391,7 +1477,8 @@ private long computeProjectionSize(List types, // Hack note - different split strategies return differently typed lists, yay Java. // This works purely by magic, because we know which strategy produces which type. if (splitStrategy instanceof ETLSplitStrategy) { - scheduleSplits((ETLSplitStrategy)splitStrategy, context, splitFutures, strategyFutures); + scheduleSplits((ETLSplitStrategy)splitStrategy, + context, splitFutures, strategyFutures, splits); } else { @SuppressWarnings("unchecked") List readySplits = (List)splitStrategy.getSplits(); @@ -1401,7 +1488,7 @@ private long computeProjectionSize(List types, // Run the last combined strategy, if any. if (combinedCtx != null && combinedCtx.combined != null) { - scheduleSplits(combinedCtx.combined, context, splitFutures, strategyFutures); + scheduleSplits(combinedCtx.combined, context, splitFutures, strategyFutures, splits); combinedCtx.combined = null; } @@ -1434,10 +1521,18 @@ private long computeProjectionSize(List types, return splits; } + @VisibleForTesting + // We could have this as a protected method w/no class, but half of Hive is static, so there. + public static class ContextFactory { + public Context create(Configuration conf, int numSplits) { + return new Context(conf, numSplits); + } + } + private static void scheduleSplits(ETLSplitStrategy splitStrategy, Context context, - List>> splitFutures, List> strategyFutures) - throws IOException { - Future ssFuture = splitStrategy.generateSplitWork(context, splitFutures); + List>> splitFutures, List> strategyFutures, + List splits) throws IOException { + Future ssFuture = splitStrategy.generateSplitWork(context, splitFutures, splits); if (ssFuture == null) return; strategyFutures.add(ssFuture); } @@ -1451,12 +1546,13 @@ private static void scheduleSplits(ETLSplitStrategy splitStrategy, Context conte private static SplitStrategy combineOrCreateETLStrategy(CombinedCtx combinedCtx, Context context, FileSystem fs, Path dir, List files, List deltas, boolean[] covered, boolean isOriginal, - UserGroupInformation ugi) { + UserGroupInformation ugi, boolean allowSyntheticFileIds) { if (!deltas.isEmpty() || combinedCtx == null) { - return new ETLSplitStrategy(context, fs, dir, files, isOriginal, deltas, covered, ugi); + return new ETLSplitStrategy( + context, fs, dir, files, isOriginal, deltas, covered, ugi, allowSyntheticFileIds); } else if (combinedCtx.combined == null) { combinedCtx.combined = new ETLSplitStrategy( - context, fs, dir, files, isOriginal, deltas, covered, ugi); + context, fs, dir, files, isOriginal, deltas, covered, ugi, allowSyntheticFileIds); combinedCtx.combineStartUs = System.nanoTime(); return null; } else { @@ -1465,11 +1561,12 @@ private static void scheduleSplits(ETLSplitStrategy splitStrategy, Context conte switch (r) { case YES: return null; case NO_AND_CONTINUE: - return new ETLSplitStrategy(context, fs, dir, files, isOriginal, deltas, covered, ugi); + return new ETLSplitStrategy( + context, fs, dir, files, isOriginal, deltas, covered, ugi, allowSyntheticFileIds); case NO_AND_SWAP: { ETLSplitStrategy oldBase = combinedCtx.combined; combinedCtx.combined = new ETLSplitStrategy( - context, fs, dir, files, isOriginal, deltas, covered, ugi); + context, fs, dir, files, isOriginal, deltas, covered, ugi, allowSyntheticFileIds); combinedCtx.combineStartUs = System.nanoTime(); return oldBase; } @@ -1484,7 +1581,13 @@ private static void scheduleSplits(ETLSplitStrategy splitStrategy, Context conte if (isDebugEnabled) { LOG.debug("getSplits started"); } - List result = generateSplitsInfo(job, numSplits); + Configuration conf = job; + if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_ORC_MS_FOOTER_CACHE_ENABLED)) { + // Create HiveConf once, since this is expensive. + conf = new HiveConf(conf, OrcInputFormat.class); + } + List result = generateSplitsInfo(conf, + new Context(conf, numSplits, createExternalCaches())); if (isDebugEnabled) { LOG.debug("getSplits finished"); } @@ -1497,10 +1600,10 @@ private static void scheduleSplits(ETLSplitStrategy splitStrategy, Context conte * Stores information relevant to split generation for an ORC File. * */ - private static class FileInfo { - private final long modificationTime; - private final long size; - private final Long fileId; + static class FileInfo { + final long modificationTime; + final long size; + final Long fileId; private final List stripeInfos; private FileMetaInfo fileMetaInfo; private final List stripeStats; @@ -1798,7 +1901,8 @@ private static boolean isStripeSatisfyPredicate( @VisibleForTesting static SplitStrategy determineSplitStrategy(CombinedCtx combinedCtx, Context context, FileSystem fs, Path dir, AcidUtils.Directory dirInfo, - List baseOrOriginalFiles, UserGroupInformation ugi) { + List baseOrOriginalFiles, UserGroupInformation ugi, + boolean allowSyntheticFileIds) { Path base = dirInfo.getBaseDirectory(); List original = dirInfo.getOriginalFiles(); List deltas = AcidUtils.serializeDeltas(dirInfo.getCurrentDirectories()); @@ -1826,20 +1930,20 @@ private static boolean isStripeSatisfyPredicate( switch(context.splitStrategyKind) { case BI: // BI strategy requested through config - return new BISplitStrategy( - context, fs, dir, baseOrOriginalFiles, isOriginal, deltas, covered); + return new BISplitStrategy(context, fs, dir, baseOrOriginalFiles, + isOriginal, deltas, covered, allowSyntheticFileIds); case ETL: // ETL strategy requested through config - return combineOrCreateETLStrategy(combinedCtx, context, fs, - dir, baseOrOriginalFiles, deltas, covered, isOriginal, ugi); + return combineOrCreateETLStrategy(combinedCtx, context, fs, dir, baseOrOriginalFiles, + deltas, covered, isOriginal, ugi, allowSyntheticFileIds); default: // HYBRID strategy if (avgFileSize > context.maxSize || totalFiles <= context.minSplits) { - return combineOrCreateETLStrategy(combinedCtx, context, fs, - dir, baseOrOriginalFiles, deltas, covered, isOriginal, ugi); + return combineOrCreateETLStrategy(combinedCtx, context, fs, dir, baseOrOriginalFiles, + deltas, covered, isOriginal, ugi, allowSyntheticFileIds); } else { - return new BISplitStrategy( - context, fs, dir, baseOrOriginalFiles, isOriginal, deltas, covered); + return new BISplitStrategy(context, fs, dir, baseOrOriginalFiles, + isOriginal, deltas, covered, allowSyntheticFileIds); } } } else { @@ -1877,196 +1981,16 @@ private static boolean isStripeSatisfyPredicate( * Represents footer cache. */ public interface FooterCache { - FileInfo[] getAndValidate(List files) throws IOException; + static final ByteBuffer NO_SPLIT_AFTER_PPD = ByteBuffer.wrap(new byte[0]); + + void getAndValidate(List files, boolean isOriginal, + FileInfo[] result, ByteBuffer[] ppdResult) throws IOException, HiveException; + boolean hasPpd(); boolean isBlocking(); void put(Long fileId, FileStatus file, FileMetaInfo fileMetaInfo, Reader orcReader) throws IOException; } - /** Local footer cache using Guava. Stores convoluted Java objects. */ - private static class LocalCache implements FooterCache { - private final Cache cache; - - public LocalCache(int numThreads, int cacheStripeDetailsSize) { - cache = CacheBuilder.newBuilder() - .concurrencyLevel(numThreads) - .initialCapacity(cacheStripeDetailsSize) - .maximumSize(cacheStripeDetailsSize) - .softValues() - .build(); - } - - @Override - public FileInfo[] getAndValidate(List files) { - // TODO: should local cache also be by fileId? Preserve the original logic for now. - FileInfo[] result = new FileInfo[files.size()]; - int i = -1; - for (HdfsFileStatusWithId fileWithId : files) { - ++i; - FileStatus file = fileWithId.getFileStatus(); - Path path = file.getPath(); - Long fileId = fileWithId.getFileId(); - FileInfo fileInfo = cache.getIfPresent(path); - if (isDebugEnabled) { - LOG.debug("Info " + (fileInfo == null ? "not " : "") + "cached for path: " + path); - } - if (fileInfo == null) continue; - if ((fileId != null && fileInfo.fileId != null && fileId == fileInfo.fileId) - || (fileInfo.modificationTime == file.getModificationTime() && - fileInfo.size == file.getLen())) { - result[i] = fileInfo; - continue; - } - // Invalidate - cache.invalidate(path); - if (isDebugEnabled) { - LOG.debug("Meta-Info for : " + path + " changed. CachedModificationTime: " - + fileInfo.modificationTime + ", CurrentModificationTime: " - + file.getModificationTime() + ", CachedLength: " + fileInfo.size - + ", CurrentLength: " + file.getLen()); - } - } - return result; - } - - public void put(Path path, FileInfo fileInfo) { - cache.put(path, fileInfo); - } - - @Override - public void put(Long fileId, FileStatus file, FileMetaInfo fileMetaInfo, Reader orcReader) - throws IOException { - cache.put(file.getPath(), new FileInfo(file.getModificationTime(), file.getLen(), - orcReader.getStripes(), orcReader.getStripeStatistics(), orcReader.getTypes(), - orcReader.getOrcProtoFileStatistics(), fileMetaInfo, orcReader.getWriterVersion(), - fileId)); - } - - @Override - public boolean isBlocking() { - return false; - } - } - - /** Metastore-based footer cache storing serialized footers. Also has a local cache. */ - public static class MetastoreCache implements FooterCache { - private final LocalCache localCache; - private boolean isWarnLogged = false; - private HiveConf conf; - - public MetastoreCache(LocalCache lc) { - localCache = lc; - } - - @Override - public FileInfo[] getAndValidate(List files) throws IOException { - // First, check the local cache. - FileInfo[] result = localCache.getAndValidate(files); - assert result.length == files.size(); - // This is an unfortunate consequence of batching/iterating thru MS results. - // TODO: maybe have a direct map call for small lists if this becomes a perf issue. - HashMap posMap = new HashMap<>(files.size()); - for (int i = 0; i < result.length; ++i) { - if (result[i] != null) continue; - HdfsFileStatusWithId file = files.get(i); - Long fileId = file.getFileId(); - if (fileId == null) { - if (!isWarnLogged || isDebugEnabled) { - LOG.warn("Not using metastore cache because fileId is missing: " - + file.getFileStatus().getPath()); - isWarnLogged = true; - } - continue; - } - posMap.put(fileId, i); - } - Iterator> iter = null; - Hive hive; - try { - hive = getHive(); - iter = hive.getFileMetadata(Lists.newArrayList(posMap.keySet()), conf).iterator(); - } catch (HiveException ex) { - throw new IOException(ex); - } - List corruptIds = null; - while (iter.hasNext()) { - Entry e = iter.next(); - int ix = posMap.get(e.getKey()); - assert result[ix] == null; - HdfsFileStatusWithId file = files.get(ix); - assert file.getFileId() == e.getKey(); - result[ix] = createFileInfoFromMs(file, e.getValue()); - if (result[ix] == null) { - if (corruptIds == null) { - corruptIds = new ArrayList<>(); - } - corruptIds.add(file.getFileId()); - } else { - localCache.put(file.getFileStatus().getPath(), result[ix]); - } - } - if (corruptIds != null) { - try { - hive.clearFileMetadata(corruptIds); - } catch (HiveException ex) { - LOG.error("Failed to clear corrupt cache data", ex); - } - } - return result; - } - - private Hive getHive() throws HiveException { - // TODO: we wish we could cache the Hive object, but it's not thread safe, and each - // threadlocal we "cache" would need to be reinitialized for every query. This is - // a huge PITA. Hive object will be cached internally, but the compat check will be - // done every time inside get(). - return Hive.getWithFastCheck(conf); - } - - private static FileInfo createFileInfoFromMs( - HdfsFileStatusWithId file, ByteBuffer bb) throws IOException { - FileStatus fs = file.getFileStatus(); - ReaderImpl.FooterInfo fi = null; - ByteBuffer original = bb.duplicate(); - try { - fi = ReaderImpl.extractMetaInfoFromFooter(bb, fs.getPath()); - } catch (Exception ex) { - byte[] data = new byte[original.remaining()]; - System.arraycopy(original.array(), original.arrayOffset() + original.position(), - data, 0, data.length); - String msg = "Failed to parse the footer stored in cache for file ID " - + file.getFileId() + " " + original + " [ " + Hex.encodeHexString(data) + " ]"; - LOG.error(msg, ex); - return null; - } - return new FileInfo(fs.getModificationTime(), fs.getLen(), fi.getStripes(), fi.getMetadata(), - fi.getFooter().getTypesList(), fi.getFooter().getStatisticsList(), fi.getFileMetaInfo(), - fi.getFileMetaInfo().writerVersion, file.getFileId()); - } - - @Override - public void put(Long fileId, FileStatus file, FileMetaInfo fileMetaInfo, Reader orcReader) - throws IOException { - localCache.put(fileId, file, fileMetaInfo, orcReader); - if (fileId != null) { - try { - getHive().putFileMetadata(Lists.newArrayList(fileId), - Lists.newArrayList(((ReaderImpl)orcReader).getSerializedFileFooter())); - } catch (HiveException e) { - throw new IOException(e); - } - } - } - - public void configure(HiveConf queryConfig) { - this.conf = queryConfig; - } - - @Override - public boolean isBlocking() { - return true; - } - } /** * Convert a Hive type property string that contains separated type names into a list of * TypeDescription objects. @@ -2262,4 +2186,8 @@ public static TypeDescription getDesiredRowTypeDescr(Configuration conf, boolean return result; } + @VisibleForTesting + protected ExternalFooterCachesByConf createExternalCaches() { + return null; // The default ones are created in case of null; tests override this. + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java index 2782d7e99de55fedfea7a6da19806eeaaf96bd37..c4a722676a4ccf4bf00594e037fa34f789101833 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcNewInputFormat.java @@ -25,6 +25,8 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.Context; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; @@ -121,9 +123,8 @@ public boolean nextKeyValue() throws IOException, InterruptedException { if (LOG.isDebugEnabled()) { LOG.debug("getSplits started"); } - List splits = - OrcInputFormat.generateSplitsInfo(ShimLoader.getHadoopShims() - .getConfiguration(jobContext)); + Configuration conf = ShimLoader.getHadoopShims().getConfiguration(jobContext); + List splits = OrcInputFormat.generateSplitsInfo(conf, createContext(conf, -1)); List result = new ArrayList(splits.size()); for(OrcSplit split: splits) { result.add(new OrcNewSplit(split)); @@ -134,4 +135,13 @@ public boolean nextKeyValue() throws IOException, InterruptedException { return result; } + // Nearly C/P from OrcInputFormat; there are too many statics everywhere to sort this out. + private Context createContext(Configuration conf, int numSplits) { + // Use threads to resolve directories into splits. + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_MS_FOOTER_CACHE_ENABLED)) { + // Create HiveConf once, since this is expensive. + conf = new HiveConf(conf, OrcInputFormat.class); + } + return new Context(conf, numSplits, null); + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java index 4a27ee7b0018b0fc11e04d62a8fccace0629e4bd..407fd621032ea160479ceaf2a3dbfe1b9dc5dcc6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java @@ -26,13 +26,13 @@ import java.util.List; import org.apache.orc.FileMetaInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.ColumnarSplit; import org.apache.hadoop.hive.ql.io.AcidInputFormat; import org.apache.hadoop.hive.ql.io.LlapAwareSplit; +import org.apache.hadoop.hive.ql.io.SyntheticFileId; import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapred.FileSplit; @@ -48,11 +48,11 @@ private boolean isOriginal; private boolean hasBase; private final List deltas = new ArrayList<>(); - private OrcFile.WriterVersion writerVersion; private long projColsUncompressedSize; - private transient Long fileId; + private transient Object fileKey; - static final int HAS_FILEID_FLAG = 8; + static final int HAS_SYNTHETIC_FILEID_FLAG = 16; + static final int HAS_LONG_FILEID_FLAG = 8; static final int BASE_FLAG = 4; static final int ORIGINAL_FLAG = 2; static final int FOOTER_FLAG = 1; @@ -64,13 +64,13 @@ protected OrcSplit() { super(null, 0, 0, (String[]) null); } - public OrcSplit(Path path, Long fileId, long offset, long length, String[] hosts, + public OrcSplit(Path path, Object fileId, long offset, long length, String[] hosts, FileMetaInfo fileMetaInfo, boolean isOriginal, boolean hasBase, List deltas, long projectedDataSize) { super(path, offset, length, hosts); - // We could avoid serializing file ID and just replace the path with inode-based path. - // However, that breaks bunch of stuff because Hive later looks up things by split path. - this.fileId = fileId; + // For HDFS, we could avoid serializing file ID and just replace the path with inode-based + // path. However, that breaks bunch of stuff because Hive later looks up things by split path. + this.fileKey = fileId; this.fileMetaInfo = fileMetaInfo; hasFooter = this.fileMetaInfo != null; this.isOriginal = isOriginal; @@ -84,10 +84,12 @@ public void write(DataOutput out) throws IOException { //serialize path, offset, length using FileSplit super.write(out); + boolean isFileIdLong = fileKey instanceof Long, isFileIdWritable = fileKey instanceof Writable; int flags = (hasBase ? BASE_FLAG : 0) | (isOriginal ? ORIGINAL_FLAG : 0) | (hasFooter ? FOOTER_FLAG : 0) | - (fileId != null ? HAS_FILEID_FLAG : 0); + (isFileIdLong ? HAS_LONG_FILEID_FLAG : 0) | + (isFileIdWritable ? HAS_SYNTHETIC_FILEID_FLAG : 0); out.writeByte(flags); out.writeInt(deltas.size()); for(AcidInputFormat.DeltaMetaData delta: deltas) { @@ -109,8 +111,10 @@ public void write(DataOutput out) throws IOException { footerBuff.limit() - footerBuff.position()); WritableUtils.writeVInt(out, fileMetaInfo.writerVersion.getId()); } - if (fileId != null) { - out.writeLong(fileId.longValue()); + if (isFileIdLong) { + out.writeLong(((Long)fileKey).longValue()); + } else if (isFileIdWritable) { + ((Writable)fileKey).write(out); } } @@ -123,7 +127,11 @@ public void readFields(DataInput in) throws IOException { hasFooter = (FOOTER_FLAG & flags) != 0; isOriginal = (ORIGINAL_FLAG & flags) != 0; hasBase = (BASE_FLAG & flags) != 0; - boolean hasFileId = (HAS_FILEID_FLAG & flags) != 0; + boolean hasLongFileId = (HAS_LONG_FILEID_FLAG & flags) != 0, + hasWritableFileId = (HAS_SYNTHETIC_FILEID_FLAG & flags) != 0; + if (hasLongFileId && hasWritableFileId) { + throw new IOException("Invalid split - both file ID types present"); + } deltas.clear(); int numDeltas = in.readInt(); @@ -148,8 +156,12 @@ public void readFields(DataInput in) throws IOException { fileMetaInfo = new FileMetaInfo(compressionType, bufferSize, metadataSize, footerBuff, writerVersion); } - if (hasFileId) { - fileId = in.readLong(); + if (hasLongFileId) { + fileKey = in.readLong(); + } else if (hasWritableFileId) { + SyntheticFileId fileId = new SyntheticFileId(); + fileId.readFields(in); + this.fileKey = fileId; } } @@ -186,8 +198,8 @@ public long getProjectedColumnsUncompressedSize() { return projColsUncompressedSize; } - public Long getFileId() { - return fileId; + public Object getFileKey() { + return fileKey; } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java index 6cec80ee968a503c48cdd487fe11ab31cc2aa592..29b51ecb98f280c82c8a1a3f9d229985d4cd5bc3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java @@ -95,7 +95,7 @@ public DiskRangeList createCacheChunk(MemoryBuffer buffer, long offset, long end return tcc; } }; - private final Long fileId; + private final Object fileKey; private final DataReader dataReader; private boolean isDataReaderOpen = false; private final CompressionCodec codec; @@ -106,10 +106,10 @@ public DiskRangeList createCacheChunk(MemoryBuffer buffer, long offset, long end private ByteBufferAllocatorPool pool; private boolean isDebugTracingEnabled; - public EncodedReaderImpl(Long fileId, List types, CompressionCodec codec, + public EncodedReaderImpl(Object fileKey, List types, CompressionCodec codec, int bufferSize, long strideRate, DataCache cache, DataReader dataReader, PoolFactory pf) throws IOException { - this.fileId = fileId; + this.fileKey = fileKey; this.codec = codec; this.types = types; this.bufferSize = bufferSize; @@ -271,14 +271,13 @@ public void readEncodedColumns(int stripeIx, StripeInformation stripe, offset += length; } - boolean hasFileId = this.fileId != null; - long fileId = hasFileId ? this.fileId : 0; + boolean hasFileId = this.fileKey != null; if (listToRead.get() == null) { // No data to read for this stripe. Check if we have some included index-only columns. // TODO: there may be a bug here. Could there be partial RG filtering on index-only column? if (hasIndexOnlyCols && (includedRgs == null)) { OrcEncodedColumnBatch ecb = POOLS.ecbPool.take(); - ecb.init(fileId, stripeIx, OrcEncodedColumnBatch.ALL_RGS, colRgs.length); + ecb.init(fileKey, stripeIx, OrcEncodedColumnBatch.ALL_RGS, colRgs.length); consumer.consumeData(ecb); } else { LOG.warn("Nothing to read for stripe [" + stripe + "]"); @@ -289,14 +288,14 @@ public void readEncodedColumns(int stripeIx, StripeInformation stripe, // 2. Now, read all of the ranges from cache or disk. DiskRangeList.MutateHelper toRead = new DiskRangeList.MutateHelper(listToRead.get()); if (isDebugTracingEnabled && LOG.isInfoEnabled()) { - LOG.info("Resulting disk ranges to read (file " + fileId + "): " + LOG.info("Resulting disk ranges to read (file " + fileKey + "): " + RecordReaderUtils.stringifyDiskRanges(toRead.next)); } BooleanRef isAllInCache = new BooleanRef(); if (hasFileId) { - cache.getFileData(fileId, toRead.next, stripeOffset, CC_FACTORY, isAllInCache); + cache.getFileData(fileKey, toRead.next, stripeOffset, CC_FACTORY, isAllInCache); if (isDebugTracingEnabled && LOG.isInfoEnabled()) { - LOG.info("Disk ranges after cache (file " + fileId + ", base offset " + stripeOffset + LOG.info("Disk ranges after cache (file " + fileKey + ", base offset " + stripeOffset + "): " + RecordReaderUtils.stringifyDiskRanges(toRead.next)); } } @@ -324,7 +323,7 @@ public void readEncodedColumns(int stripeIx, StripeInformation stripe, } } if (isDebugTracingEnabled) { - LOG.info("Disk ranges after pre-read (file " + fileId + ", base offset " + LOG.info("Disk ranges after pre-read (file " + fileKey + ", base offset " + stripeOffset + "): " + RecordReaderUtils.stringifyDiskRanges(toRead.next)); } iter = toRead.next; // Reset the iter to start. @@ -337,13 +336,14 @@ public void readEncodedColumns(int stripeIx, StripeInformation stripe, boolean isLastRg = rgIx == rgCount - 1; // Create the batch we will use to return data for this RG. OrcEncodedColumnBatch ecb = POOLS.ecbPool.take(); - ecb.init(fileId, stripeIx, rgIx, colRgs.length); + ecb.init(fileKey, stripeIx, rgIx, colRgs.length); boolean isRGSelected = true; for (int colIxMod = 0; colIxMod < colRgs.length; ++colIxMod) { + // TODO: simplify this now that high-level cache has been removed. if (colRgs[colIxMod] != null && !colRgs[colIxMod][rgIx]) { // RG x col filtered. isRGSelected = false; - continue; // TODO: this would be invalid with HL cache, where RG x col can be excluded. + continue; } ColumnReadContext ctx = colCtxs[colIxMod]; OrcProto.RowIndexEntry index = ctx.rowIndex.getEntry(rgIx), @@ -663,8 +663,8 @@ public DiskRangeList readEncodedStream(long baseOffset, DiskRangeList start, lon } // 6. Finally, put uncompressed data to cache. - if (fileId != null) { - long[] collisionMask = cache.putFileData(fileId, cacheKeys, targetBuffers, baseOffset); + if (fileKey != null) { + long[] collisionMask = cache.putFileData(fileKey, cacheKeys, targetBuffers, baseOffset); processCacheCollisions(collisionMask, toDecompress, targetBuffers, csd.getCacheBuffers()); } @@ -914,8 +914,8 @@ private DiskRangeList preReadUncompressedStream(long baseOffset, } // 6. Finally, put uncompressed data to cache. - if (fileId != null) { - long[] collisionMask = cache.putFileData(fileId, cacheKeys, targetBuffers, baseOffset); + if (fileKey != null) { + long[] collisionMask = cache.putFileData(fileKey, cacheKeys, targetBuffers, baseOffset); processCacheCollisions(collisionMask, toCache, targetBuffers, null); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/OrcBatchKey.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/OrcBatchKey.java index da673a5a2c38e171b7d3c7abdd1e9626b458144f..9bb171e29af5243999cb4a3535ae59fb64bac539 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/OrcBatchKey.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/OrcBatchKey.java @@ -19,28 +19,33 @@ package org.apache.hadoop.hive.ql.io.orc.encoded; public class OrcBatchKey { - public long file; + public Object fileKey; public int stripeIx, rgIx; + // Make sure all the numbers are converted to long for size estimation. public OrcBatchKey(long file, int stripeIx, int rgIx) { set(file, stripeIx, rgIx); } - public void set(long file, int stripeIx, int rgIx) { - this.file = file; + public OrcBatchKey(Object file, int stripeIx, int rgIx) { + set(file, stripeIx, rgIx); + } + + public void set(Object file, int stripeIx, int rgIx) { + this.fileKey = file; this.stripeIx = stripeIx; this.rgIx = rgIx; } @Override public String toString() { - return "[" + file + ", stripe " + stripeIx + ", rgIx " + rgIx + "]"; + return "[" + fileKey + ", stripe " + stripeIx + ", rgIx " + rgIx + "]"; } @Override public int hashCode() { final int prime = 31; - int result = prime + (int)(file ^ (file >>> 32)); + int result = prime + fileKey.hashCode(); return (prime * result + rgIx) * prime + stripeIx; } @@ -50,11 +55,12 @@ public boolean equals(Object obj) { if (!(obj instanceof OrcBatchKey)) return false; OrcBatchKey other = (OrcBatchKey)obj; // Strings are interned and can thus be compared like this. - return stripeIx == other.stripeIx && rgIx == other.rgIx && file == other.file; + return stripeIx == other.stripeIx && rgIx == other.rgIx + && (fileKey == null ? (other.fileKey == null) : fileKey.equals(other.fileKey)); } @Override public OrcBatchKey clone() throws CloneNotSupportedException { - return new OrcBatchKey(file, stripeIx, rgIx); + return new OrcBatchKey(fileKey, stripeIx, rgIx); } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/OrcCacheKey.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/OrcCacheKey.java deleted file mode 100644 index 9a0158ec0650206fff237338c0b84eaec09bf205..0000000000000000000000000000000000000000 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/OrcCacheKey.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql.io.orc.encoded; - -public class OrcCacheKey extends OrcBatchKey { - public int colIx; - - public OrcCacheKey(long file, int stripeIx, int rgIx, int colIx) { - super(file, stripeIx, rgIx); - this.colIx = colIx; - } - - public OrcCacheKey(OrcBatchKey batchKey, int colIx) { - super(batchKey.file, batchKey.stripeIx, batchKey.rgIx); - this.colIx = colIx; - } - - public OrcBatchKey copyToPureBatchKey() { - return new OrcBatchKey(file, stripeIx, rgIx); - } - - @Override - public String toString() { - return "[" + file + ", stripe " + stripeIx + ", rgIx " + rgIx + ", rgIx " + colIx + "]"; - } - - @Override - public int hashCode() { - final int prime = 31; - return super.hashCode() * prime + colIx; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (!(obj instanceof OrcCacheKey)) return false; - OrcCacheKey other = (OrcCacheKey)obj; - // Strings are interned and can thus be compared like this. - return stripeIx == other.stripeIx && rgIx == other.rgIx - && file == other.file && other.colIx == colIx; - } -} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java index 246ead61e5b2c5d0ab1de79181537926a5b0edda..4405232289971ae56728d32863404118760ba770 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/Reader.java @@ -49,11 +49,11 @@ * We assume the sort will stay the same for backward compat. */ public static final int MAX_DATA_STREAMS = OrcProto.Stream.Kind.ROW_INDEX.getNumber(); - public void init(long fileId, int stripeIx, int rgIx, int columnCount) { + public void init(Object fileKey, int stripeIx, int rgIx, int columnCount) { if (batchKey == null) { - batchKey = new OrcBatchKey(fileId, stripeIx, rgIx); + batchKey = new OrcBatchKey(fileKey, stripeIx, rgIx); } else { - batchKey.set(fileId, stripeIx, rgIx); + batchKey.set(fileKey, stripeIx, rgIx); } resetColumnArrays(columnCount); } @@ -61,12 +61,12 @@ public void init(long fileId, int stripeIx, int rgIx, int columnCount) { /** * Creates the encoded reader. - * @param fileId File ID to read, to use for cache lookups and such. + * @param fileKey File ID to read, to use for cache lookups and such. * @param dataCache Data cache to use for cache lookups. * @param dataReader Data reader to read data not found in cache (from disk, HDFS, and such). * @param pf Pool factory to create object pools. * @return The reader. */ EncodedReader encodedReader( - Long fileId, DataCache dataCache, DataReader dataReader, PoolFactory pf) throws IOException; + Object fileKey, DataCache dataCache, DataReader dataReader, PoolFactory pf) throws IOException; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java index b0ac5034f3af8519eb4a30c7efe0ca7e036647a7..4856fb3ceb71a6b04a93e621db0830a5d1234650 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/ReaderImpl.java @@ -34,8 +34,8 @@ public ReaderImpl(Path path, ReaderOptions options) throws IOException { @Override public EncodedReader encodedReader( - Long fileId, DataCache dataCache, DataReader dataReader, PoolFactory pf) throws IOException { - return new EncodedReaderImpl(fileId, types, + Object fileKey, DataCache dataCache, DataReader dataReader, PoolFactory pf) throws IOException { + return new EncodedReaderImpl(fileKey, types, codec, bufferSize, rowIndexStride, dataCache, dataReader, pf); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/StreamUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/StreamUtils.java index 9ac53afa373d230d824b133464d4e3beff59e4fa..cef765c42619c5d4cea35f0a720e4661cbe5960b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/StreamUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/StreamUtils.java @@ -35,7 +35,6 @@ * Create SettableUncompressedStream from stream buffer. * * @param streamName - stream name - * @param fileId - file id * @param streamBuffer - stream buffer * @return - SettableUncompressedStream * @throws IOException diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 80208c2db57d22d1ab3ced813a68ff07d4988e74..6d27f557ad0ee76b9439e3ce114decc3c526cfe0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -89,6 +89,7 @@ import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.MetadataPpdResult; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Order; @@ -3485,7 +3486,7 @@ private boolean logDumpPhase(String phase) { } public Iterable> getFileMetadata( - List fileIds, Configuration conf) throws HiveException { + List fileIds) throws HiveException { try { return getMSC().getFileMetadata(fileIds); } catch (TException e) { @@ -3493,6 +3494,15 @@ private boolean logDumpPhase(String phase) { } } + public Iterable> getFileMetadataByExpr( + List fileIds, ByteBuffer sarg, boolean doGetFooters) throws HiveException { + try { + return getMSC().getFileMetadataBySarg(fileIds, sarg, doGetFooters); + } catch (TException e) { + throw new HiveException(e); + } + } + public void clearFileMetadata(List fileIds) throws HiveException { try { getMSC().clearFileMetadata(fileIds); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java index f273d25dd9a1bd49e48e3c21f850d1518d977aec..4bcf6bf1d89b198d1123ed129d4d07d8d1843bc7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/stats/annotation/StatsRulesProcFactory.java @@ -26,8 +26,6 @@ import java.util.Map.Entry; import java.util.Stack; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -80,6 +78,8 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -500,6 +500,181 @@ private long getMaxNulls(Statistics stats, ExprNodeDesc pred) { return maxNoNulls; } + private long evaluateComparator(Statistics stats, ExprNodeGenericFuncDesc genFunc) { + long numRows = stats.getNumRows(); + GenericUDF udf = genFunc.getGenericUDF(); + + ExprNodeColumnDesc columnDesc; + ExprNodeConstantDesc constantDesc; + boolean upperBound; + String boundValue = null; + if (genFunc.getChildren().get(0) instanceof ExprNodeColumnDesc && + genFunc.getChildren().get(1) instanceof ExprNodeConstantDesc) { + columnDesc = (ExprNodeColumnDesc) genFunc.getChildren().get(0); + constantDesc = (ExprNodeConstantDesc) genFunc.getChildren().get(1); + // Comparison to null will always return false + if (constantDesc.getValue() == null) { + return 0; + } + if (udf instanceof GenericUDFOPEqualOrGreaterThan || + udf instanceof GenericUDFOPGreaterThan) { + boundValue = constantDesc.getValue().toString(); + upperBound = false; + } else { + boundValue = constantDesc.getValue().toString(); + upperBound = true; + } + } else if (genFunc.getChildren().get(1) instanceof ExprNodeColumnDesc && + genFunc.getChildren().get(0) instanceof ExprNodeConstantDesc) { + columnDesc = (ExprNodeColumnDesc) genFunc.getChildren().get(1); + constantDesc = (ExprNodeConstantDesc) genFunc.getChildren().get(0); + // Comparison to null will always return false + if (constantDesc.getValue() == null) { + return 0; + } + if (udf instanceof GenericUDFOPEqualOrGreaterThan || + udf instanceof GenericUDFOPGreaterThan) { + boundValue = constantDesc.getValue().toString(); + upperBound = true; + } else { + boundValue = constantDesc.getValue().toString(); + upperBound = false; + } + } else { + // default + return numRows / 3; + } + + ColStatistics cs = stats.getColumnStatisticsFromColName(columnDesc.getColumn()); + if (cs != null && cs.getRange() != null && + cs.getRange().maxValue != null && cs.getRange().minValue != null) { + String colTypeLowerCase = columnDesc.getTypeString().toLowerCase(); + try { + if (colTypeLowerCase.equals(serdeConstants.TINYINT_TYPE_NAME)) { + byte value = new Byte(boundValue); + byte maxValue = cs.getRange().maxValue.byteValue(); + byte minValue = cs.getRange().minValue.byteValue(); + if (upperBound) { + if (maxValue < value) { + return numRows; + } + if (minValue > value) { + return 0; + } + } else { + if (minValue > value) { + return numRows; + } + if (maxValue < value) { + return 0; + } + } + } else if (colTypeLowerCase.equals(serdeConstants.SMALLINT_TYPE_NAME)) { + short value = new Short(boundValue); + short maxValue = cs.getRange().maxValue.shortValue(); + short minValue = cs.getRange().minValue.shortValue(); + if (upperBound) { + if (maxValue < value) { + return numRows; + } + if (minValue > value) { + return 0; + } + } else { + if (minValue > value) { + return numRows; + } + if (maxValue < value) { + return 0; + } + } + } else if (colTypeLowerCase.equals(serdeConstants.INT_TYPE_NAME) || + colTypeLowerCase.equals(serdeConstants.DATE_TYPE_NAME)) { + // Date is an integer internally + int value = new Integer(boundValue); + int maxValue = cs.getRange().maxValue.intValue(); + int minValue = cs.getRange().minValue.intValue(); + if (upperBound) { + if (maxValue < value) { + return numRows; + } + if (minValue > value) { + return 0; + } + } else { + if (minValue > value) { + return numRows; + } + if (maxValue < value) { + return 0; + } + } + } else if (colTypeLowerCase.equals(serdeConstants.BIGINT_TYPE_NAME)) { + long value = new Long(boundValue); + long maxValue = cs.getRange().maxValue.longValue(); + long minValue = cs.getRange().minValue.longValue(); + if (upperBound) { + if (maxValue < value) { + return numRows; + } + if (minValue > value) { + return 0; + } + } else { + if (minValue > value) { + return numRows; + } + if (maxValue < value) { + return 0; + } + } + } else if (colTypeLowerCase.equals(serdeConstants.FLOAT_TYPE_NAME)) { + float value = new Float(boundValue); + float maxValue = cs.getRange().maxValue.floatValue(); + float minValue = cs.getRange().minValue.floatValue(); + if (upperBound) { + if (maxValue < value) { + return numRows; + } + if (minValue > value) { + return 0; + } + } else { + if (minValue > value) { + return numRows; + } + if (maxValue < value) { + return 0; + } + } + } else if (colTypeLowerCase.equals(serdeConstants.DOUBLE_TYPE_NAME)) { + double value = new Double(boundValue); + double maxValue = cs.getRange().maxValue.doubleValue(); + double minValue = cs.getRange().minValue.doubleValue(); + if (upperBound) { + if (maxValue < value) { + return numRows; + } + if (minValue > value) { + return 0; + } + } else { + if (minValue > value) { + return numRows; + } + if (maxValue < value) { + return 0; + } + } + } + } catch (NumberFormatException nfe) { + return numRows / 3; + } + } + // default + return numRows / 3; + } + private long evaluateChildExpr(Statistics stats, ExprNodeDesc child, AnnotateStatsProcCtx aspCtx, List neededCols, FilterOperator fop, long evaluatedRowCount) throws CloneNotSupportedException { @@ -578,9 +753,10 @@ private long evaluateChildExpr(Statistics stats, ExprNodeDesc child, } else if (udf instanceof GenericUDFOPNotEqual) { return numRows; } else if (udf instanceof GenericUDFOPEqualOrGreaterThan - || udf instanceof GenericUDFOPEqualOrLessThan || udf instanceof GenericUDFOPGreaterThan + || udf instanceof GenericUDFOPEqualOrLessThan + || udf instanceof GenericUDFOPGreaterThan || udf instanceof GenericUDFOPLessThan) { - return numRows / 3; + return evaluateComparator(stats, genFunc); } else if (udf instanceof GenericUDFOPNotNull) { return evaluateNotNullExpr(stats, genFunc); } else if (udf instanceof GenericUDFOPNull) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java index 2ecdde9799683cd104323af170d8905fb3005b9e..6d0f14a6418caefaacc0326fc0a946178af673d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/OperationLog.java @@ -39,7 +39,11 @@ private final LogFile logFile; private LoggingLevel opLoggingLevel = LoggingLevel.UNKNOWN; - public static enum LoggingLevel { + public PrintStream getPrintStream() { + return logFile.getPrintStream(); + } + + public enum LoggingLevel { NONE, EXECUTION, PERFORMANCE, VERBOSE, UNKNOWN } @@ -221,5 +225,9 @@ private void resetIn() { } return logs; } + + public PrintStream getPrintStream() { + return out; + } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java index 9d139ba8e640d739ca69a291cf9b1fa6d3bb3897..d8acf945f49f118b6fc80230b5c3630df84e3ce5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java @@ -724,6 +724,8 @@ public static ColStatistics getColStatistics(ColumnStatisticsObj cso, String tab } } else if (colTypeLowerCase.equals(serdeConstants.DATE_TYPE_NAME)) { cs.setAvgColLen(JavaDataModel.get().lengthOfDate()); + cs.setRange(csd.getDateStats().getLowValue().getDaysSinceEpoch(), + csd.getDateStats().getHighValue().getDaysSinceEpoch()); } else { // Columns statistics for complex datatypes are not supported yet return null; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index 4fafe8c3217ac5942f2bc044223b8e197e9285a8..1a64f3a2fcdb20ed913e92b66de0947b16ef0723 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -65,12 +65,12 @@ import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; import org.apache.hadoop.hive.ql.io.AcidInputFormat; import org.apache.hadoop.hive.ql.io.AcidOutputFormat; -import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.io.IOConstants; import org.apache.hadoop.hive.ql.io.InputFormatChecker; +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.Context; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.SplitStrategy; import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg; import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf; @@ -622,8 +622,8 @@ public void testEtlCombinedStrategy() throws Exception { public SplitStrategy createOrCombineStrategy(OrcInputFormat.Context context, MockFileSystem fs, String path, OrcInputFormat.CombinedCtx combineCtx) throws IOException { OrcInputFormat.AcidDirInfo adi = createAdi(context, fs, path); - return OrcInputFormat.determineSplitStrategy( - combineCtx, context, adi.fs, adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, null); + return OrcInputFormat.determineSplitStrategy(combineCtx, context, + adi.fs, adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, null, true); } public OrcInputFormat.AcidDirInfo createAdi( @@ -636,7 +636,7 @@ public void testEtlCombinedStrategy() throws Exception { OrcInputFormat.Context context, OrcInputFormat.FileGenerator gen) throws IOException { OrcInputFormat.AcidDirInfo adi = gen.call(); return OrcInputFormat.determineSplitStrategy( - null, context, adi.fs, adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, null); + null, context, adi.fs, adi.splitPath, adi.acidInfo, adi.baseOrOriginalFiles, null, true); } public static class MockBlock { @@ -1113,8 +1113,8 @@ public void testAddSplit() throws Exception { OrcInputFormat.Context context = new OrcInputFormat.Context(conf); OrcInputFormat.SplitGenerator splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs, - AcidUtils.createOriginalObj(null, fs.getFileStatus(new Path("/a/file"))), null, true, - new ArrayList(), true, null, null), null); + fs.getFileStatus(new Path("/a/file")), null, true, + new ArrayList(), true, null, null), null, true); OrcSplit result = splitter.createSplit(0, 200, null); assertEquals(0, result.getStart()); assertEquals(200, result.getLength()); @@ -1154,8 +1154,8 @@ public void testSplitGenerator() throws Exception { OrcInputFormat.Context context = new OrcInputFormat.Context(conf); OrcInputFormat.SplitGenerator splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs, - AcidUtils.createOriginalObj(null, fs.getFileStatus(new Path("/a/file"))), null, true, - new ArrayList(), true, null, null), null); + fs.getFileStatus(new Path("/a/file")), null, true, + new ArrayList(), true, null, null), null, true); List results = splitter.call(); OrcSplit result = results.get(0); assertEquals(3, result.getStart()); @@ -1177,8 +1177,8 @@ public void testSplitGenerator() throws Exception { HiveConf.setLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 0); context = new OrcInputFormat.Context(conf); splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs, - AcidUtils.createOriginalObj(null, fs.getFileStatus(new Path("/a/file"))), null, true, - new ArrayList(), true, null, null), null); + fs.getFileStatus(new Path("/a/file")), null, true, + new ArrayList(), true, null, null), null, true); results = splitter.call(); for(int i=0; i < stripeSizes.length; ++i) { assertEquals("checking stripe " + i + " size", @@ -1206,7 +1206,7 @@ public void testProjectedColumnSize() throws Exception { OrcInputFormat.SplitGenerator splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs, fs.getFileStatus(new Path("/a/file")), null, true, - new ArrayList(), true, null, null), null); + new ArrayList(), true, null, null), null, true); List results = splitter.call(); OrcSplit result = results.get(0); assertEquals(3, results.size()); @@ -1229,7 +1229,7 @@ public void testProjectedColumnSize() throws Exception { splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs, fs.getFileStatus(new Path("/a/file")), null, true, new ArrayList(), - true, null, null), null); + true, null, null), null, true); results = splitter.call(); assertEquals(5, results.size()); for (int i = 0; i < stripeSizes.length; ++i) { @@ -1249,7 +1249,7 @@ public void testProjectedColumnSize() throws Exception { splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs, fs.getFileStatus(new Path("/a/file")), null, true, new ArrayList(), - true, null, null), null); + true, null, null), null, true); results = splitter.call(); assertEquals(1, results.size()); result = results.get(0); @@ -2165,7 +2165,7 @@ public void testDoAs() throws Exception { ugi.doAs(new PrivilegedExceptionAction() { @Override public Void run() throws Exception { - OrcInputFormat.generateSplitsInfo(conf, -1); + OrcInputFormat.generateSplitsInfo(conf, new Context(conf, -1, null)); return null; } }); @@ -2184,7 +2184,7 @@ public Void run() throws Exception { } assertEquals(1, OrcInputFormat.Context.getCurrentThreadPoolSize()); FileInputFormat.setInputPaths(conf, "mock:/ugi/2"); - List splits = OrcInputFormat.generateSplitsInfo(conf, -1); + List splits = OrcInputFormat.generateSplitsInfo(conf, new Context(conf, -1, null)); assertEquals(1, splits.size()); } finally { MockFileSystem.clearGlobalFiles(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java index 7a93b545979e8ae79d6647aad342723346ef0929..62a0ab03b75df5ab1c92923c7ff10fd6c5a17ede 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcSplitElimination.java @@ -18,18 +18,34 @@ package org.apache.hadoop.hive.ql.io.orc; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.api.MetadataPpdResult; +import org.apache.hadoop.hive.metastore.filemeta.OrcFileMetadataHandler; +import org.apache.hadoop.hive.metastore.hbase.MetadataStore; import org.apache.hadoop.hive.ql.exec.SerializationUtilities; +import org.apache.hadoop.hive.ql.io.orc.ExternalCache.ExternalFooterCachesByConf; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionExpressionForMetastore; import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; @@ -43,6 +59,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.FileInputFormat; +import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.InputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; @@ -50,6 +67,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.collect.Lists; @@ -76,7 +95,7 @@ JobConf conf; FileSystem fs; - Path testFilePath; + Path testFilePath, testFilePath2; @Rule public TestName testCaseName = new TestName(); @@ -94,17 +113,15 @@ public void openFileSystem() throws Exception { fs = FileSystem.getLocal(conf); testFilePath = new Path(workDir, "TestOrcFile." + testCaseName.getMethodName() + ".orc"); + testFilePath2 = new Path(workDir, "TestOrcFile." + + testCaseName.getMethodName() + ".2.orc"); fs.delete(testFilePath, false); + fs.delete(testFilePath2, false); } @Test public void testSplitEliminationSmallMaxSplit() throws Exception { - ObjectInspector inspector; - synchronized (TestOrcFile.class) { - inspector = ObjectInspectorFactory - .getReflectionObjectInspector(AllTypesRow.class, - ObjectInspectorFactory.ObjectInspectorOptions.JAVA); - } + ObjectInspector inspector = createIO(); Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector, 100000, CompressionKind.NONE, 10000, 10000); writeData(writer); @@ -116,13 +133,10 @@ public void testSplitEliminationSmallMaxSplit() throws Exception { GenericUDF udf = new GenericUDFOPEqualOrLessThan(); List childExpr = Lists.newArrayList(); - ExprNodeColumnDesc col = new ExprNodeColumnDesc(Long.class, "userid", "T", false); - ExprNodeConstantDesc con = new ExprNodeConstantDesc(100); - childExpr.add(col); - childExpr.add(con); - ExprNodeGenericFuncDesc en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr); - String sargStr = SerializationUtilities.serializeExpression(en); - conf.set("hive.io.filter.expr.serialized", sargStr); + ExprNodeConstantDesc con; + ExprNodeGenericFuncDesc en; + String sargStr; + createTestSarg(inspector, udf, childExpr); InputSplit[] splits = in.getSplits(conf, 1); assertEquals(5, splits.length); @@ -177,12 +191,7 @@ public void testSplitEliminationSmallMaxSplit() throws Exception { @Test public void testSplitEliminationLargeMaxSplit() throws Exception { - ObjectInspector inspector; - synchronized (TestOrcFile.class) { - inspector = ObjectInspectorFactory - .getReflectionObjectInspector(AllTypesRow.class, - ObjectInspectorFactory.ObjectInspectorOptions.JAVA); - } + ObjectInspector inspector = createIO(); Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector, 100000, CompressionKind.NONE, 10000, 10000); writeData(writer); @@ -194,13 +203,10 @@ public void testSplitEliminationLargeMaxSplit() throws Exception { GenericUDF udf = new GenericUDFOPEqualOrLessThan(); List childExpr = Lists.newArrayList(); - ExprNodeColumnDesc col = new ExprNodeColumnDesc(Long.class, "userid", "T", false); - ExprNodeConstantDesc con = new ExprNodeConstantDesc(100); - childExpr.add(col); - childExpr.add(con); - ExprNodeGenericFuncDesc en = new ExprNodeGenericFuncDesc(inspector, udf, childExpr); - String sargStr = SerializationUtilities.serializeExpression(en); - conf.set("hive.io.filter.expr.serialized", sargStr); + ExprNodeConstantDesc con; + ExprNodeGenericFuncDesc en; + String sargStr; + createTestSarg(inspector, udf, childExpr); InputSplit[] splits = in.getSplits(conf, 1); assertEquals(2, splits.length); @@ -266,12 +272,7 @@ public void testSplitEliminationLargeMaxSplit() throws Exception { @Test public void testSplitEliminationComplexExpr() throws Exception { - ObjectInspector inspector; - synchronized (TestOrcFile.class) { - inspector = ObjectInspectorFactory - .getReflectionObjectInspector(AllTypesRow.class, - ObjectInspectorFactory.ObjectInspectorOptions.JAVA); - } + ObjectInspector inspector = createIO(); Writer writer = OrcFile.createWriter(fs, testFilePath, conf, inspector, 100000, CompressionKind.NONE, 10000, 10000); writeData(writer); @@ -385,6 +386,342 @@ public void testSplitEliminationComplexExpr() throws Exception { assertEquals(1, splits.length); } + private static class OrcInputFormatForTest extends OrcInputFormat { + public static void clearLocalCache() { + OrcInputFormat.Context.clearLocalCache(); + } + static MockExternalCaches caches = new MockExternalCaches(); + @Override + protected ExternalFooterCachesByConf createExternalCaches() { + return caches; + } + } + + private static class MockExternalCaches + implements ExternalFooterCachesByConf, ExternalFooterCachesByConf.Cache, MetadataStore { + private static class MockItem { + ByteBuffer data; + ByteBuffer[] extraCols; + ByteBuffer[] extraData; + + @Override + public String toString() { + return (data == null ? 0 : data.remaining()) + " bytes" + + (extraCols == null ? "" : ("; " + extraCols.length + " extras")); + } + } + private final Map cache = new ConcurrentHashMap<>(); + private final OrcFileMetadataHandler handler = new OrcFileMetadataHandler(); + private final AtomicInteger putCount = new AtomicInteger(0), + getCount = new AtomicInteger(0), getHitCount = new AtomicInteger(0), + getByExprCount = new AtomicInteger(0), getHitByExprCount = new AtomicInteger(); + + public void resetCounts() { + getByExprCount.set(0); + getCount.set(0); + putCount.set(0); + getHitCount.set(0); + getHitByExprCount.set(0); + } + + @Override + public Cache getCache(HiveConf conf) throws IOException { + handler.configure(conf, new PartitionExpressionForMetastore(), this); + return this; + } + + @Override + public Iterator> getFileMetadataByExpr( + List fileIds, ByteBuffer sarg, boolean doGetFooters) throws HiveException { + getByExprCount.incrementAndGet(); + ByteBuffer[] metadatas = new ByteBuffer[fileIds.size()]; + ByteBuffer[] ppdResults = new ByteBuffer[fileIds.size()]; + boolean[] eliminated = new boolean[fileIds.size()]; + try { + byte[] bb = new byte[sarg.remaining()]; + System.arraycopy(sarg.array(), sarg.arrayOffset(), bb, 0, sarg.remaining()); + handler.getFileMetadataByExpr(fileIds, bb, metadatas, ppdResults, eliminated); + } catch (IOException e) { + throw new HiveException(e); + } + Map result = new HashMap<>(); + for (int i = 0; i < metadatas.length; ++i) { + long fileId = fileIds.get(i); + ByteBuffer metadata = metadatas[i]; + if (metadata == null) continue; + getHitByExprCount.incrementAndGet(); + metadata = eliminated[i] ? null : metadata; + MetadataPpdResult mpr = new MetadataPpdResult(); + ByteBuffer bitset = eliminated[i] ? null : ppdResults[i]; + mpr.setMetadata(doGetFooters ? metadata : null); + mpr.setIncludeBitset(bitset); + result.put(fileId, mpr); + } + return result.entrySet().iterator(); + } + + @Override + public void clearFileMetadata(List fileIds) throws HiveException { + for (Long id : fileIds) { + cache.remove(id); + } + } + + @Override + public Iterator> getFileMetadata(List fileIds) + throws HiveException { + getCount.incrementAndGet(); + HashMap result = new HashMap<>(); + for (Long id : fileIds) { + MockItem mi = cache.get(id); + if (mi == null) continue; + getHitCount.incrementAndGet(); + result.put(id, mi.data); + } + return result.entrySet().iterator(); + } + + @Override + public void putFileMetadata(ArrayList fileIds, + ArrayList values) throws HiveException { + putCount.incrementAndGet(); + ByteBuffer[] addedCols = handler.createAddedCols(); + ByteBuffer[][] addedVals = null; + if (addedCols != null) { + addedVals = handler.createAddedColVals(values); + } + try { + storeFileMetadata(fileIds, values, addedCols, addedVals); + } catch (IOException | InterruptedException e) { + throw new HiveException(e); + } + } + + // MetadataStore + @Override + public void getFileMetadata(List fileIds, ByteBuffer[] result) throws IOException { + for (int i = 0; i < fileIds.size(); ++i) { + MockItem mi = cache.get(fileIds.get(i)); + result[i] = (mi == null ? null : mi.data); + } + } + + @Override + public void storeFileMetadata(List fileIds, List metadataBuffers, + ByteBuffer[] addedCols, ByteBuffer[][] addedVals) + throws IOException, InterruptedException { + for (int i = 0; i < fileIds.size(); ++i) { + ByteBuffer value = (metadataBuffers != null) ? metadataBuffers.get(i) : null; + ByteBuffer[] av = addedVals == null ? null : addedVals[i]; + storeFileMetadata(fileIds.get(i), value, addedCols, av); + } + } + + @Override + public void storeFileMetadata(long fileId, ByteBuffer metadata, + ByteBuffer[] addedCols, ByteBuffer[] addedVals) throws IOException, InterruptedException { + if (metadata == null) { + cache.remove(metadata); + return; + } + MockItem mi = new MockItem(); + mi.data = metadata; + if (addedVals != null) { + mi.extraCols = addedCols; + mi.extraData = addedVals; + } + cache.put(fileId, mi); + } + } + + private static final Logger LOG = LoggerFactory.getLogger(TestOrcSplitElimination.class); + + @Test + public void testExternalFooterCache() throws Exception { + testFooterExternalCacheImpl(false); + } + + @Test + public void testExternalFooterCachePpd() throws Exception { + testFooterExternalCacheImpl(true); + } + + private final static class FsWithHash { + private FileSplit fs; + public FsWithHash(FileSplit fs) { + this.fs = fs; + } + @Override + public int hashCode() { + if (fs == null) return 0; + final int prime = 31; + int result = prime * 1 + fs.getPath().hashCode(); + result = prime * result + Long.valueOf(fs.getStart()).hashCode(); + return prime * result + Long.valueOf(fs.getLength()).hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!(obj instanceof FsWithHash)) return false; + FsWithHash other = (FsWithHash)obj; + if ((fs == null) != (other.fs == null)) return false; + if (fs == null && other.fs == null) return true; + return fs.getStart() == other.fs.getStart() && fs.getLength() == other.fs.getLength() + && fs.getPath().equals(other.fs.getPath()); + } + } + + private void testFooterExternalCacheImpl(boolean isPpd) throws IOException { + ObjectInspector inspector = createIO(); + writeFile(inspector, testFilePath); + writeFile(inspector, testFilePath2); + + GenericUDF udf = new GenericUDFOPEqualOrLessThan(); + List childExpr = Lists.newArrayList(); + createTestSarg(inspector, udf, childExpr); + setupExternalCacheConfig(isPpd, testFilePath + "," + testFilePath2); + // Get the base values w/o cache. + conf.setBoolean(ConfVars.HIVE_ORC_MS_FOOTER_CACHE_ENABLED.varname, false); + OrcInputFormatForTest.clearLocalCache(); + OrcInputFormat in0 = new OrcInputFormat(); + InputSplit[] originals = in0.getSplits(conf, -1); + assertEquals(10, originals.length); + HashSet originalHs = new HashSet<>(); + for (InputSplit original : originals) { + originalHs.add(new FsWithHash((FileSplit)original)); + } + + // Populate the cache. + conf.setBoolean(ConfVars.HIVE_ORC_MS_FOOTER_CACHE_ENABLED.varname, true); + OrcInputFormatForTest in = new OrcInputFormatForTest(); + OrcInputFormatForTest.clearLocalCache(); + OrcInputFormatForTest.caches.resetCounts(); + OrcInputFormatForTest.caches.cache.clear(); + InputSplit[] splits = in.getSplits(conf, -1); + // Puts, gets, hits, unused, unused. + @SuppressWarnings("static-access") + AtomicInteger[] counts = { in.caches.putCount, + isPpd ? in.caches.getByExprCount : in.caches.getCount, + isPpd ? in.caches.getHitByExprCount : in.caches.getHitCount, + isPpd ? in.caches.getCount : in.caches.getByExprCount, + isPpd ? in.caches.getHitCount : in.caches.getHitByExprCount }; + + verifySplits(originalHs, splits); + verifyCallCounts(counts, 2, 2, 0); + assertEquals(2, OrcInputFormatForTest.caches.cache.size()); + + // Verify we can get from cache. + OrcInputFormatForTest.clearLocalCache(); + OrcInputFormatForTest.caches.resetCounts(); + splits = in.getSplits(conf, -1); + verifySplits(originalHs, splits); + verifyCallCounts(counts, 0, 2, 2); + + // Verify ORC SARG still works. + OrcInputFormatForTest.clearLocalCache(); + OrcInputFormatForTest.caches.resetCounts(); + childExpr.set(1, new ExprNodeConstantDesc(5)); + conf.set("hive.io.filter.expr.serialized", SerializationUtilities.serializeExpression( + new ExprNodeGenericFuncDesc(inspector, udf, childExpr))); + splits = in.getSplits(conf, -1); + InputSplit[] filtered = { originals[0], originals[4], originals[5], originals[9] }; + originalHs = new HashSet<>(); + for (InputSplit original : filtered) { + originalHs.add(new FsWithHash((FileSplit)original)); + } + verifySplits(originalHs, splits); + verifyCallCounts(counts, 0, 2, 2); + + // Verify corrupted cache value gets replaced. + OrcInputFormatForTest.clearLocalCache(); + OrcInputFormatForTest.caches.resetCounts(); + Map.Entry e = + OrcInputFormatForTest.caches.cache.entrySet().iterator().next(); + Long key = e.getKey(); + byte[] someData = new byte[8]; + ByteBuffer toCorrupt = e.getValue().data; + System.arraycopy(toCorrupt.array(), toCorrupt.arrayOffset(), someData, 0, someData.length); + toCorrupt.putLong(0, 0L); + splits = in.getSplits(conf, -1); + verifySplits(originalHs, splits); + if (!isPpd) { // Recovery is not implemented yet for PPD path. + ByteBuffer restored = OrcInputFormatForTest.caches.cache.get(key).data; + byte[] newData = new byte[someData.length]; + System.arraycopy(restored.array(), restored.arrayOffset(), newData, 0, newData.length); + assertArrayEquals(someData, newData); + } + } + + private void verifyCallCounts(AtomicInteger[] counts, int puts, int gets, int hits) { + assertEquals("puts", puts, counts[0].get()); + assertEquals("gets", gets, counts[1].get()); + assertEquals("hits", hits, counts[2].get()); + assertEquals("unused1", 0, counts[3].get()); + assertEquals("unused2", 0, counts[4].get()); + } + + private void verifySplits(HashSet originalHs, InputSplit[] splits) { + if (originalHs.size() != splits.length) { + String s = "Expected ["; + for (FsWithHash fwh : originalHs) { + s += toString(fwh.fs) + ", "; + } + s += "], actual ["; + for (InputSplit fs : splits) { + s += toString((FileSplit)fs) + ", "; + } + fail(s + "]"); + } + for (int i = 0; i < splits.length; ++i) { + FileSplit fs = (FileSplit)splits[i]; + if (!originalHs.contains(new FsWithHash((FileSplit)splits[i]))) { + String s = " in ["; + for (FsWithHash fwh : originalHs) { + s += toString(fwh.fs) + ", "; + } + fail("Cannot find " + toString(fs) + s); + } + } + + } + + private static String toString(FileSplit fs) { + return "{" + fs.getPath() + ", " + fs.getStart() + ", " + fs.getLength() + "}"; + } + + private void setupExternalCacheConfig(boolean isPpd, String paths) { + FileInputFormat.setInputPaths(conf, paths); + conf.set(ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname, "ETL"); + conf.setLong(HiveConf.ConfVars.MAPREDMINSPLITSIZE.varname, 1000); + conf.setLong(HiveConf.ConfVars.MAPREDMAXSPLITSIZE.varname, 5000); + conf.setBoolean(ConfVars.HIVE_ORC_MS_FOOTER_CACHE_PPD.varname, isPpd); + conf.setBoolean(ConfVars.HIVEOPTINDEXFILTER.varname, isPpd); + } + + private ObjectInspector createIO() { + synchronized (TestOrcFile.class) { + return ObjectInspectorFactory + .getReflectionObjectInspector(AllTypesRow.class, + ObjectInspectorFactory.ObjectInspectorOptions.JAVA); + } + } + + private void writeFile(ObjectInspector inspector, Path filePath) throws IOException { + Writer writer = OrcFile.createWriter( + fs, filePath, conf, inspector, 100000, CompressionKind.NONE, 10000, 10000); + writeData(writer); + writer.close(); + } + + private void createTestSarg( + ObjectInspector inspector, GenericUDF udf, List childExpr) { + childExpr.add(new ExprNodeColumnDesc(Long.class, "userid", "T", false)); + childExpr.add(new ExprNodeConstantDesc(100)); + conf.set("hive.io.filter.expr.serialized", SerializationUtilities.serializeExpression( + new ExprNodeGenericFuncDesc(inspector, udf, childExpr))); + } + private void writeData(Writer writer) throws IOException { for (int i = 0; i < 25000; i++) { if (i == 0) { diff --git a/ql/src/test/queries/clientpositive/annotate_stats_filter.q b/ql/src/test/queries/clientpositive/annotate_stats_filter.q index 436c0530d9479a5b533b37eedd71b01e840eb29f..a352a77bc269dc97af928b5a94ab2abbbaa79f04 100644 --- a/ql/src/test/queries/clientpositive/annotate_stats_filter.q +++ b/ql/src/test/queries/clientpositive/annotate_stats_filter.q @@ -83,9 +83,17 @@ explain select * from loc_orc where (year=2001 and year is null) or (state='CA') -- numRows: 1 rawDataSize: 102 explain select * from loc_orc where (year=2001 or year is null) and (state='CA'); --- all inequality conditions rows/3 is the rules --- numRows: 2 rawDataSize: 204 +-- inequality conditions falling out of range. total or zero (converted to one) +-- numRows: 1 rawDataSize: 102 +-- numRows: 8 rawDataSize: 804 explain select * from loc_orc where locid < 30; explain select * from loc_orc where locid > 30; explain select * from loc_orc where locid <= 30; explain select * from loc_orc where locid >= 30; + +-- all inequality conditions falling within range. rows/3 is the rules +-- numRows: 2 rawDataSize: 204 +explain select * from loc_orc where locid < 3; +explain select * from loc_orc where locid > 3; +explain select * from loc_orc where locid <= 3; +explain select * from loc_orc where locid >= 3; diff --git a/ql/src/test/queries/clientpositive/encryption_drop_partition.q b/ql/src/test/queries/clientpositive/encryption_drop_partition.q index a26aa19aaa086703dc4b388f7f5c1f7c330f3864..57dfabdc717d6343050d690d222bcdfddd4bef0b 100644 --- a/ql/src/test/queries/clientpositive/encryption_drop_partition.q +++ b/ql/src/test/queries/clientpositive/encryption_drop_partition.q @@ -12,7 +12,17 @@ CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse. INSERT INTO encrypted_table_dp PARTITION(p)(p,key,value) values('2014-09-23', 1, 'foo'),('2014-09-24', 2, 'bar'); SELECT * FROM encrypted_table_dp; + +CREATE EXTERNAL TABLE encrypted_ext_table_dp (key INT, value STRING) partitioned by (p STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table_dp'; +ALTER TABLE encrypted_ext_table_dp ADD PARTITION (p='2014-09-23') LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table_dp/p=2014-09-23'; +SELECT * FROM encrypted_ext_table_dp; +ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23'); +SELECT * FROM encrypted_ext_table_dp; +DROP TABLE encrypted_ext_table_dp; + +SELECT * FROM encrypted_table_dp; ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23'); SELECT * FROM encrypted_table_dp; ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PURGE; SELECT * FROM encrypted_table_dp; +DROP TABLE encrypted_table_dp PURGE; diff --git a/ql/src/test/queries/clientpositive/encryption_drop_table.q b/ql/src/test/queries/clientpositive/encryption_drop_table.q index 193beea635de236d59338043374cecb3e51f61c5..2ae3c69e02edc8e48801e021e5c845d92a26d7d5 100644 --- a/ql/src/test/queries/clientpositive/encryption_drop_table.q +++ b/ql/src/test/queries/clientpositive/encryption_drop_table.q @@ -10,9 +10,16 @@ CRYPTO CREATE_KEY --keyName key_128 --bitLength 128; CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table; INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src; + +CREATE EXTERNAL TABLE encrypted_ext_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table'; +SHOW TABLES; + +DROP TABLE default.encrypted_ext_table; SHOW TABLES; + DROP TABLE default.encrypted_table; SHOW TABLES; + DROP TABLE default.encrypted_table PURGE; SHOW TABLES; -CRYPTO DELETE_KEY --keyName key_128; \ No newline at end of file +CRYPTO DELETE_KEY --keyName key_128; diff --git a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out index b09ad0392a3d3ed330b828023965329a3183e92a..7e697f1ac8aec489fc4be88cca18360049d2ac6f 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out @@ -856,12 +856,14 @@ STAGE PLANS: Processor Tree: ListSink -PREHOOK: query: -- all inequality conditions rows/3 is the rules --- numRows: 2 rawDataSize: 204 +PREHOOK: query: -- inequality conditions falling out of range. total or zero (converted to one) +-- numRows: 1 rawDataSize: 102 +-- numRows: 8 rawDataSize: 804 explain select * from loc_orc where locid < 30 PREHOOK: type: QUERY -POSTHOOK: query: -- all inequality conditions rows/3 is the rules --- numRows: 2 rawDataSize: 204 +POSTHOOK: query: -- inequality conditions falling out of range. total or zero (converted to one) +-- numRows: 1 rawDataSize: 102 +-- numRows: 8 rawDataSize: 804 explain select * from loc_orc where locid < 30 POSTHOOK: type: QUERY STAGE DEPENDENCIES: @@ -877,14 +879,14 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid < 30) (type: boolean) - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -913,14 +915,14 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 30) (type: boolean) - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -949,14 +951,14 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid <= 30) (type: boolean) - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -985,6 +987,154 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid >= 30) (type: boolean) + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: -- all inequality conditions falling within range. rows/3 is the rules +-- numRows: 2 rawDataSize: 204 +explain select * from loc_orc where locid < 3 +PREHOOK: type: QUERY +POSTHOOK: query: -- all inequality conditions falling within range. rows/3 is the rules +-- numRows: 2 rawDataSize: 204 +explain select * from loc_orc where locid < 3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (locid < 3) (type: boolean) + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from loc_orc where locid > 3 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from loc_orc where locid > 3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (locid > 3) (type: boolean) + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from loc_orc where locid <= 3 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from loc_orc where locid <= 3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (locid <= 3) (type: boolean) + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) + outputColumnNames: _col0, _col1, _col2, _col3 + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + File Output Operator + compressed: false + Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: explain select * from loc_orc where locid >= 3 +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from loc_orc where locid >= 3 +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 804 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: (locid >= 3) (type: boolean) Statistics: Num rows: 2 Data size: 204 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int) diff --git a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out index ff952522b2140a4a27bf4e72bb38a00ae3000203..64a57fe301d889e015ed565da5b469d7342bcdd3 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_join_pkfk.q.out @@ -342,31 +342,31 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (s_store_sk > 0) (type: boolean) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: s_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: ss Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (ss_store_sk > 0) (type: boolean) - Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ss_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE Reduce Operator Tree: Join Operator condition map: @@ -375,10 +375,10 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 136 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 136 Data size: 544 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -472,16 +472,16 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((s_floor_space > 0) and s_store_sk is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: s_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: ss Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE @@ -505,10 +505,10 @@ STAGE PLANS: 0 _col0 (type: int) 1 _col0 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 393 Data size: 1572 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 393 Data size: 1572 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 964 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -688,46 +688,46 @@ STAGE PLANS: Statistics: Num rows: 1000 Data size: 3856 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (ss_store_sk > 1000) (type: boolean) - Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ss_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 333 Data size: 1284 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: s Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (s_store_sk > 1000) (type: boolean) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: s_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: s Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (s_store_sk > 1000) (type: boolean) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: s_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 16 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Reduce Operator Tree: Join Operator condition map: @@ -738,14 +738,14 @@ STAGE PLANS: 1 _col0 (type: int) 2 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 213 Data size: 852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 213 Data size: 852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 213 Data size: 852 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -789,16 +789,16 @@ STAGE PLANS: Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: ((s_floor_space > 1000) and s_store_sk is not null) (type: boolean) - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: s_store_sk (type: int) outputColumnNames: _col0 - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE Reduce Output Operator key expressions: _col0 (type: int) sort order: + Map-reduce partition columns: _col0 (type: int) - Statistics: Num rows: 4 Data size: 32 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 96 Basic stats: COMPLETE Column stats: COMPLETE TableScan alias: s Statistics: Num rows: 12 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE @@ -824,14 +824,14 @@ STAGE PLANS: 1 _col0 (type: int) 2 _col0 (type: int) outputColumnNames: _col1 - Statistics: Num rows: 508 Data size: 2032 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 821 Data size: 3284 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: _col1 (type: int) outputColumnNames: _col0 - Statistics: Num rows: 508 Data size: 2032 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 821 Data size: 3284 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 508 Data size: 2032 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 821 Data size: 3284 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/annotate_stats_part.q.out b/ql/src/test/results/clientpositive/annotate_stats_part.q.out index 186f7af0b76cf7a9243671687647c4dce2aa8663..131cf6a67678498faaaadbf6ae26dea1617c09c5 100644 --- a/ql/src/test/results/clientpositive/annotate_stats_part.q.out +++ b/ql/src/test/results/clientpositive/annotate_stats_part.q.out @@ -493,11 +493,11 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 0) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: locid (type: int) outputColumnNames: _col0 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain select locid,year from loc_orc where locid>0 and year='2001' @@ -517,11 +517,11 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 0) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: locid (type: int), '2001' (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 644 Basic stats: COMPLETE Column stats: COMPLETE ListSink PREHOOK: query: explain select * from (select locid,year from loc_orc) test where locid>0 and year='2001' @@ -541,10 +541,10 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 0) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 28 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: locid (type: int), '2001' (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 184 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 7 Data size: 644 Basic stats: COMPLETE Column stats: COMPLETE ListSink diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out index 067bf823f096681c949dc19265955e6a5b0010de..2643006a081bab126ed9f186a363d13b087457b7 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out @@ -41,6 +41,74 @@ POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24 #### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging 1 foo 2014-09-23 2 bar 2014-09-24 +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_ext_table_dp +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_ext_table_dp +#### A masked pattern was here #### +PREHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +PREHOOK: Output: default@encrypted_ext_table_dp +#### A masked pattern was here #### +POSTHOOK: type: ALTERTABLE_ADDPARTS +#### A masked pattern was here #### +POSTHOOK: Output: default@encrypted_ext_table_dp +POSTHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23 +PREHOOK: query: SELECT * FROM encrypted_ext_table_dp +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_ext_table_dp +PREHOOK: Input: default@encrypted_ext_table_dp@p=2014-09-23 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging +POSTHOOK: query: SELECT * FROM encrypted_ext_table_dp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_ext_table_dp +POSTHOOK: Input: default@encrypted_ext_table_dp@p=2014-09-23 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging +1 foo 2014-09-23 +PREHOOK: query: ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23') +PREHOOK: type: ALTERTABLE_DROPPARTS +PREHOOK: Input: default@encrypted_ext_table_dp +PREHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23 +POSTHOOK: query: ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23') +POSTHOOK: type: ALTERTABLE_DROPPARTS +POSTHOOK: Input: default@encrypted_ext_table_dp +POSTHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23 +PREHOOK: query: SELECT * FROM encrypted_ext_table_dp +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_ext_table_dp +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging +POSTHOOK: query: SELECT * FROM encrypted_ext_table_dp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_ext_table_dp +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging +PREHOOK: query: DROP TABLE encrypted_ext_table_dp +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_ext_table_dp +PREHOOK: Output: default@encrypted_ext_table_dp +POSTHOOK: query: DROP TABLE encrypted_ext_table_dp +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_ext_table_dp +POSTHOOK: Output: default@encrypted_ext_table_dp +PREHOOK: query: SELECT * FROM encrypted_table_dp +PREHOOK: type: QUERY +PREHOOK: Input: default@encrypted_table_dp +PREHOOK: Input: default@encrypted_table_dp@p=2014-09-23 +PREHOOK: Input: default@encrypted_table_dp@p=2014-09-24 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging +POSTHOOK: query: SELECT * FROM encrypted_table_dp +POSTHOOK: type: QUERY +POSTHOOK: Input: default@encrypted_table_dp +POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-23 +POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24 +#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging +1 foo 2014-09-23 +2 bar 2014-09-24 PREHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PREHOOK: type: ALTERTABLE_DROPPARTS PREHOOK: Input: default@encrypted_table_dp @@ -79,3 +147,11 @@ POSTHOOK: Input: default@encrypted_table_dp POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24 #### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging 2 bar 2014-09-24 +PREHOOK: query: DROP TABLE encrypted_table_dp PURGE +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_table_dp +PREHOOK: Output: default@encrypted_table_dp +POSTHOOK: query: DROP TABLE encrypted_table_dp PURGE +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_table_dp +POSTHOOK: Output: default@encrypted_table_dp diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out index 55eefa0993eab1c6f52e1a8b10d98dc5d60c7a0e..c5007ee36386437ae3dbe19a8c70310f4854eaf9 100644 --- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out +++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out @@ -24,6 +24,33 @@ POSTHOOK: Input: default@src POSTHOOK: Output: default@encrypted_table POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] +#### A masked pattern was here #### +PREHOOK: type: CREATETABLE +#### A masked pattern was here #### +PREHOOK: Output: database:default +PREHOOK: Output: default@encrypted_ext_table +#### A masked pattern was here #### +POSTHOOK: type: CREATETABLE +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@encrypted_ext_table +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:default +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:default +encrypted_ext_table +encrypted_table +src +PREHOOK: query: DROP TABLE default.encrypted_ext_table +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@encrypted_ext_table +PREHOOK: Output: default@encrypted_ext_table +POSTHOOK: query: DROP TABLE default.encrypted_ext_table +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@encrypted_ext_table +POSTHOOK: Output: default@encrypted_ext_table PREHOOK: query: SHOW TABLES PREHOOK: type: SHOWTABLES PREHOOK: Input: database:default diff --git a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out index b7a81742aa9b51f3aad5be48e6212d6c94f9865b..b501f9745b9e3a60f581a19064ab3284b559c8c3 100644 --- a/ql/src/test/results/clientpositive/tez/explainuser_1.q.out +++ b/ql/src/test/results/clientpositive/tez/explainuser_1.q.out @@ -388,9 +388,9 @@ Stage-0 PartitionCols:_col0, _col1 Group By Operator [GBY_35] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_34] (rows=2 width=16) + Select Operator [SEL_34] (rows=3 width=16) Output:["_col2","_col6"] - Filter Operator [FIL_33] (rows=2 width=16) + Filter Operator [FIL_33] (rows=3 width=16) predicate:((_col1 > 0) or (_col6 >= 0)) Merge Join Operator [MERGEJOIN_52] (rows=3 width=16) Conds:RS_30._col0=RS_31._col0(Inner),Output:["_col1","_col2","_col6"] @@ -491,14 +491,14 @@ Stage-0 Output:["_col2","_col6"] Filter Operator [FIL_30] (rows=1 width=16) predicate:(((_col1 > 0) or (_col6 >= 0)) and ((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0)) - Merge Join Operator [MERGEJOIN_48] (rows=2 width=16) + Merge Join Operator [MERGEJOIN_48] (rows=3 width=16) Conds:RS_27._col0=RS_28._col0(Inner),Output:["_col1","_col2","_col6"] <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_28] PartitionCols:_col0 - Select Operator [SEL_26] (rows=5 width=71) + Select Operator [SEL_26] (rows=18 width=79) Output:["_col0","_col1"] - Filter Operator [FIL_46] (rows=5 width=71) + Filter Operator [FIL_46] (rows=18 width=79) predicate:((c_int > 0) and key is not null) TableScan [TS_24] (rows=20 width=80) default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] @@ -664,14 +664,14 @@ Stage-0 Output:["_col2","_col6"] Filter Operator [FIL_29] (rows=1 width=20) predicate:(((_col1 + _col4) >= 0) and ((_col1 > 0) or (_col6 >= 0)) and ((_col6 >= 1) or (_col2 >= 1)) and ((UDFToLong(_col6) + _col2) >= 0)) - Merge Join Operator [MERGEJOIN_42] (rows=3 width=20) + Merge Join Operator [MERGEJOIN_42] (rows=4 width=20) Conds:RS_25._col0=RS_26._col0(Outer),RS_25._col0=RS_27._col0(Right Outer),Output:["_col1","_col2","_col4","_col6"] <-Map 10 [SIMPLE_EDGE] SHUFFLE [RS_27] PartitionCols:_col0 - Select Operator [SEL_24] (rows=6 width=74) + Select Operator [SEL_24] (rows=20 width=80) Output:["_col0","_col1"] - Filter Operator [FIL_41] (rows=6 width=74) + Filter Operator [FIL_41] (rows=20 width=80) predicate:(c_int > 0) TableScan [TS_22] (rows=20 width=80) default@cbo_t3,cbo_t3,Tbl:COMPLETE,Col:COMPLETE,Output:["key","c_int"] @@ -744,9 +744,9 @@ Stage-0 PartitionCols:_col0, _col1 Group By Operator [GBY_29] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_28] (rows=2 width=16) + Select Operator [SEL_28] (rows=3 width=16) Output:["_col2","_col6"] - Filter Operator [FIL_27] (rows=2 width=16) + Filter Operator [FIL_27] (rows=3 width=16) predicate:((_col1 > 0) or (_col6 >= 0)) Merge Join Operator [MERGEJOIN_43] (rows=3 width=16) Conds:RS_24._col0=RS_25._col0(Inner),Output:["_col1","_col2","_col6"] @@ -1201,9 +1201,9 @@ Stage-0 Stage-1 Reducer 3 File Output Operator [FS_19] - Select Operator [SEL_18] (rows=14 width=101) + Select Operator [SEL_18] (rows=21 width=101) Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_17] (rows=14 width=101) + Filter Operator [FIL_17] (rows=21 width=101) predicate:((_col1 > 0) or (_col6 >= 0)) Merge Join Operator [MERGEJOIN_28] (rows=21 width=101) Conds:RS_14._col0=RS_15._col0(Inner),Output:["_col1","_col2","_col3","_col4","_col6"] @@ -1257,9 +1257,9 @@ Stage-0 Stage-1 Reducer 2 File Output Operator [FS_14] - Select Operator [SEL_13] (rows=12 width=101) + Select Operator [SEL_13] (rows=24 width=101) Output:["_col0","_col1","_col2","_col3","_col4"] - Filter Operator [FIL_12] (rows=12 width=101) + Filter Operator [FIL_12] (rows=24 width=101) predicate:(((_col1 + _col4) = 2) and ((_col1 > 0) or (_col6 >= 0)) and ((_col4 + 1) = 2)) Merge Join Operator [MERGEJOIN_19] (rows=72 width=101) Conds:RS_8._col0=RS_9._col0(Right Outer),RS_8._col0=RS_10._col0(Right Outer),Output:["_col1","_col2","_col3","_col4","_col6"] @@ -1487,9 +1487,9 @@ Stage-0 PartitionCols:_col0, _col1 Group By Operator [GBY_41] (rows=1 width=20) Output:["_col0","_col1","_col2"],aggregations:["count()"],keys:_col2, _col6 - Select Operator [SEL_40] (rows=2 width=16) + Select Operator [SEL_40] (rows=3 width=16) Output:["_col2","_col6"] - Filter Operator [FIL_39] (rows=2 width=16) + Filter Operator [FIL_39] (rows=3 width=16) predicate:((_col1 > 0) or (_col6 >= 0)) Merge Join Operator [MERGEJOIN_61] (rows=3 width=16) Conds:RS_36._col0=RS_37._col0(Inner),Output:["_col1","_col2","_col6"] diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java index 9046589b5f347d8504de86933adbb0d7290c6605..1273588776d34f416839bbe7e8918a5c076d3ab3 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java +++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java @@ -57,7 +57,7 @@ * @param gotAllData An out param - whether all the requested data was found in cache. * @return The new or modified list of DiskRange-s, where some ranges may contain cached data. */ - DiskRangeList getFileData(long fileId, DiskRangeList range, long baseOffset, + DiskRangeList getFileData(Object fileKey, DiskRangeList range, long baseOffset, DiskRangeListFactory factory, BooleanRef gotAllData); /** @@ -79,7 +79,7 @@ DiskRangeList getFileData(long fileId, DiskRangeList range, long baseOffset, * @return null if all data was put; bitmask indicating which chunks were not put otherwise; * the replacement chunks from cache are updated directly in the array. */ - long[] putFileData(long fileId, DiskRange[] ranges, MemoryBuffer[] data, long baseOffset); + long[] putFileData(Object fileKey, DiskRange[] ranges, MemoryBuffer[] data, long baseOffset); /** * Releases the buffer returned by getFileData/provided to putFileData back to cache. diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java index 3ef7abee134e8f9801c3009bdb7511b852aea51d..ddba8899b817c098490947bc3385cdea322aeb22 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java +++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/EncodedColumnBatch.java @@ -82,13 +82,6 @@ public void setIndexBaseOffset(int indexBaseOffset) { protected ColumnStreamData[][] columnData; /** Column indexes included in the batch. Correspond to columnData elements. */ protected int[] columnIxs; - // TODO: Maybe remove when solving the pooling issue. - /** Generation version necessary to sync pooling reuse with the fact that two separate threads - * operate on batches - the one that decodes them, and potential separate thread w/a "stop" call - * that cleans them up. We don't want the decode thread to use the ECB that was thrown out and - * reused, so it remembers the version and checks it after making sure no cleanup thread can ever - * get to this ECB anymore. All this sync is ONLY needed because of high level cache code. */ - public int version = Integer.MIN_VALUE; public void reset() { if (columnData == null) return; @@ -117,7 +110,7 @@ public void setAllStreamsData(int colIxMod, int colIx, ColumnStreamData[] sbs) { } public BatchKey getBatchKey() { - return batchKey; + return batchKey; // TODO#: who uses this? can we remove fileId? } public ColumnStreamData[][] getColumnData() { diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java index dc71db4fb221cc32fa603f1884688ea93f5aac74..469a3dad47c79e51d24115db469647596e7b9785 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java @@ -99,5 +99,4 @@ public Class getValueClass() { * */ public List getLiteralList(); - } diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java index be5e67ba23122bb5cddbd34217cc8abdf1bfa2cc..8c5bab25df31185673f90351645e907848181e30 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java +++ b/storage-api/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java @@ -171,7 +171,6 @@ public static void setColumnName(PredicateLeaf leaf, String newName) { } } - private final List leaves; private final ExpressionTree expression;