diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 5cf1609..7dc9e95 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -295,6 +295,7 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal llapDaemonVarsSetLocal.add(ConfVars.LLAP_LRFU_LAMBDA.varname); llapDaemonVarsSetLocal.add(ConfVars.LLAP_CACHE_ALLOW_SYNTHETIC_FILEID.varname); llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_USE_FILEID_PATH.varname); + llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_DECODING_METRICS_PERCENTILE_INTERVALS.varname); llapDaemonVarsSetLocal.add(ConfVars.LLAP_ORC_ENABLE_TIME_COUNTERS.varname); llapDaemonVarsSetLocal.add(ConfVars.LLAP_IO_THREADPOOL_SIZE.varname); llapDaemonVarsSetLocal.add(ConfVars.LLAP_KERBEROS_PRINCIPAL.varname); @@ -2602,10 +2603,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Chooses whether query fragments will run in container or in llap"), LLAP_OBJECT_CACHE_ENABLED("hive.llap.object.cache.enabled", true, "Cache objects (plans, hashtables, etc) in llap"), - LLAP_QUEUE_METRICS_PERCENTILE_INTERVALS("hive.llap.queue.metrics.percentiles.intervals", "", + LLAP_IO_DECODING_METRICS_PERCENTILE_INTERVALS("hive.llap.io.decoding.metrics.percentiles.intervals", "30", "Comma-delimited set of integers denoting the desired rollover intervals (in seconds)\n" + - "for percentile latency metrics on the LLAP daemon producer-consumer queue.\n" + - "By default, percentile latency metrics are disabled."), + "for percentile latency metrics on the LLAP daemon IO decoding time.\n" + + "hive.llap.queue.metrics.percentiles.intervals"), LLAP_IO_THREADPOOL_SIZE("hive.llap.io.threadpool.size", 10, "Specify the number of threads to use for low-level IO thread pool."), LLAP_KERBEROS_PRINCIPAL(HIVE_LLAP_DAEMON_SERVICE_PRINCIPAL_NAME, "", diff --git a/common/src/java/org/apache/hive/common/util/FixedSizedObjectPool.java b/common/src/java/org/apache/hive/common/util/FixedSizedObjectPool.java index 600c443..3e80613 100644 --- a/common/src/java/org/apache/hive/common/util/FixedSizedObjectPool.java +++ b/common/src/java/org/apache/hive/common/util/FixedSizedObjectPool.java @@ -142,6 +142,11 @@ public void offer(T t) { tryOffer(t); } + @Override + public int size() { + return pool.length; + } + @VisibleForTesting public boolean tryOffer(T t) { if (t == null || pool.length == 0) return false; // 0 size means no-pooling case - passthru. diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/metrics/LlapMetricsSystem.java b/llap-common/src/java/org/apache/hadoop/hive/llap/metrics/LlapMetricsSystem.java new file mode 100644 index 0000000..710182d --- /dev/null +++ b/llap-common/src/java/org/apache/hadoop/hive/llap/metrics/LlapMetricsSystem.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.metrics; + +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; + +/** + * Metrics system for llap daemon. We do not use DefaultMetricsSystem here to safegaurd against + * Tez accidentally shutting it down. + */ +public enum LlapMetricsSystem { + INSTANCE; + + private AtomicReference impl = + new AtomicReference(new MetricsSystemImpl()); + + /** + * Convenience method to initialize the metrics system + * @param prefix for the metrics system configuration + * @return the metrics system instance + */ + public static MetricsSystem initialize(String prefix) { + return INSTANCE.impl.get().init(prefix); + } + + /** + * @return the metrics system object + */ + public static MetricsSystem instance() { + return INSTANCE.impl.get(); + } + + /** + * Shutdown the metrics system + */ + public static void shutdown() { + INSTANCE.impl.get().shutdown(); + } +} diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java b/llap-common/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java new file mode 100644 index 0000000..c22ce4e --- /dev/null +++ b/llap-common/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.metrics; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.UUID; + +/** + * Utility methods for metrics system. + */ +public class MetricsUtils { + private static final String LOCALHOST = "localhost"; + public static final String METRICS_PROCESS_NAME = "LlapDaemon"; + + + public static String getHostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + return LOCALHOST; + } + } + + public static String getUUID() { + return String.valueOf(UUID.randomUUID()); + } +} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java index 824ff33..9f7e5c9 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cache/BuddyAllocator.java @@ -113,6 +113,9 @@ public BuddyAllocator(boolean isDirectVal, int minAllocVal, int maxAllocVal, int this.metrics = metrics; metrics.incrAllocatedArena(); + metrics.setArenaSize(arenaSize); + metrics.setMinAllocationSize(minAllocation); + metrics.setMaxAllocationSize(maxAllocation); } // TODO: would it make sense to return buffers asynchronously? diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index a0250cb..33b41e8 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -194,6 +194,11 @@ public LlapDaemon(Configuration daemonConf, int numExecutors, long executorMemor String sessionId = MetricsUtils.getUUID(); daemonConf.set("llap.daemon.metrics.sessionid", sessionId); this.metrics = LlapDaemonExecutorMetrics.create(displayName, sessionId, numExecutors); + this.metrics.setMemoryPerInstance(executorMemoryBytes); + this.metrics.setCacheMemoryPerInstance(ioMemoryBytes); + this.metrics.setJvmMaxMemory(maxJvmMemory); + this.metrics.setWaitQueueSize(waitQueueSize); + this.metrics.setRpcNumHandlers(numHandlers); metrics.getJvmMetrics().setPauseMonitor(pauseMonitor); this.llapDaemonInfoBean = MBeans.register("LlapDaemon", "LlapDaemonInfo", this); LOG.info("Started LlapMetricsSystem with displayName: " + displayName + diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java index 36f8dec..0e07773 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapIoImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.llap.io.api.impl; import java.io.IOException; +import java.util.Arrays; import java.util.concurrent.Executors; import javax.management.ObjectName; @@ -44,7 +45,7 @@ import org.apache.hadoop.hive.llap.io.decode.OrcColumnVectorProducer; import org.apache.hadoop.hive.llap.io.metadata.OrcMetadataCache; import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics; -import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics; import org.apache.hadoop.hive.llap.metrics.MetricsUtils; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.io.NullWritable; @@ -66,7 +67,7 @@ private final ColumnVectorProducer cvp; private final ListeningExecutorService executor; private LlapDaemonCacheMetrics cacheMetrics; - private LlapDaemonQueueMetrics queueMetrics; + private LlapDaemonIOMetrics ioMetrics; private ObjectName buddyAllocatorMXBean; private Allocator allocator; @@ -75,15 +76,27 @@ private LlapIoImpl(Configuration conf) throws IOException { boolean useLowLevelCache = LlapIoImpl.MODE_CACHE.equalsIgnoreCase(ioMode), useAllocOnly = !useLowLevelCache && LlapIoImpl.MODE_ALLOCATOR.equalsIgnoreCase(ioMode); LOG.info("Initializing LLAP IO in {} mode", ioMode); - String displayName = "LlapDaemonCacheMetrics-" + MetricsUtils.getHostName(); String sessionId = conf.get("llap.daemon.metrics.sessionid"); this.cacheMetrics = LlapDaemonCacheMetrics.create(displayName, sessionId); - displayName = "LlapDaemonQueueMetrics-" + MetricsUtils.getHostName(); - int[] intervals = conf.getInts(String.valueOf( - HiveConf.ConfVars.LLAP_QUEUE_METRICS_PERCENTILE_INTERVALS)); - this.queueMetrics = LlapDaemonQueueMetrics.create(displayName, sessionId, intervals); + displayName = "LlapDaemonIOMetrics-" + MetricsUtils.getHostName(); + String[] strIntervals = HiveConf.getTrimmedStringsVar(conf, + HiveConf.ConfVars.LLAP_IO_DECODING_METRICS_PERCENTILE_INTERVALS); + int[] intervals = null; + if (strIntervals != null) { + intervals = new int[strIntervals.length]; + try { + for (int i = 0; i < strIntervals.length; i++) { + intervals[i] = Integer.valueOf(strIntervals[i]); + } + } catch (NumberFormatException e) { + LOG.warn("Ignoring {} because of invalid values", + HiveConf.ConfVars.LLAP_IO_DECODING_METRICS_PERCENTILE_INTERVALS); + // ignore + } + } + this.ioMetrics = LlapDaemonIOMetrics.create(displayName, sessionId, intervals); LOG.info("Started llap daemon metrics with displayName: {} sessionId: {}", displayName, sessionId); @@ -123,10 +136,10 @@ private LlapIoImpl(Configuration conf) throws IOException { int numThreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.LLAP_IO_THREADPOOL_SIZE); executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(numThreads, new ThreadFactoryBuilder().setNameFormat("IO-Elevator-Thread-%d").setDaemon(true).build())); - + ioMetrics.setIoThreadPoolSize(numThreads); // TODO: this should depends on input format and be in a map, or something. this.cvp = new OrcColumnVectorProducer( - metadataCache, orcCache, bufferManager, conf, cacheMetrics, queueMetrics); + metadataCache, orcCache, bufferManager, conf, cacheMetrics, ioMetrics); LOG.info("LLAP IO initialized"); registerMXBeans(); @@ -143,14 +156,6 @@ private void registerMXBeans() { return new LlapInputFormat(sourceInputFormat, cvp, executor); } - public LlapDaemonCacheMetrics getCacheMetrics() { - return cacheMetrics; - } - - public LlapDaemonQueueMetrics getQueueMetrics() { - return queueMetrics; - } - @Override public void close() { LOG.info("Closing LlapIoImpl.."); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java index 137acb0..0ba7c09 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/EncodedDataConsumer.java @@ -23,7 +23,7 @@ import org.apache.hadoop.hive.common.io.encoded.EncodedColumnBatch; import org.apache.hadoop.hive.llap.ConsumerFeedback; import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch; -import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics; import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; import org.apache.hive.common.util.FixedSizedObjectPool; @@ -33,15 +33,15 @@ private ConsumerFeedback upstreamFeedback; private final Consumer downstreamConsumer; private Callable readCallable; - private final LlapDaemonQueueMetrics queueMetrics; + private final LlapDaemonIOMetrics ioMetrics; // Note that the pool is per EDC - within EDC, CVBs are expected to have the same schema. private final static int CVB_POOL_SIZE = 128; protected final FixedSizedObjectPool cvbPool; public EncodedDataConsumer(Consumer consumer, final int colCount, - LlapDaemonQueueMetrics queueMetrics) { + LlapDaemonIOMetrics ioMetrics) { this.downstreamConsumer = consumer; - this.queueMetrics = queueMetrics; + this.ioMetrics = ioMetrics; cvbPool = new FixedSizedObjectPool(CVB_POOL_SIZE, new Pool.PoolObjectHelper() { @Override @@ -53,6 +53,7 @@ public void resetBeforeOffer(ColumnVectorBatch t) { // Don't reset anything, we are reusing column vectors. } }); + this.ioMetrics.setColumnVectorBatchPoolSize(cvbPool.size()); } public void init(ConsumerFeedback upstreamFeedback, @@ -61,6 +62,10 @@ public void init(ConsumerFeedback upstreamFeedback, this.readCallable = readCallable; } + public LlapDaemonIOMetrics getIOMetrics() { + return ioMetrics; + } + @Override public Callable getReadCallable() { return readCallable; @@ -75,7 +80,7 @@ public void consumeData(BatchType data) { long start = System.currentTimeMillis(); decodeBatch(data, downstreamConsumer); long end = System.currentTimeMillis(); - queueMetrics.addProcessingTime(end - start); + ioMetrics.addDecodeBatchTime(end - start); returnSourceData(data); } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java index 024c485..7db519c 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcColumnVectorProducer.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hive.llap.io.encoded.OrcEncodedDataReader; import org.apache.hadoop.hive.llap.io.metadata.OrcMetadataCache; import org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheMetrics; -import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics; import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; import org.apache.hadoop.mapred.FileSplit; @@ -44,11 +44,11 @@ private final Configuration conf; private boolean _skipCorrupt; // TODO: get rid of this private LlapDaemonCacheMetrics cacheMetrics; - private LlapDaemonQueueMetrics queueMetrics; + private LlapDaemonIOMetrics ioMetrics; public OrcColumnVectorProducer(OrcMetadataCache metadataCache, LowLevelCacheImpl lowLevelCache, BufferUsageManager bufferManager, - Configuration conf, LlapDaemonCacheMetrics metrics, LlapDaemonQueueMetrics queueMetrics) { + Configuration conf, LlapDaemonCacheMetrics cacheMetrics, LlapDaemonIOMetrics ioMetrics) { LlapIoImpl.LOG.info("Initializing ORC column vector producer"); this.metadataCache = metadataCache; @@ -56,8 +56,8 @@ public OrcColumnVectorProducer(OrcMetadataCache metadataCache, this.bufferManager = bufferManager; this.conf = conf; this._skipCorrupt = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_ORC_SKIP_CORRUPT_DATA); - this.cacheMetrics = metrics; - this.queueMetrics = queueMetrics; + this.cacheMetrics = cacheMetrics; + this.ioMetrics = ioMetrics; } @Override @@ -67,7 +67,7 @@ public ReadPipeline createReadPipeline( QueryFragmentCounters counters) { cacheMetrics.incrCacheReadRequests(); OrcEncodedDataConsumer edc = new OrcEncodedDataConsumer(consumer, columnIds.size(), - _skipCorrupt, counters, queueMetrics); + _skipCorrupt, counters, ioMetrics); OrcEncodedDataReader reader = new OrcEncodedDataReader(lowLevelCache, bufferManager, metadataCache, conf, split, columnIds, sarg, columnNames, edc, counters); edc.init(reader, reader); diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java index 7ee263d..0651557 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/decode/OrcEncodedDataConsumer.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hive.llap.io.api.impl.ColumnVectorBatch; import org.apache.hadoop.hive.llap.io.metadata.OrcFileMetadata; import org.apache.hadoop.hive.llap.io.metadata.OrcStripeMetadata; -import org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueMetrics; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics; import org.apache.hadoop.hive.ql.exec.vector.ColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.orc.CompressionCodec; @@ -52,8 +52,8 @@ public OrcEncodedDataConsumer( Consumer consumer, int colCount, boolean skipCorrupt, - QueryFragmentCounters counters, LlapDaemonQueueMetrics queueMetrics) { - super(consumer, colCount, queueMetrics); + QueryFragmentCounters counters, LlapDaemonIOMetrics ioMetrics) { + super(consumer, colCount, ioMetrics); // TODO: get rid of this this.skipCorrupt = skipCorrupt; this.counters = counters; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index fb0867d..eb953c7 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -26,6 +26,7 @@ import java.util.List; import org.apache.hadoop.hive.llap.counters.LlapIOCounters; +import org.apache.hadoop.hive.llap.metrics.LlapDaemonIOMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -66,7 +67,6 @@ import org.apache.hadoop.hive.ql.io.orc.encoded.Reader; import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl; import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.SargApplier; -import org.apache.hadoop.hive.ql.io.orc.encoded.Consumer; import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedOrcFile; import org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReader; import org.apache.hadoop.hive.ql.io.orc.encoded.OrcBatchKey; @@ -180,6 +180,9 @@ public OrcEncodedDataReader(LowLevelCache lowLevelCache, BufferUsageManager buff } catch (IOException e) { throw new RuntimeException(e); } + LlapDaemonIOMetrics ioMetrics = consumer.getIOMetrics(); + ioMetrics.setColumnStreamDataPoolSize(CSD_POOL.size()); + ioMetrics.setEncodedColumnBatchPoolSize(ECB_POOL.size()); } @Override diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheInfo.java index 8f0e9d8..191345e 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheInfo.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheInfo.java @@ -34,7 +34,10 @@ CacheHitRatio("Ratio of disk ranges cached vs requested"), CacheReadRequests("Number of disk range requests to cache"), CacheAllocatedArena("Number of arenas allocated"), - CacheNumLockedBuffers("Number of locked buffers in cache"); + CacheNumLockedBuffers("Number of locked buffers in cache"), + CacheArenaSize("Size of arena used by allocator"), + CacheMinAllocationSize("Minimum allocation size used by allocator"), + CacheMaxAllocationSize("Maximum allocation size used by allocator"); private final String desc; diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java index b89c6c4..bb76da5 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java @@ -18,11 +18,14 @@ package org.apache.hadoop.hive.llap.metrics; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheAllocatedArena; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheArenaSize; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheCapacityRemaining; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheCapacityTotal; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheCapacityUsed; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheHitBytes; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheHitRatio; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheMaxAllocationSize; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheMinAllocationSize; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheNumLockedBuffers; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheReadRequests; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonCacheInfo.CacheRequestedBytes; @@ -65,6 +68,12 @@ MutableCounterLong cacheAllocatedArena; @Metric MutableCounterLong cacheNumLockedBuffers; + @Metric + MutableGaugeLong arenaSize; + @Metric + MutableGaugeLong minAllocationSize; + @Metric + MutableGaugeLong maxAllocationSize; private LlapDaemonCacheMetrics(String name, String sessionId) { this.name = name; @@ -106,6 +115,18 @@ public void incrCacheNumLockedBuffers() { cacheNumLockedBuffers.incr(); } + public void setArenaSize(long value) { + arenaSize.set(value); + } + + public void setMinAllocationSize(long value) { + minAllocationSize.set(value); + } + + public void setMaxAllocationSize(long value) { + maxAllocationSize.set(value); + } + public void decrCacheNumLockedBuffers() { cacheNumLockedBuffers.incr(-1); } @@ -145,7 +166,10 @@ private void getCacheStats(MetricsRecordBuilder rb) { .addCounter(CacheHitBytes, cacheHitBytes.value()) .addCounter(CacheAllocatedArena, cacheAllocatedArena.value()) .addCounter(CacheNumLockedBuffers, cacheNumLockedBuffers.value()) - .addGauge(CacheHitRatio, cacheHitRatio); + .addGauge(CacheHitRatio, cacheHitRatio) + .addGauge(CacheArenaSize, arenaSize.value()) + .addGauge(CacheMinAllocationSize, minAllocationSize.value()) + .addGauge(CacheMaxAllocationSize, maxAllocationSize.value()); } } diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorInfo.java index e4739dc..941d926 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorInfo.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorInfo.java @@ -26,6 +26,12 @@ */ public enum LlapDaemonExecutorInfo implements MetricsInfo { ExecutorMetrics("Llap daemon cache related metrics"), + ExecutorThreadCountPerInstance("Total number of executor threads per node"), + ExecutorMemoryPerInstance("Total memory for executors per node in bytes"), + ExecutorCacheMemoryPerInstance("Total Cache memory per node in bytes"), + ExecutorJvmMaxMemory("Max memory available for JVM in bytes"), + ExecutorWaitQueueSize("Size of wait queue per node"), + ExecutorRpcNumHandlers("Number of RPC handlers per node"), ExecutorThreadCPUTime("Cpu time in nanoseconds"), ExecutorThreadUserTime("User time in nanoseconds"), ExecutorTotalRequestsHandled("Total number of requests handled by the container"), diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java index 7919200..894880f 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java @@ -17,8 +17,13 @@ */ package org.apache.hadoop.hive.llap.metrics; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorCacheMemoryPerInstance; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorJvmMaxMemory; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorMemoryPerInstance; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorNumQueuedRequests; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorRpcNumHandlers; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorThreadCPUTime; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorThreadCountPerInstance; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorThreadUserTime; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorTotalAskedToDie; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorTotalExecutionFailure; @@ -26,6 +31,7 @@ import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorTotalRequestsHandled; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorTotalSuccess; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorMetrics; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.ExecutorWaitQueueSize; import static org.apache.hadoop.hive.llap.metrics.LlapDaemonExecutorInfo.PreemptionTimeLost; import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName; import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId; @@ -46,6 +52,7 @@ import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableGaugeInt; import org.apache.hadoop.metrics2.lib.MutableGaugeLong; import org.apache.hadoop.metrics2.source.JvmMetrics; @@ -78,7 +85,16 @@ MutableCounterLong executorTotalExecutionFailed; @Metric MutableCounterLong preemptionTimeLost; - + @Metric + MutableGaugeLong cacheMemoryPerInstance; + @Metric + MutableGaugeLong memoryPerInstance; + @Metric + MutableGaugeLong jvmMaxMemory; + @Metric + MutableGaugeInt waitQueueSize; + @Metric + MutableGaugeInt rpcNumHandlers; private LlapDaemonExecutorMetrics(String displayName, JvmMetrics jm, String sessionId, int numExecutors) { @@ -151,6 +167,25 @@ public void incrExecutorTotalKilled() { executorTotalIKilled.incr(); } + public void setCacheMemoryPerInstance(long value) { + cacheMemoryPerInstance.set(value); + } + + public void setMemoryPerInstance(long value) { + memoryPerInstance.set(value); + } + + public void setJvmMaxMemory(long value) { + jvmMaxMemory.set(value); + } + + public void setWaitQueueSize(int size) { + waitQueueSize.set(size); + } + + public void setRpcNumHandlers(int numHandlers) { + rpcNumHandlers.set(numHandlers); + } private void getExecutorStats(MetricsRecordBuilder rb) { updateThreadMetrics(rb); @@ -160,7 +195,13 @@ private void getExecutorStats(MetricsRecordBuilder rb) { .addCounter(ExecutorTotalSuccess, executorTotalSuccess.value()) .addCounter(ExecutorTotalExecutionFailure, executorTotalExecutionFailed.value()) .addCounter(ExecutorTotalInterrupted, executorTotalIKilled.value()) - .addCounter(PreemptionTimeLost, preemptionTimeLost.value()); + .addCounter(PreemptionTimeLost, preemptionTimeLost.value()) + .addGauge(ExecutorThreadCountPerInstance, numExecutors) + .addGauge(ExecutorMemoryPerInstance, memoryPerInstance.value()) + .addGauge(ExecutorCacheMemoryPerInstance, cacheMemoryPerInstance.value()) + .addGauge(ExecutorJvmMaxMemory, jvmMaxMemory.value()) + .addGauge(ExecutorWaitQueueSize, waitQueueSize.value()) + .addGauge(ExecutorRpcNumHandlers, rpcNumHandlers.value()); } private void updateThreadMetrics(MetricsRecordBuilder rb) { diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonIOInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonIOInfo.java new file mode 100644 index 0000000..79f004b --- /dev/null +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonIOInfo.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.metrics; + +import org.apache.hadoop.metrics2.MetricsInfo; + +import com.google.common.base.Objects; + +/** + * Llap daemon I/O elevator metrics + */ +public enum LlapDaemonIOInfo implements MetricsInfo { + IOMetrics("Llap daemon I/O elevator metrics"), + IoThreadPoolSize("Size of the thread pool used by IO elevator"), + EncodedColumnBatchPoolSize("Size of the object pool that stores encoded column batches"), + ColumnStreamDataPoolSize("Size of the object pool that stores column stream data"), + ColumnVectorBatchPoolSize("Size of the object pool that stores column vector batches"), + PercentileDecodingTime("Percentile decoding time for encoded column batch"), + MaxDecodingTime("Max time for decoding an encoded column batch"); + + private final String desc; + + LlapDaemonIOInfo(String desc) { + this.desc = desc; + } + + @Override + public String description() { + return desc; + } + + @Override + public String toString() { + return Objects.toStringHelper(this) + .add("name", name()).add("description", desc) + .toString(); + } +} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonIOMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonIOMetrics.java new file mode 100644 index 0000000..f3def75 --- /dev/null +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonIOMetrics.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.llap.metrics; + +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonIOInfo.ColumnStreamDataPoolSize; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonIOInfo.ColumnVectorBatchPoolSize; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonIOInfo.EncodedColumnBatchPoolSize; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonIOInfo.IoThreadPoolSize; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonIOInfo.MaxDecodingTime; +import static org.apache.hadoop.hive.llap.metrics.LlapDaemonIOInfo.IOMetrics; +import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName; +import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId; + +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.annotation.Metric; +import org.apache.hadoop.metrics2.annotation.Metrics; +import org.apache.hadoop.metrics2.lib.MetricsRegistry; +import org.apache.hadoop.metrics2.lib.MutableGaugeInt; +import org.apache.hadoop.metrics2.lib.MutableGaugeLong; +import org.apache.hadoop.metrics2.lib.MutableQuantiles; +import org.apache.hadoop.metrics2.lib.MutableRate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * + */ +@Metrics(about = "LlapDaemon IO Metrics", context = "io") +public class LlapDaemonIOMetrics implements MetricsSource { + protected static final Logger LOG = LoggerFactory.getLogger(LlapDaemonIOMetrics.class); + private final String name; + private final String sessionId; + private final MetricsRegistry registry; + private long maxTime = Long.MIN_VALUE; + + @Metric + MutableGaugeInt encodedColumnBatchPoolSize; + @Metric + MutableGaugeInt columnStreamDataPoolSize; + @Metric + MutableGaugeInt columnVectorBatchPool; + @Metric + MutableGaugeInt ioThreadPoolSize; + @Metric + MutableRate rateOfDecoding; + final MutableQuantiles[] decodingTimes; + @Metric + MutableGaugeLong maxDecodingTime; + + private LlapDaemonIOMetrics(String displayName, String sessionId, int[] intervals) { + this.name = displayName; + this.sessionId = sessionId; + this.registry = new MetricsRegistry("LlapDaemonIORegistry"); + this.registry.tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME).tag(SessionId, sessionId); + + final int len = intervals == null ? 0 : intervals.length; + this.decodingTimes = new MutableQuantiles[len]; + for (int i=0; i maxTime) { + maxTime = latency; + maxDecodingTime.set(maxTime); + } + for (MutableQuantiles q : decodingTimes) { + q.add(latency); + } + } + + private void getIoStats(MetricsRecordBuilder rb) { + rb.addGauge(EncodedColumnBatchPoolSize, encodedColumnBatchPoolSize.value()) + .addGauge(ColumnStreamDataPoolSize, columnStreamDataPoolSize.value()) + .addGauge(ColumnVectorBatchPoolSize, columnVectorBatchPool.value()) + .addGauge(IoThreadPoolSize, ioThreadPoolSize.value()) + .addGauge(MaxDecodingTime, maxDecodingTime.value()); + rateOfDecoding.snapshot(rb, true); + + for (MutableQuantiles q : decodingTimes) { + q.snapshot(rb, true); + } + } + +} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueInfo.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueInfo.java deleted file mode 100644 index 7df7877..0000000 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueInfo.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.llap.metrics; - -import org.apache.hadoop.metrics2.MetricsInfo; - -import com.google.common.base.Objects; - -/** - * Llap daemon producer / consumer queue related metrics. - */ -public enum LlapDaemonQueueInfo implements MetricsInfo { - QueueMetrics("Llap daemon producer/consumer queue related metrics"), - QueueSize("Size of the queue used by producer and consumer"), - PercentileProcessingTime("Percentiles processing time for an element from queue"), - MaxProcessingTime("Max processing time for an element from queue so far"); - - private final String desc; - - LlapDaemonQueueInfo(String desc) { - this.desc = desc; - } - - @Override - public String description() { - return desc; - } - - @Override - public String toString() { - return Objects.toStringHelper(this) - .add("name", name()).add("description", desc) - .toString(); - } -} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java deleted file mode 100644 index 10a0124..0000000 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java +++ /dev/null @@ -1,116 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.llap.metrics; - -import static org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueInfo.MaxProcessingTime; -import static org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueInfo.QueueMetrics; -import static org.apache.hadoop.hive.llap.metrics.LlapDaemonQueueInfo.QueueSize; -import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName; -import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId; - -import org.apache.hadoop.metrics2.MetricsCollector; -import org.apache.hadoop.metrics2.MetricsRecordBuilder; -import org.apache.hadoop.metrics2.MetricsSource; -import org.apache.hadoop.metrics2.MetricsSystem; -import org.apache.hadoop.metrics2.annotation.Metric; -import org.apache.hadoop.metrics2.annotation.Metrics; -import org.apache.hadoop.metrics2.lib.MetricsRegistry; -import org.apache.hadoop.metrics2.lib.MutableGaugeInt; -import org.apache.hadoop.metrics2.lib.MutableGaugeLong; -import org.apache.hadoop.metrics2.lib.MutableQuantiles; -import org.apache.hadoop.metrics2.lib.MutableRate; - -/** - * - */ -@Metrics(about = "LlapDaemon Queue Metrics", context = "queue") -public class LlapDaemonQueueMetrics implements MetricsSource { - private final String name; - private final String sessionId; - private final MetricsRegistry registry; - private long maxTime = Long.MIN_VALUE; - - @Metric - MutableGaugeInt queueSize; - @Metric - MutableRate rateOfProcessing; - final MutableQuantiles[] processingTimes; - @Metric - MutableGaugeLong maxProcessingTime; - - private LlapDaemonQueueMetrics(String displayName, String sessionId, int[] intervals) { - this.name = displayName; - this.sessionId = sessionId; - this.registry = new MetricsRegistry("LlapDaemonQueueRegistry"); - this.registry.tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME).tag(SessionId, sessionId); - - final int len = intervals == null ? 0 : intervals.length; - this.processingTimes = new MutableQuantiles[len]; - for (int i=0; i maxTime) { - maxTime = latency; - maxProcessingTime.set(maxTime); - } - for (MutableQuantiles q : processingTimes) { - q.add(latency); - } - } - - private void getQueueStats(MetricsRecordBuilder rb) { - rb.addGauge(QueueSize, queueSize.value()) - .addGauge(MaxProcessingTime, maxProcessingTime.value()) - .addGauge(MaxProcessingTime, maxProcessingTime.value()); - rateOfProcessing.snapshot(rb, true); - - for (MutableQuantiles q : processingTimes) { - q.snapshot(rb, true); - } - } -} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapMetricsSystem.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapMetricsSystem.java deleted file mode 100644 index 710182d..0000000 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapMetricsSystem.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.llap.metrics; - -import java.util.concurrent.atomic.AtomicReference; - -import org.apache.hadoop.metrics2.MetricsSystem; -import org.apache.hadoop.metrics2.impl.MetricsSystemImpl; - -/** - * Metrics system for llap daemon. We do not use DefaultMetricsSystem here to safegaurd against - * Tez accidentally shutting it down. - */ -public enum LlapMetricsSystem { - INSTANCE; - - private AtomicReference impl = - new AtomicReference(new MetricsSystemImpl()); - - /** - * Convenience method to initialize the metrics system - * @param prefix for the metrics system configuration - * @return the metrics system instance - */ - public static MetricsSystem initialize(String prefix) { - return INSTANCE.impl.get().init(prefix); - } - - /** - * @return the metrics system object - */ - public static MetricsSystem instance() { - return INSTANCE.impl.get(); - } - - /** - * Shutdown the metrics system - */ - public static void shutdown() { - INSTANCE.impl.get().shutdown(); - } -} diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java deleted file mode 100644 index c22ce4e..0000000 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.llap.metrics; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.UUID; - -/** - * Utility methods for metrics system. - */ -public class MetricsUtils { - private static final String LOCALHOST = "localhost"; - public static final String METRICS_PROCESS_NAME = "LlapDaemon"; - - - public static String getHostName() { - try { - return InetAddress.getLocalHost().getHostName(); - } catch (UnknownHostException e) { - return LOCALHOST; - } - } - - public static String getUUID() { - return String.valueOf(UUID.randomUUID()); - } -} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java index f4cfa53..8ccedb7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/encoded/EncodedReaderImpl.java @@ -1317,10 +1317,16 @@ public void resetBeforeOffer(CacheChunk t) { /** Pool factory that is used if another one isn't specified - just creates the objects. */ private static class NoopPoolFactory implements PoolFactory { @Override - public Pool createPool(int size, final PoolObjectHelper helper) { + public Pool createPool(final int size, final PoolObjectHelper helper) { return new Pool() { public void offer(T t) { } + + @Override + public int size() { + return size; + } + public T take() { return helper.create(); } diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java b/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java index e41a515..272bbdd 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java +++ b/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java @@ -29,4 +29,5 @@ T take(); void offer(T t); + int size(); } \ No newline at end of file