From 5386d3101d6f93e38d0a2e7034db29ac4b85304a Mon Sep 17 00:00:00 2001 From: kangkaisen Date: Tue, 10 Jan 2017 14:02:18 +0800 Subject: [PATCH] KYLIN-2377 Add kylin client query timeout --- .../java/org/apache/kylin/common/KylinConfigBase.java | 4 ++++ .../main/java/org/apache/kylin/cube/CubeInstance.java | 1 + .../apache/kylin/metadata/realization/IRealization.java | 5 ++++- .../java/org/apache/kylin/storage/StorageContext.java | 16 +++++++++++++++- .../kylin/storage/gtrecord/GTCubeStorageQueryBase.java | 2 ++ .../storage/gtrecord/SequentialCubeTupleIterator.java | 5 +++++ .../org/apache/kylin/storage/hybrid/HybridInstance.java | 1 + 7 files changed, 32 insertions(+), 2 deletions(-) diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index d6774ff..d5ed1a2 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -863,6 +863,10 @@ abstract public class KylinConfigBase implements Serializable { return udfMap; } + public int getQueryTimeoutSeconds() { + return Integer.parseInt(this.getOptional("kylin.query.timeout-seconds", "0")); + } + // ============================================================================ // SERVER // ============================================================================ diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 1d60575..fb9a7a7 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -178,6 +178,7 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, return sizeRecordSize; } + @Override public KylinConfig getConfig() { return config; } diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java index a0243f4..aafc0f0 100644 --- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java +++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/IRealization.java @@ -21,6 +21,7 @@ package org.apache.kylin.metadata.realization; import java.util.List; import java.util.Set; +import org.apache.kylin.common.KylinConfig; import org.apache.kylin.metadata.model.ColumnDesc; import org.apache.kylin.metadata.model.DataModelDesc; import org.apache.kylin.metadata.model.IStorageAware; @@ -42,7 +43,7 @@ public interface IRealization extends IStorageAware { public DataModelDesc getModel(); public Set getAllColumns(); - + public Set getAllColumnDescs(); public List getAllDimensions(); @@ -60,4 +61,6 @@ public interface IRealization extends IStorageAware { public long getDateRangeEnd(); public boolean supportsLimitPushDown(); + + public KylinConfig getConfig(); } diff --git a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java index ec46f83..ab0ea73 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/StorageContext.java @@ -41,6 +41,7 @@ public class StorageContext { private int finalPushDownLimit = Integer.MAX_VALUE; private boolean hasSort = false; private boolean acceptPartialResult = false; + private long deadline; private boolean exactAggregation = false; private boolean needStorageAggregation = false; @@ -123,6 +124,19 @@ public class StorageContext { } } + public long getDeadline() { + return this.deadline; + } + + public void setDeadline(IRealization realization) { + int timeout = realization.getConfig().getQueryTimeoutSeconds() * 1000; + if (timeout == 0) { + this.deadline = Long.MAX_VALUE; + } else { + this.deadline = timeout + System.currentTimeMillis(); + } + } + public void markSort() { this.hasSort = true; } @@ -202,5 +216,5 @@ public class StorageContext { public void setStorageQuery(IStorageQuery storageQuery) { this.storageQuery = storageQuery; } - + } diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java index 4fcfad1..4dbdf94 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java @@ -120,6 +120,8 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery { // set limit push down enableStorageLimitIfPossible(cuboid, groups, derivedPostAggregation, groupsD, filter, loosenedColumnD, sqlDigest.aggregations, context); + // set query deadline + context.setDeadline(cubeInstance); // set cautious threshold to prevent out of memory setThresholdIfNecessary(dimensionsD, metrics, context); diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java index c621215..c6b2c6c 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/SequentialCubeTupleIterator.java @@ -28,6 +28,7 @@ import javax.annotation.Nullable; import org.apache.kylin.common.KylinConfig; import org.apache.kylin.cube.cuboid.Cuboid; +import org.apache.kylin.gridtable.GTScanTimeoutException; import org.apache.kylin.metadata.model.FunctionDesc; import org.apache.kylin.metadata.model.TblColRef; import org.apache.kylin.metadata.tuple.ITuple; @@ -141,6 +142,10 @@ public class SequentialCubeTupleIterator implements ITupleIterator { @Override public ITuple next() { + if (scanCount % 100 == 1 && System.currentTimeMillis() > context.getDeadline()) { + throw new GTScanTimeoutException("Query Timeout!"); + } + // prevent the big query to make the Query Server OOM if (scanCount++ > SCAN_THRESHOLD) { throw new ScanOutOfLimitException("Scan count exceed the scan threshold: " + SCAN_THRESHOLD); diff --git a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java index 4f0e446..1b113ee 100644 --- a/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java +++ b/core-storage/src/main/java/org/apache/kylin/storage/hybrid/HybridInstance.java @@ -251,6 +251,7 @@ public class HybridInstance extends RootPersistentEntity implements IRealization return getType() + "[name=" + name + "]"; } + @Override public KylinConfig getConfig() { return config; } -- 2.10.1 (Apple Git-78)