From 3cc825e091d9e6d9c806259528d870994d5c7bab Mon Sep 17 00:00:00 2001 From: Mohit Kataria Date: Thu, 4 Apr 2019 19:22:18 +0530 Subject: [PATCH] OAK-8148:[Indexing] Implement sling metric to calculate number of slow queries(relative to all queries) --- .../oak/query/QueryEngineSettings.java | 40 +++++++++---------- .../oak/query/ast/SelectorImpl.java | 2 +- .../oak/query/stats/QueryStatsData.java | 29 +++++++++++++- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java index f7593abe69..118138bbb9 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryEngineSettings.java @@ -29,16 +29,16 @@ import org.apache.jackrabbit.oak.stats.StatisticsProvider; * Settings of the query engine. */ public class QueryEngineSettings implements QueryEngineSettingsMBean, QueryLimits { - + /** * the flag used to turn on/off the optimisations on top of the {@code org.apache.jackrabbit.oak.query.Query} object. * {@code -Doak.query.sql2optimisation} */ public static final String SQL2_OPTIMISATION_FLAG = "oak.query.sql2optimisation"; - + public static final String SQL2_OPTIMISATION_FLAG_2 = "oak.query.sql2optimisation2"; - - public static final boolean SQL2_OPTIMIZATION_2 = + + public static final boolean SQL2_OPTIMIZATION_2 = Boolean.parseBoolean(System.getProperty(SQL2_OPTIMISATION_FLAG_2, "true")); public static final String OAK_QUERY_LIMIT_IN_MEMORY = "oak.queryLimitInMemory"; @@ -59,24 +59,24 @@ public class QueryEngineSettings implements QueryEngineSettingsMBean, QueryLimit private static final boolean DEFAULT_FULL_TEXT_COMPARISON_WITHOUT_INDEX = Boolean.getBoolean("oak.queryFullTextComparisonWithoutIndex"); - + private long limitInMemory = DEFAULT_QUERY_LIMIT_IN_MEMORY; - + private long limitReads = DEFAULT_QUERY_LIMIT_READS; - + private boolean failTraversal = DEFAULT_FAIL_TRAVERSAL; - - private boolean fullTextComparisonWithoutIndex = + + private boolean fullTextComparisonWithoutIndex = DEFAULT_FULL_TEXT_COMPARISON_WITHOUT_INDEX; - - private boolean sql2Optimisation = + + private boolean sql2Optimisation = Boolean.parseBoolean(System.getProperty(SQL2_OPTIMISATION_FLAG, "true")); private static final String OAK_FAST_QUERY_SIZE = "oak.fastQuerySize"; public static final boolean DEFAULT_FAST_QUERY_SIZE = Boolean.getBoolean(OAK_FAST_QUERY_SIZE); private boolean fastQuerySize = DEFAULT_FAST_QUERY_SIZE; - private QueryStatsMBeanImpl queryStats = new QueryStatsMBeanImpl(this); + private final QueryStatsMBeanImpl queryStats = new QueryStatsMBeanImpl(this); /** * StatisticsProvider used to record query side metrics. @@ -95,22 +95,22 @@ public class QueryEngineSettings implements QueryEngineSettingsMBean, QueryLimit public long getLimitInMemory() { return limitInMemory; } - + @Override public void setLimitInMemory(long limitInMemory) { this.limitInMemory = limitInMemory; } - + @Override public long getLimitReads() { return limitReads; } - + @Override public void setLimitReads(long limitReads) { this.limitReads = limitReads; } - + @Override public boolean getFailTraversal() { return failTraversal; @@ -135,11 +135,11 @@ public class QueryEngineSettings implements QueryEngineSettingsMBean, QueryLimit public void setFullTextComparisonWithoutIndex(boolean fullTextComparisonWithoutIndex) { this.fullTextComparisonWithoutIndex = fullTextComparisonWithoutIndex; } - + public boolean getFullTextComparisonWithoutIndex() { return fullTextComparisonWithoutIndex; } - + public boolean isSql2Optimisation() { return sql2Optimisation; } @@ -147,7 +147,7 @@ public class QueryEngineSettings implements QueryEngineSettingsMBean, QueryLimit public QueryStatsMBean getQueryStats() { return queryStats; } - + public QueryStatsReporter getQueryStatsReporter() { return queryStats; } @@ -167,5 +167,5 @@ public class QueryEngineSettings implements QueryEngineSettingsMBean, QueryLimit ", fastQuerySize=" + fastQuerySize + '}'; } - + } \ No newline at end of file diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java index 10e64a6627..b5f2ce8410 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java @@ -503,7 +503,7 @@ public class SelectorImpl extends SourceImpl { private boolean nextInternal() { while (cursor != null && cursor.hasNext()) { scanCount++; - query.getQueryExecutionStats().scan(1, scanCount); + query.getQueryExecutionStats().scan(1, scanCount, query.getSettings()); currentRow = cursor.next(); if (isParent) { // we must not check whether the _parent_ is readable diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsData.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsData.java index 05d675a442..c3b38c1afb 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsData.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/stats/QueryStatsData.java @@ -17,6 +17,10 @@ package org.apache.jackrabbit.oak.query.stats; import org.apache.jackrabbit.oak.commons.json.JsopBuilder; +import org.apache.jackrabbit.oak.query.QueryEngineSettings; +import org.apache.jackrabbit.oak.stats.CounterStats; +import org.apache.jackrabbit.oak.stats.HistogramStats; +import org.apache.jackrabbit.oak.stats.StatsOptions; public class QueryStatsData { @@ -43,6 +47,8 @@ public class QueryStatsData { private long readNanos; private long maxTimeNanos; private boolean captureStackTraces; + private boolean isSlowQuery = false; + private boolean updateTotalQueryHistogram = true; public QueryStatsData(String query, String language) { this.query = query; @@ -138,6 +144,12 @@ public class QueryStatsData { public class QueryExecutionStats { long time; + private final long SLOW_QUERY_HISTOGRAM = 1; + private final long TOTAL_QUERY_HISTOGRAM = 0; + private final String SLOW_QUERY_PERCENTILE_METRICS_NAME = "SLOW_QUERY_PERCENTILE_METRICS"; + private final String SLOW_QUERY_COUNT_NAME = "SLOW_QUERY_COUNT"; + private final int SLOW_QUERY_LIMIT_SCANNED = + Integer.getInteger("oak.query.slowScanLimit", 100000); public void execute(long nanos) { QueryRecorder.record(query, internal); @@ -171,10 +183,23 @@ public class QueryStatsData { maxTimeNanos = Math.max(maxTimeNanos, time); } - public void scan(long count, long max) { + public void scan(long count, long max, QueryEngineSettings queryEngineSettings) { totalRowsScanned += count; maxRowsScanned = Math.max(maxRowsScanned, max); - } + long maxScannedLimit = Math.min(SLOW_QUERY_LIMIT_SCANNED, queryEngineSettings.getLimitReads()); + if (updateTotalQueryHistogram) { + updateTotalQueryHistogram = false; + HistogramStats histogramStats = queryEngineSettings.getStatisticsProvider().getHistogram(SLOW_QUERY_PERCENTILE_METRICS_NAME, StatsOptions.METRICS_ONLY); + histogramStats.update(TOTAL_QUERY_HISTOGRAM); + } + if (totalRowsScanned >= maxScannedLimit && !isSlowQuery) { + isSlowQuery = true; + HistogramStats histogramStats = queryEngineSettings.getStatisticsProvider().getHistogram(SLOW_QUERY_PERCENTILE_METRICS_NAME, StatsOptions.METRICS_ONLY); + histogramStats.update(SLOW_QUERY_HISTOGRAM); + CounterStats slowQueryCounter = queryEngineSettings.getStatisticsProvider().getCounterStats(SLOW_QUERY_COUNT_NAME, StatsOptions.METRICS_ONLY); + slowQueryCounter.inc(); + } + } } } \ No newline at end of file -- 2.17.1